Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : /*!
12 : * \file kernel_base_check.cpp
13 : * \brief
14 : */
15 : #include "kernel_base_check.h"
16 : #include "kernel_check_params.h"
17 : #include "kernel_utils.h"
18 : #include "model/model_factory_mask.h"
19 :
20 : namespace AscendC {
21 : namespace check {
22 : constexpr const uint64_t CONST_UINT32_MAX = 0xffffffff;
23 :
24 : // extract total element num in each repeat. Used in normal mode
25 96 : uint64_t GetMaskLength(std::vector<uint64_t>& maskArray, const uint32_t dtypeSize)
26 : {
27 96 : uint64_t maskLen = 0;
28 96 : if (maskArray[static_cast<int32_t>(CommonParams::MASK_HIGH_IDX)] == 0) {
29 : // get last element index in mask_low_idx
30 0 : for (uint32_t i = 0; i < static_cast<uint32_t>(CommonParams::MASK_MAX_ELE_LEN); ++i) {
31 0 : if ((maskArray[static_cast<int32_t>(CommonParams::MASK_LOW_IDX)] & (CONST_MASK_VALUE >> i)) != 0) {
32 0 : maskLen = static_cast<uint64_t>(CommonParams::MASK_MAX_ELE_LEN) - static_cast<uint64_t>(i);
33 0 : break;
34 : }
35 : }
36 : } else {
37 : // get last element index in mask_high_idx, then add 64
38 96 : for (uint32_t i = 0; i < static_cast<uint32_t>(CommonParams::MASK_MAX_ELE_LEN); ++i) {
39 96 : if ((maskArray[static_cast<int32_t>(CommonParams::MASK_HIGH_IDX)] & (CONST_MASK_VALUE >> i)) != 0) {
40 96 : maskLen = static_cast<uint64_t>(CommonParams::MASK_MAX_ELE_LEN) +
41 96 : static_cast<uint64_t>(CommonParams::MASK_MAX_ELE_LEN) - static_cast<uint64_t>(i);
42 96 : break;
43 : }
44 : }
45 : }
46 96 : if (dtypeSize >= sizeof(uint32_t)) {
47 0 : uint64_t maxElePerRep = DEFAULT_BLOCK_SIZE / dtypeSize;
48 0 : maskLen = (maskLen >= maxElePerRep) ? maxElePerRep : maskLen;
49 : }
50 96 : return maskLen;
51 : }
52 :
53 556 : bool CheckTensorSizeOverflow(
54 : uint64_t expectedSize, uint64_t tensorSize, const std::string& tensorName, const std::string& apiName,
55 : const ModeType mode)
56 : {
57 556 : std::string curMode = "";
58 556 : if (mode == ModeType::NORM_MODE) {
59 296 : curMode = " when in normal mode";
60 260 : } else if (mode == ModeType::COUNTER_MODE) {
61 0 : curMode = " when in counter mode";
62 : }
63 556 : ASCENDC_CHECK_AND_LOG(expectedSize <= tensorSize, {
64 : CHECK_LOG_ERROR(
65 : "Failed to check %s size in %s%s, tensor size "
66 : "needs to be at least %lu bytes, while current tensor size is only %lu bytes.",
67 : tensorName.c_str(), apiName.c_str(), curMode.c_str(), expectedSize, tensorSize);
68 : });
69 444 : return true;
70 556 : }
71 :
72 200 : bool TikcppBaseCheck::CheckTensorOverflowHigh(
73 : const uint32_t dtypeSize, const uint64_t bufferSize, const uint32_t calCount, const std::string& tensorName) const
74 : {
75 200 : uint64_t needSize = static_cast<uint64_t>(dtypeSize * calCount);
76 200 : if (Int4Setter::Instance().GetInt4()) {
77 0 : needSize = static_cast<uint64_t>(calCount / INT4_TWO);
78 0 : Int4Setter::Instance().ResetInt4();
79 : }
80 200 : ASCENDC_CHECK(CheckTensorSizeOverflow(needSize, bufferSize, tensorName, apiName));
81 136 : return true;
82 : }
83 :
84 196 : bool TikcppBaseCheck::UpdateMaskArrayAndCheck(std::vector<uint64_t>& maskArray, const uint32_t maxByteLen) const
85 : {
86 : // do not use mask given by user, use mask value stored in registers
87 196 : if (!MaskSetter::Instance().GetMask()) {
88 0 : maskArray = {ModelFactoryGetMaskHigh(), ModelFactoryGetMaskLow()};
89 0 : CHECK_LOG_INFO(
90 : "Due to isSetMask = false, maskArray is changed to maskHigh %lu, maskLow %lu.", ModelFactoryGetMaskHigh(),
91 : ModelFactoryGetMaskLow());
92 : }
93 196 : MaskSetter::Instance().SetMask(true);
94 196 : if (maxByteLen >= sizeof(int32_t) && maskArray.size() == 2) { // when size 2 need to update maskHigh
95 : // Example: mask[64, 64] but dtype is float, maximum only read 1 64. maskHigh is unused, need to set to 0.
96 0 : maskArray[0] = 0; // in counter / norm mode, both maskHigh is in fact 0
97 : }
98 196 : if (maskArray.size() != 1 && maskArray.size() != 2) { // mask = len 1, mask[2] = len 2
99 0 : ASCENDC_CHECK_AND_LOG(
100 : false, { CHECK_LOG_ERROR("maskArray size is %lu, which should be 1 or 2.", maskArray.size()); });
101 196 : } else if (maskArray.size() == 1) {
102 : // when norm mode, update mask value to <= 128 when array size is 1
103 144 : if (ModelFactoryGetMaskMode() == 0) {
104 144 : uint64_t maxElePerRep = DEFAULT_BLOCK_SIZE / maxByteLen;
105 144 : maskArray[0] = std::min(maskArray[0], maxElePerRep);
106 : }
107 144 : ASCENDC_CHECK_AND_LOG(CheckMaskImm(maskArray[0]), {
108 : CHECK_LOG_ERROR(
109 : "When maskArray size is 1, mask value %lu "
110 : "is invalid",
111 : maskArray[0]);
112 : });
113 : } else {
114 52 : ASCENDC_CHECK_AND_LOG(CheckMaskArray(maskArray), {
115 : CHECK_LOG_ERROR(
116 : "When maskArray size is 2, maskHigh %lu, "
117 : "maskLow %lu is invalid",
118 : maskArray[0], maskArray[1]);
119 : });
120 : }
121 196 : return true;
122 : }
123 :
124 916 : bool TikcppBaseCheck::CheckTensorScope(
125 : const uint8_t logicPos, const uint8_t expectedPos, const std::string& tensorInfo, const std::string& posInfo) const
126 : {
127 916 : auto& hwNameMap = GlobalParams::Instance().hardwareNameMap;
128 : // tensorPos从逻辑位置转换到物理位置
129 916 : uint8_t hardPos = static_cast<uint8_t>(GetPhyType(static_cast<TPosition>(logicPos)));
130 916 : if (hwNameMap.find(hardPos) == hwNameMap.end() || hwNameMap.find(expectedPos) == hwNameMap.end()) {
131 0 : CHECK_LOG_ERROR(
132 : "the tensorPos/expectPos hardware pos not found, tensorPos index is %hhu, expectedPos is %hhu.", hardPos,
133 : expectedPos);
134 0 : return false;
135 : }
136 :
137 916 : ASCENDC_CHECK_AND_LOG(hardPos == expectedPos, {
138 : const std::string supportedPos = GetPositionDisplay(static_cast<Hardware>(expectedPos), posInfo);
139 : const std::string currentPos = GetPositionDisplay(static_cast<TPosition>(logicPos));
140 : CHECK_LOG_ERROR(
141 : "Failed to check %s tensor position in %s, "
142 : "supported positions are %s, current position is %s.",
143 : tensorInfo.c_str(), apiName.c_str(), supportedPos.c_str(), currentPos.c_str());
144 : });
145 832 : return true;
146 : }
147 :
148 792 : bool TikcppBaseCheck::CheckBufferSizeOverFlow(
149 : const uint64_t localSize, const uint64_t bufferSize, const std::string& errMsg) const
150 : {
151 792 : if (localSize > bufferSize) {
152 0 : CHECK_LOG_ERROR(
153 : "%s, Allocated buffer size overflow, allocate buffer size is %lu, the limit size is %lu.", errMsg.c_str(),
154 : localSize, bufferSize);
155 0 : return false;
156 : }
157 792 : return true;
158 : }
159 :
160 : // check vector mask input range
161 52 : bool TikcppBaseCheck::CheckMaskArray(std::vector<uint64_t> maskArray) const
162 : {
163 : // counter mode
164 52 : if (ModelFactoryGetMaskMode() == 1) {
165 0 : if (maskArray[1] > CONST_UINT32_MAX) { // only use maskLow[0:32] as counter mask. Only give warning!
166 0 : CHECK_LOG_WARNING(
167 : "Failed to check mask array in counter mode, maskLow must be in range 0 ~ %lu, current "
168 : "value is %lu",
169 : CONST_UINT32_MAX, maskArray[1]);
170 : }
171 0 : return true;
172 : }
173 : // normal mode
174 : #if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 1001 || __NPU_ARCH__ == 2002)
175 26 : ASCENDC_CHECK_AND_LOG(maskArray[0] != 0ULL || maskArray[1] != 0ULL, {
176 : CHECK_LOG_ERROR(
177 : "During calculation in "
178 : "normal mode in Ascend910 / Ascend310p / Ascend610, maskHigh %lu and maskLow %lu cannot be both 0.",
179 : maskArray[0], maskArray[1]);
180 : });
181 : #endif
182 52 : return true;
183 : }
184 :
185 : // check mask value is in valid range
186 144 : bool TikcppBaseCheck::CheckMaskImm(const uint64_t mask) const
187 : {
188 : // counter mode: mask means element num
189 144 : if (ModelFactoryGetMaskMode() == 1) {
190 0 : if (mask > CONST_UINT32_MAX) { // only use mask[0:32] as counter mask. Only give warning!
191 0 : CHECK_LOG_WARNING(
192 : "Failed to check mask in counter mode, mask must be in range 0 ~ %lu, current value "
193 : "is %lu",
194 : CONST_UINT32_MAX, mask);
195 : }
196 0 : return true;
197 : }
198 : // normal mode
199 : #if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 1001 || __NPU_ARCH__ == 2002)
200 72 : ASCENDC_CHECK_AND_LOG(mask != 0ULL, {
201 : CHECK_LOG_ERROR(
202 : "During calculation in normal mode in Ascend910 / Ascend310p "
203 : "/ Ascend610, mask %lu cannot be 0.",
204 : mask);
205 : });
206 : #endif
207 144 : return true;
208 : }
209 :
210 : // when counter mode, split into norm mode with main block and tail block
211 0 : void CounterSplitMainTail(
212 : std::vector<uint64_t>& maskArray, const uint32_t dtypeBytes, uint64_t& mainRepeatTimes, uint64_t& tailRepeatTimes,
213 : std::vector<uint64_t>& mainMaskArray, std::vector<uint64_t>& tailMaskArray)
214 : {
215 0 : uint32_t oneRepeatNum = ONE_REPEAT_BYTE_SIZE / dtypeBytes; // when counter mode, always full mask
216 0 : uint64_t elementNum = (maskArray.size() == 1) ? maskArray[0] : maskArray[1]; // maskLow means element num
217 0 : mainRepeatTimes = elementNum / oneRepeatNum;
218 0 : uint64_t tailEleNum = elementNum % oneRepeatNum;
219 0 : tailRepeatTimes = (tailEleNum == 0) ? 0 : 1;
220 0 : mainMaskArray = {oneRepeatNum};
221 0 : tailMaskArray = {tailEleNum};
222 0 : }
223 :
224 : // calculate max extent, aka the offset of the end of all effective element
225 : // maskLen: each repeat calculate the first maskLen elements
226 : // blockLen: element num per block
227 : // return: in unit of element
228 228 : uint64_t CalculateVectorMaxOffset(
229 : const uint64_t repeatTimes, const uint64_t blkStride, const uint64_t repStride, const uint64_t maskLen,
230 : const uint64_t blockLen)
231 : {
232 228 : if (repeatTimes == 0) {
233 4 : return 0;
234 : }
235 224 : ASSERT(blockLen != 0);
236 224 : uint64_t blkNumLastRep = DivCeil(maskLen, blockLen); // last repeat needs x blocks for maskLen elements
237 224 : uint64_t eleNumLastBlk = ((maskLen % blockLen) != 0) ? (maskLen % blockLen) : blockLen;
238 224 : uint64_t maxOffset = ((repeatTimes - 1) * repStride + (blkNumLastRep - 1) * blkStride) * blockLen + eleNumLastBlk;
239 224 : return maxOffset;
240 : }
241 :
242 : namespace {
243 : // Given repeatTimes and stride etc, to return total buffersize needed in unit of Bytes
244 228 : uint64_t CalculateNeededTensorSize(
245 : std::vector<uint64_t>& maskArray, const uint32_t dtypeBytes, const uint64_t repeatTimes, const uint64_t blkStride,
246 : const uint64_t repStride)
247 : {
248 228 : uint64_t maskVal = (maskArray.size() == 1) ? maskArray[0] : GetMaskLength(maskArray, dtypeBytes);
249 228 : ASSERT(dtypeBytes != 0);
250 228 : uint64_t eleNumPerBlock = static_cast<uint64_t>(PlatFormParams::ONE_BLK_SIZE) / dtypeBytes;
251 228 : uint64_t maxOffset = CalculateVectorMaxOffset(repeatTimes, blkStride, repStride, maskVal, eleNumPerBlock);
252 228 : maxOffset = maxOffset * dtypeBytes;
253 228 : if (Int4Setter::Instance().GetInt4()) {
254 0 : eleNumPerBlock = static_cast<uint64_t>(PlatFormParams::ONE_BLK_SIZE) * INT4_TWO;
255 0 : maxOffset = CalculateVectorMaxOffset(repeatTimes, blkStride, repStride, maskVal, eleNumPerBlock);
256 0 : maxOffset = maxOffset / INT4_TWO;
257 0 : Int4Setter::Instance().ResetInt4();
258 : }
259 228 : return maxOffset;
260 : }
261 :
262 : // in counter mode, check whether the data calculated in cmd exceed the tensor size
263 : // need to convert to norm mode with main block and tail block
264 0 : bool CheckTensorOverflowLowCounter(
265 : std::vector<uint64_t>& maskArray, const TensorOverflowParams& params, const std::string& tensorName,
266 : const std::string& apiName)
267 : {
268 0 : std::vector<uint64_t> mainMaskArray = {0};
269 0 : std::vector<uint64_t> tailMaskArray = {0};
270 0 : uint64_t mainRepeatTimes = 0;
271 0 : uint64_t tailRepeatTimes = 0;
272 0 : CounterSplitMainTail(maskArray, params.dtypeSize, mainRepeatTimes, tailRepeatTimes, mainMaskArray, tailMaskArray);
273 : // when counter mode, repeatTimes given by user is not used
274 : // Need to compare: endpoint of mainBlock VS endpoint of tailBlock
275 : // Especially scenes where blkStride is much larger than repStride. mainBlock endpoint will be larger!!
276 : uint64_t mainBlkSize =
277 0 : CalculateNeededTensorSize(mainMaskArray, params.dtypeSize, mainRepeatTimes, params.blkStride, params.repStride);
278 0 : uint64_t maxOffset = mainBlkSize;
279 0 : if (tailRepeatTimes > 0) { // calculate tail block from the last repStride in main block
280 0 : uint64_t tailRepeatStart = mainRepeatTimes * params.repStride * ONE_BLK_SIZE;
281 0 : uint64_t tailBlkSize = CalculateNeededTensorSize(
282 0 : tailMaskArray, params.dtypeSize, tailRepeatTimes, params.blkStride, params.repStride);
283 0 : maxOffset = std::max(mainBlkSize, tailRepeatStart + tailBlkSize);
284 : }
285 0 : ASCENDC_CHECK(CheckTensorSizeOverflow(maxOffset, params.bufferSize, tensorName, apiName, ModeType::COUNTER_MODE));
286 0 : return true;
287 0 : }
288 :
289 : // in normal mode, check whether the data calculated in cmd exceed the tensor size
290 228 : bool CheckTensorOverflowLowNorm(
291 : std::vector<uint64_t>& maskArray, const TensorOverflowParams& params, const std::string& tensorName,
292 : const std::string& apiName)
293 : {
294 : uint64_t maxOffset =
295 228 : CalculateNeededTensorSize(maskArray, params.dtypeSize, params.repeatTimes, params.blkStride, params.repStride);
296 228 : ASCENDC_CHECK(CheckTensorSizeOverflow(maxOffset, params.bufferSize, tensorName, apiName, ModeType::NORM_MODE));
297 200 : return true;
298 : }
299 :
300 : // in counter mode, check whether the data calculated in cmd exceed the tensor size for GatherMask
301 : #if defined(__NPU_ARCH__) && ((__NPU_ARCH__ == 2201) || (__NPU_ARCH__ == 3002) || (__NPU_ARCH__ == 3102) || \
302 : (__NPU_ARCH__ == 3510) || (__NPU_ARCH__ == 5102))
303 0 : bool CheckTensorOverflowLowCounterGatherMask(
304 : std::vector<uint64_t>& maskArray, const TensorOverflowParams& params, const std::string& tensorName)
305 : {
306 0 : uint64_t maskValue = (maskArray.size() == 1) ? maskArray[0] : maskArray[1];
307 0 : uint64_t eleNumPerBlock = ONE_BLK_SIZE / params.dtypeSize;
308 0 : uint64_t blockNumPerRep = (maskValue + eleNumPerBlock - 1) / eleNumPerBlock; // total blks needed for mask num
309 0 : uint64_t repeatOffset = (params.repeatTimes - 1) * params.repStride * ONE_BLK_SIZE; // last repeat start pos
310 0 : uint64_t extraNum = blockNumPerRep * eleNumPerBlock - maskValue; // elements that do not need in tail blocks
311 0 : uint64_t blockOffset = ((blockNumPerRep - 1) * params.blkStride + 1) * ONE_BLK_SIZE; // endblk pos(include extraNum)
312 0 : uint64_t maxOffset = repeatOffset + blockOffset - extraNum * params.dtypeSize;
313 0 : ASCENDC_CHECK(
314 : CheckTensorSizeOverflow(maxOffset, params.bufferSize, tensorName, "GatherMask", ModeType::COUNTER_MODE));
315 0 : return true;
316 : }
317 : #endif
318 : } // namespace
319 :
320 : // check whether the needed size overflow the allocated buffersize, the unit is bytes.
321 212 : bool TikcppBaseCheck::CheckTensorOverflowLow(
322 : std::vector<uint64_t>& maskArray, const TensorOverflowParams& params, const std::string& tensorName) const
323 : {
324 212 : if (ModelFactoryGetMaskMode() == 1) { // counter mode
325 0 : return CheckTensorOverflowLowCounter(maskArray, params, tensorName, apiName);
326 : }
327 212 : return CheckTensorOverflowLowNorm(maskArray, params, tensorName, apiName);
328 : }
329 :
330 : // check whether the needed size overflow the allocated buffersize, the unit is bytes.
331 0 : bool TikcppBaseCheck::CheckTensorOverflowLowGathermask(
332 : std::vector<uint64_t>& maskArray, const TensorOverflowParams& params, const std::string& tensorName) const
333 : {
334 0 : if (params.isCounter) { // counter mode
335 : #if defined(__NPU_ARCH__) && ((__NPU_ARCH__ == 2201) || (__NPU_ARCH__ == 3002) || (__NPU_ARCH__ == 3102) || \
336 : (__NPU_ARCH__ == 3510) || (__NPU_ARCH__ == 5102))
337 0 : return CheckTensorOverflowLowCounterGatherMask(maskArray, params, tensorName);
338 : #else
339 0 : return CheckTensorOverflowLowCounter(maskArray, params, tensorName, apiName);
340 : #endif
341 : }
342 0 : std::vector<uint64_t> maskArrayReal = {FULL_MASK, FULL_MASK}; // when norm mode, GatherMask neglect mask
343 0 : return CheckTensorOverflowLowNorm(maskArrayReal, params, tensorName, apiName);
344 0 : }
345 :
346 16 : bool TikcppBaseCheck::CheckTensorOverflowLowBrcb(
347 : const TensorOverflowParams& params, const std::string& tensorName) const
348 : {
349 32 : std::vector<uint64_t> maskArrayReal = {FULL_MASK, FULL_MASK};
350 32 : return CheckTensorOverflowLowNorm(maskArrayReal, params, tensorName, apiName);
351 16 : }
352 :
353 704 : bool TikcppBaseCheck::CheckTensorAddrAlign(
354 : const uint64_t tensorAddr, const uint8_t phyPos, const uint64_t alignBytes, const std::string& tensorInfo) const
355 : {
356 : uint64_t hardwareBaseAddr = static_cast<uint64_t>(
357 704 : reinterpret_cast<uintptr_t>(ConstDefiner::Instance().GetHardwareBaseAddr(static_cast<Hardware>(phyPos))));
358 704 : uint64_t tensorAbsPos = tensorAddr - hardwareBaseAddr;
359 704 : ASCENDC_CHECK_AND_LOG(((tensorAbsPos % alignBytes) == 0), {
360 : CHECK_LOG_ERROR(
361 : "Failed to check %s tensor address "
362 : "alignment in %s, current tensor address is %lu, which should be %lu byte aligned.",
363 : tensorInfo.c_str(), apiName.c_str(), tensorAbsPos, alignBytes);
364 : });
365 696 : return true;
366 : }
367 :
368 : } // namespace check
369 : } // namespace AscendC
|
