Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "task_exception.h"
12 : #include "log.h"
13 : #include "aicpu_hccl_sqcq.h"
14 : #include "sqe_context_utils.h"
15 :
16 : namespace hccl {
17 33 : TaskException::TaskException() {}
18 :
19 32 : TaskException::~TaskException() {}
20 :
21 0 : HcclResult TaskException::Init(u32 devId, u32 localUserRank, const std::string& identifier)
22 : {
23 0 : devId_ = devId;
24 0 : localUserRank_ = localUserRank;
25 0 : identifier_ = identifier;
26 0 : HCCL_INFO(
27 : "%s success, devId[%u], localUserRank[%u], identifier[%s]", __func__, devId_, localUserRank_,
28 : identifier_.c_str());
29 0 : return HCCL_SUCCESS;
30 : }
31 :
32 0 : HcclResult TaskException::RegisterOpInfo(void* opInfo, u32 size)
33 : {
34 0 : CHK_PTR_NULL(opInfo);
35 0 : CHK_PRT_RET(
36 : size == 0 || size > OP_INFO_MAX_SIZE,
37 : HCCL_ERROR("%s fail, size[%u], expect [1, %u]", __func__, size, OP_INFO_MAX_SIZE), HCCL_E_PARA);
38 :
39 0 : opRingBufferIdx_ = (opRingBufferIdx_ + 1) % OPINFO_RING_BUFFER_MAX;
40 0 : indOpInfos_[opRingBufferIdx_].opIndex = opRingBufferIdx_;
41 0 : CHK_SAFETY_FUNC_RET(memcpy_s(indOpInfos_[opRingBufferIdx_].opInfo, size, reinterpret_cast<uint8_t*>(opInfo), size));
42 0 : HCCL_DEBUG("%s success, opRingBufferIdx_[%u], opInfo[%p], size[%u]", __func__, opRingBufferIdx_, opInfo, size);
43 0 : return HCCL_SUCCESS;
44 : }
45 :
46 0 : HcclResult TaskException::RegisterOpInfoCallback(HcommGetOpInfoCallback callback)
47 : {
48 0 : CHK_PTR_NULL(callback);
49 0 : indOpInfos_[opRingBufferIdx_].callback = callback;
50 0 : HCCL_DEBUG("%s success, opRingBufferIdx_[%u], callback[%p]", __func__, opRingBufferIdx_, callback);
51 0 : return HCCL_SUCCESS;
52 : }
53 :
54 0 : bool TaskException::IsRepeatPrint(u32 streamId, u32 opIndex, u32 sqHead)
55 : {
56 0 : auto it = threadPrintState_.find(streamId);
57 0 : if (it != threadPrintState_.end()) {
58 0 : return it->second.first == opIndex && it->second.second == sqHead;
59 : }
60 0 : return false;
61 : }
62 :
63 0 : HcclResult TaskException::PrintTaskException(hccl::Stream& stream)
64 : {
65 0 : u32 sqHead = 0U;
66 0 : u32 sqTail = 0U;
67 0 : CHK_RET(QuerySqStatus(devId_, stream.sqId(), sqHead, sqTail));
68 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
69 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
70 0 : CHK_PTR_NULL(sqeContextBuffer);
71 0 : if (sqHead == sqTail) { // 流上task已经执行完，不打印
72 0 : HCCL_RUN_INFO(
73 : "%s skip, group:%s, streamId:%d, sqHead is equal to sqTail:%u", __func__, identifier_.c_str(), stream.id(),
74 : sqTail);
75 : // 打印最后执行的200个task
76 0 : PrintTaskExceptionTaskQue(sqTail, sqeContextBuffer);
77 0 : return HCCL_SUCCESS;
78 : }
79 0 : u32 opIndex = indOpInfos_[sqeContextBuffer->rtsDfxInfo[sqHead].opRingBufferIdx].opIndex;
80 0 : if (IsRepeatPrint(stream.id(), opIndex, sqHead)) { // 避免重复打印
81 0 : HCCL_RUN_INFO(
82 : "%s skip, group:%s, streamId:%d, opIndex:%u, sqHead:%u, has already been printed", __func__,
83 : identifier_.c_str(), stream.id(), opIndex, sqHead);
84 0 : return HCCL_SUCCESS;
85 : }
86 :
87 0 : threadPrintState_[stream.id()] = {opIndex, sqHead}; // 打印当前流的信息，记录流的位置
88 0 : HCCL_RUN_INFO(
89 : "%s start, group:%s, streamId:%d, opIndex:%u, sqHead:%u, sqTail:%u", __func__, identifier_.c_str(), stream.id(),
90 : opIndex, sqHead, sqTail);
91 :
92 0 : HCCL_ERROR(
93 : "%s base information is streamId:%d, sqid:%u, head:%u, tail:%u, %s", __func__, stream.id(), stream.sqId(),
94 : sqHead, sqTail, GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer).c_str());
95 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
96 0 : return HCCL_SUCCESS;
97 : }
98 :
99 0 : HcclResult TaskException::PrintTaskExceptionByTaskId(u8 sqeType, u16 taskId, hccl::Stream& stream, u32 tail)
100 : {
101 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
102 0 : CHK_PTR_NULL(sqeContext);
103 0 : HCCL_ERROR("%s streamId:%d tail:%u cqeType:%u", __func__, stream.id(), tail, sqeType);
104 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
105 0 : CHK_PTR_NULL(sqeContextBuffer);
106 0 : uint8_t* sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + (tail - 1) * HCCL_SQE_SIZE;
107 0 : rtStarsSqeHeader_t* const sqeHeader = reinterpret_cast<rtStarsSqeHeader_t* const>(sqeMirrorBufferAddr);
108 0 : CHK_PTR_NULL(sqeHeader);
109 :
110 0 : s32 taskNum = sqeHeader->taskId - taskId;
111 0 : HCCL_DEBUG("%s tail sqe taskId[%u] cqe taskId[%u] cqe type[%u]", __func__, sqeHeader->taskId, taskId, sqeType);
112 0 : s32 sqeIdx = tail - taskNum - 1;
113 0 : u32 sqHead = (sqeIdx + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
114 :
115 0 : HCCL_ERROR(
116 : "[TaskException]base information is streamId:%d, sqid:%u, head:%u, tail:%u, %s", stream.id(), stream.sqId(),
117 : sqHead, tail, GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer).c_str());
118 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
119 0 : return HCCL_SUCCESS;
120 : }
121 :
122 0 : std::string TaskException::GetTaskExceptionTaskInfo(u32 sqHead, SqeRingBuffer* sqeContextBuffer)
123 : {
124 0 : SqeInfo sqeInfo;
125 0 : SqeContextUtils::QuerySqeInfo(
126 0 : sqeContextBuffer->rtsMirrorBuffer + sqHead * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[sqHead],
127 : sqeContextBuffer->addInfo[sqHead], &sqeInfo);
128 :
129 0 : std::stringstream ss;
130 0 : ss << "type:" << SqeContextUtils::RtsqTaskTypeToStr(sqeInfo.type) << ", ";
131 0 : ss << "localRank:" << localUserRank_ << ", ";
132 0 : ss << "remoteRank:" << sqeContextBuffer->rtsDfxInfo[sqHead].remoteRank << ", ";
133 0 : ss << "taskId:" << sqeInfo.taskId << ", ";
134 0 : ss << "notifyId:" << sqeInfo.notifyId << ", ";
135 0 : ss << "length:" << sqeInfo.length << ", ";
136 0 : ss << "addr1High:0x" << std::hex << sqeInfo.addr1High << ", ";
137 0 : ss << "addr1Low:0x" << std::hex << sqeInfo.addr1Low << ", ";
138 0 : ss << "addr2High:0x" << std::hex << sqeInfo.addr2High << ", ";
139 0 : ss << "addr2Low:0x" << std::hex << sqeInfo.addr2Low << ".";
140 0 : return ss.str();
141 0 : }
142 :
143 0 : void TaskException::PrintTaskExceptionTaskQue(u32 sqIdx, SqeRingBuffer* sqeContextBuffer)
144 : {
145 0 : const u32 sqeNum = 200; // 打印当前位置的前200个task
146 : // 记录上一次打印的算子信息
147 0 : IndOpInfo& lastOpInfo = indOpInfos_[sqeContextBuffer->rtsDfxInfo[sqIdx].opRingBufferIdx];
148 0 : u32 opIndex = lastOpInfo.opIndex; // 算子序号
149 0 : std::stringstream ss;
150 0 : ss << "OP(" << opIndex << ")";
151 0 : u32 tasksInCurrentLine = 0;
152 0 : const u32 maxTasksPerLine = 50; // 每行最多50个task，避免日志过长
153 0 : for (u32 i = 0; i < sqeNum; i++) {
154 0 : u32 newSqIdx = (sqIdx - i + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
155 0 : IndOpInfo& newOpInfo = indOpInfos_[sqeContextBuffer->rtsDfxInfo[newSqIdx].opRingBufferIdx];
156 0 : u32 newOpIdx = newOpInfo.opIndex;
157 0 : if (newOpIdx != opIndex) { // 不同一个算子，或已经到打印的最后一个位置
158 0 : PrintTaskExceptionOpInfo(lastOpInfo);
159 0 : HCCL_ERROR("[TaskException]task sequence is %s", ss.str().c_str());
160 0 : opIndex = newOpIdx;
161 0 : lastOpInfo = newOpInfo;
162 0 : ss.str("");
163 0 : ss << "OP(" << opIndex << ")";
164 0 : tasksInCurrentLine = 0;
165 : }
166 : // 输入task缩写
167 0 : ss << "," << GetTaskBriefsInfo(newSqIdx, sqeContextBuffer);
168 0 : tasksInCurrentLine++;
169 : // 超过每行最大task数时切分打印
170 0 : if (tasksInCurrentLine >= maxTasksPerLine && i < sqeNum) {
171 0 : HCCL_ERROR("[TaskException]task sequence is %s", ss.str().c_str());
172 0 : ss.str("");
173 0 : ss << "OP(" << opIndex << ")";
174 0 : tasksInCurrentLine = 0;
175 : }
176 : }
177 : // 打印剩余的task
178 0 : if (tasksInCurrentLine > 0) {
179 0 : HCCL_ERROR("[TaskException]task sequence is %s", ss.str().c_str());
180 : }
181 0 : return;
182 0 : }
183 :
184 0 : void TaskException::PrintTaskExceptionOpInfo(IndOpInfo& indOp)
185 : {
186 0 : if (indOp.callback == nullptr) {
187 0 : HCCL_ERROR(
188 : "[TaskException][%s] fail, indOp callback is nullptr, group:%s, opIndex:%u", __func__, identifier_.c_str(),
189 : indOp.opIndex);
190 0 : return;
191 : }
192 : char opInfoTmp[OPINFO_RING_BUFFER_MAX];
193 0 : indOp.callback(reinterpret_cast<void*>(indOp.opInfo), opInfoTmp, OPINFO_RING_BUFFER_MAX);
194 0 : HCCL_ERROR(
195 : "[TaskException]opData information is group:%s, opIndex:%u, %s", identifier_.c_str(), indOp.opIndex, opInfoTmp);
196 : }
197 :
198 0 : std::string TaskException::GetTaskBriefsInfo(u32 idx, SqeRingBuffer* sqeContextBuffer)
199 : {
200 0 : uint8_t* sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE;
201 0 : rtStarsSqeHeader_t* const sqeHeader = reinterpret_cast<rtStarsSqeHeader_t* const>(sqeMirrorBufferAddr);
202 0 : uint8_t sqeType = sqeHeader->type;
203 :
204 0 : SqeInfo sqeInfo;
205 0 : SqeContextUtils::QuerySqeInfo(
206 0 : sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[idx],
207 : sqeContextBuffer->addInfo[idx], &sqeInfo);
208 0 : uint8_t subType = sqeInfo.subType;
209 :
210 0 : std::stringstream ss;
211 0 : std::string taskName = "UN";
212 0 : switch (sqeType) {
213 0 : case RT_STARS_SQE_TYPE_NOTIFY_RECORD:
214 0 : taskName = "NR"; // Notify Record
215 0 : break;
216 0 : case RT_STARS_SQE_TYPE_WRITE_VALUE:
217 0 : if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE) {
218 0 : taskName = "NR";
219 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_EVENT_RESET) {
220 0 : taskName = "NW"; // Notify Wait
221 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_RDMA_DB_SEND) {
222 0 : taskName = "RS"; // Rdma Send
223 : }
224 0 : break;
225 0 : case RT_STARS_SQE_TYPE_NOTIFY_WAIT:
226 0 : taskName = "NW";
227 0 : break;
228 0 : case RT_STARS_SQE_TYPE_EVENT_WAIT:
229 0 : taskName = "NW";
230 0 : break;
231 0 : case RT_STARS_SQE_TYPE_SDMA:
232 0 : taskName = "SD"; // SDMA
233 0 : break;
234 0 : case RT_STARS_SQE_TYPE_PLACE_HOLDER:
235 0 : taskName = "PH";
236 0 : break;
237 0 : default:
238 0 : break;
239 : }
240 :
241 0 : ss << taskName << "(";
242 0 : if (sqeContextBuffer->rtsDfxInfo[idx].remoteRank != INVALID_VALUE_RANKID) {
243 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].remoteRank;
244 : } else {
245 0 : ss << "/";
246 : }
247 0 : ss << ",";
248 0 : if (sqeContextBuffer->rtsDfxInfo[idx].notifyId != INVALID_VALUE_RANKID) {
249 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].notifyId;
250 : } else {
251 0 : ss << "/";
252 : }
253 0 : ss << ")";
254 0 : return ss.str();
255 0 : }
256 : } // namespace hccl
|
