Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "task_exception.h"
12 : #include "log.h"
13 : #include "aicpu_hccl_sqcq.h"
14 : #include "sqe_context_utils.h"
15 :
16 : namespace hccl {
17 32 : TaskException::TaskException() {}
18 :
19 31 : TaskException::~TaskException() {}
20 :
21 0 : HcclResult TaskException::Init(u32 devId, u32 localUserRank, const std::string &identifier)
22 : {
23 0 : devId_ = devId;
24 0 : localUserRank_ = localUserRank;
25 0 : identifier_ = identifier;
26 0 : HCCL_INFO("%s success, devId[%u], localUserRank[%u], identifier[%s]",
27 : __func__, devId_, localUserRank_, identifier_.c_str());
28 0 : return HCCL_SUCCESS;
29 : }
30 :
31 0 : HcclResult TaskException::RegisterOpInfo(void* opInfo, u32 size)
32 : {
33 0 : CHK_PTR_NULL(opInfo);
34 0 : CHK_PRT_RET(size == 0 || size > OP_INFO_MAX_SIZE,
35 : HCCL_ERROR("%s fail, size[%u], expect [1, %u]", __func__, size, OP_INFO_MAX_SIZE), HCCL_E_PARA);
36 :
37 0 : opRingBufferIdx_ = (opRingBufferIdx_ + 1) % OPINFO_RING_BUFFER_MAX;
38 0 : indOpInfos_[opRingBufferIdx_].opIndex = opRingBufferIdx_;
39 0 : CHK_SAFETY_FUNC_RET(memcpy_s(indOpInfos_[opRingBufferIdx_].opInfo, size, reinterpret_cast<uint8_t *>(opInfo), size));
40 0 : HCCL_DEBUG("%s success, opRingBufferIdx_[%u], opInfo[%p], size[%u]", __func__, opRingBufferIdx_, opInfo, size);
41 0 : return HCCL_SUCCESS;
42 : }
43 :
44 0 : HcclResult TaskException::RegisterOpInfoCallback(HcommGetOpInfoCallback callback)
45 : {
46 0 : CHK_PTR_NULL(callback);
47 0 : indOpInfos_[opRingBufferIdx_].callback = callback;
48 0 : HCCL_DEBUG("%s success, opRingBufferIdx_[%u], callback[%p]", __func__, opRingBufferIdx_, callback);
49 0 : return HCCL_SUCCESS;
50 : }
51 :
52 0 : bool TaskException::IsRepeatPrint(u32 streamId, u32 opIndex, u32 sqHead)
53 : {
54 0 : auto it = threadPrintState_.find(streamId);
55 0 : if (it != threadPrintState_.end()) {
56 0 : return it->second.first == opIndex && it->second.second == sqHead;
57 : }
58 0 : return false;
59 : }
60 :
61 0 : HcclResult TaskException::PrintTaskException(hccl::Stream& stream)
62 : {
63 0 : u32 sqHead = 0U;
64 0 : u32 sqTail = 0U;
65 0 : CHK_RET(QuerySqStatus(devId_, stream.sqId(), sqHead, sqTail));
66 0 : HcclSqeContext *sqeContext = stream.GetSqeContextPtr();
67 0 : SqeRingBuffer *sqeContextBuffer = &(sqeContext->buffer);
68 0 : CHK_PTR_NULL(sqeContextBuffer);
69 0 : if (sqHead == sqTail) { // 流上task已经执行完，不打印
70 0 : HCCL_RUN_INFO("%s skip, group:%s, streamId:%d, sqHead is equal to sqTail:%u",
71 : __func__, identifier_.c_str(), stream.id(), sqTail);
72 : // 打印最后执行的200个task
73 0 : PrintTaskExceptionTaskQue(sqTail, sqeContextBuffer);
74 0 : return HCCL_SUCCESS;
75 : }
76 0 : u32 opIndex = indOpInfos_[sqeContextBuffer->rtsDfxInfo[sqHead].opRingBufferIdx].opIndex;
77 0 : if (IsRepeatPrint(stream.id(), opIndex, sqHead)) { // 避免重复打印
78 0 : HCCL_RUN_INFO("%s skip, group:%s, streamId:%d, opIndex:%u, sqHead:%u, has already been printed",
79 : __func__, identifier_.c_str(), stream.id(), opIndex, sqHead);
80 0 : return HCCL_SUCCESS;
81 : }
82 :
83 0 : threadPrintState_[stream.id()] = {opIndex, sqHead}; // 打印当前流的信息，记录流的位置
84 0 : HCCL_RUN_INFO("%s start, group:%s, streamId:%d, opIndex:%u, sqHead:%u, sqTail:%u",
85 : __func__, identifier_.c_str(), stream.id(), opIndex, sqHead, sqTail);
86 :
87 0 : HCCL_ERROR("%s base information is streamId:%d, sqid:%u, head:%u, tail:%u, %s",
88 : __func__, stream.id(), stream.sqId(), sqHead, sqTail, GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer).c_str());
89 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
90 0 : return HCCL_SUCCESS;
91 : }
92 :
93 0 : HcclResult TaskException::PrintTaskExceptionByTaskId(u8 sqeType, u16 taskId, hccl::Stream &stream, u32 tail)
94 : {
95 0 : HcclSqeContext *sqeContext = stream.GetSqeContextPtr();
96 0 : CHK_PTR_NULL(sqeContext);
97 0 : HCCL_ERROR("%s streamId:%d tail:%u cqeType:%u", __func__, stream.id(), tail, sqeType);
98 0 : SqeRingBuffer *sqeContextBuffer = &(sqeContext->buffer);
99 0 : CHK_PTR_NULL(sqeContextBuffer);
100 0 : uint8_t *sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + (tail - 1) * HCCL_SQE_SIZE;
101 0 : rtStarsSqeHeader_t * const sqeHeader = reinterpret_cast<rtStarsSqeHeader_t * const>(sqeMirrorBufferAddr);
102 0 : CHK_PTR_NULL(sqeHeader);
103 :
104 0 : s32 taskNum = sqeHeader->taskId - taskId;
105 0 : HCCL_DEBUG("%s tail sqe taskId[%u] cqe taskId[%u] cqe type[%u]", __func__, sqeHeader->taskId, taskId, sqeType);
106 0 : s32 sqeIdx = tail - taskNum - 1;
107 0 : u32 sqHead = (sqeIdx + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
108 :
109 0 : HCCL_ERROR("[TaskException]base information is streamId:%d, sqid:%u, head:%u, tail:%u, %s",
110 : stream.id(), stream.sqId(), sqHead, tail, GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer).c_str());
111 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
112 0 : return HCCL_SUCCESS;
113 : }
114 :
115 0 : std::string TaskException::GetTaskExceptionTaskInfo(u32 sqHead, SqeRingBuffer *sqeContextBuffer)
116 : {
117 0 : SqeInfo sqeInfo;
118 0 : SqeContextUtils::QuerySqeInfo(sqeContextBuffer->rtsMirrorBuffer + sqHead * HCCL_SQE_SIZE,
119 0 : sqeContextBuffer->rtsqSqeType[sqHead], sqeContextBuffer->addInfo[sqHead], &sqeInfo);
120 :
121 0 : std::stringstream ss;
122 0 : ss << "type:" << SqeContextUtils::RtsqTaskTypeToStr(sqeInfo.type) << ", ";
123 0 : ss << "localRank:" << localUserRank_ << ", ";
124 0 : ss << "remoteRank:" << sqeContextBuffer->rtsDfxInfo[sqHead].remoteRank << ", ";
125 0 : ss << "taskId:" << sqeInfo.taskId << ", ";
126 0 : ss << "notifyId:" << sqeInfo.notifyId << ", ";
127 0 : ss << "length:" << sqeInfo.length << ", ";
128 0 : ss << "addr1High:0x" << std::hex << sqeInfo.addr1High << ", ";
129 0 : ss << "addr1Low:0x" << std::hex << sqeInfo.addr1Low << ", ";
130 0 : ss << "addr2High:0x" << std::hex << sqeInfo.addr2High << ", ";
131 0 : ss << "addr2Low:0x" << std::hex << sqeInfo.addr2Low << ".";
132 0 : return ss.str();
133 0 : }
134 :
135 0 : void TaskException::PrintTaskExceptionTaskQue(u32 sqIdx, SqeRingBuffer *sqeContextBuffer)
136 : {
137 0 : const u32 sqeNum = 200; //打印当前位置的前200个task
138 : // 记录上一次打印的算子信息
139 0 : IndOpInfo& lastOpInfo = indOpInfos_[sqeContextBuffer->rtsDfxInfo[sqIdx].opRingBufferIdx];
140 0 : u32 opIndex = lastOpInfo.opIndex; // 算子序号
141 0 : std::stringstream ss;
142 0 : ss << "OP(" << opIndex << ")";
143 0 : u32 tasksInCurrentLine = 0;
144 0 : const u32 maxTasksPerLine = 50; // 每行最多50个task，避免日志过长
145 0 : for (u32 i = 0; i < sqeNum; i++) {
146 0 : u32 newSqIdx = (sqIdx - i + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
147 0 : IndOpInfo& newOpInfo = indOpInfos_[sqeContextBuffer->rtsDfxInfo[newSqIdx].opRingBufferIdx];
148 0 : u32 newOpIdx = newOpInfo.opIndex;
149 0 : if (newOpIdx != opIndex) { // 不同一个算子，或已经到打印的最后一个位置
150 0 : PrintTaskExceptionOpInfo(lastOpInfo);
151 0 : HCCL_ERROR("[TaskException]task sequence is %s", ss.str().c_str());
152 0 : opIndex = newOpIdx;
153 0 : lastOpInfo = newOpInfo;
154 0 : ss.str("");
155 0 : ss << "OP(" << opIndex << ")";
156 0 : tasksInCurrentLine = 0;
157 : }
158 : // 输入task缩写
159 0 : ss << "," << GetTaskBriefsInfo(newSqIdx, sqeContextBuffer);
160 0 : tasksInCurrentLine++;
161 : // 超过每行最大task数时切分打印
162 0 : if (tasksInCurrentLine >= maxTasksPerLine && i < sqeNum) {
163 0 : HCCL_ERROR("[TaskException]task sequence is %s", ss.str().c_str());
164 0 : ss.str("");
165 0 : ss << "OP(" << opIndex << ")";
166 0 : tasksInCurrentLine = 0;
167 : }
168 : }
169 : // 打印剩余的task
170 0 : if (tasksInCurrentLine > 0) {
171 0 : HCCL_ERROR("[TaskException]task sequence is %s", ss.str().c_str());
172 : }
173 0 : return;
174 0 : }
175 :
176 0 : void TaskException::PrintTaskExceptionOpInfo(IndOpInfo& indOp)
177 : {
178 0 : if (indOp.callback == nullptr) {
179 0 : HCCL_ERROR("[TaskException][%s] fail, indOp callback is nullptr, group:%s, opIndex:%u",
180 : __func__, identifier_.c_str(), indOp.opIndex);
181 0 : return;
182 : }
183 : char opInfoTmp[OPINFO_RING_BUFFER_MAX];
184 0 : indOp.callback(reinterpret_cast<void *>(indOp.opInfo), opInfoTmp, OPINFO_RING_BUFFER_MAX);
185 0 : HCCL_ERROR("[TaskException]opData information is group:%s, opIndex:%u, %s",
186 : identifier_.c_str(), indOp.opIndex, opInfoTmp);
187 : }
188 :
189 0 : std::string TaskException::GetTaskBriefsInfo(u32 idx, SqeRingBuffer *sqeContextBuffer)
190 : {
191 0 : uint8_t *sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE;
192 0 : rtStarsSqeHeader_t * const sqeHeader = reinterpret_cast<rtStarsSqeHeader_t * const>(sqeMirrorBufferAddr);
193 0 : uint8_t sqeType = sqeHeader->type;
194 :
195 0 : SqeInfo sqeInfo;
196 0 : SqeContextUtils::QuerySqeInfo(sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE,
197 0 : sqeContextBuffer->rtsqSqeType[idx], sqeContextBuffer->addInfo[idx], &sqeInfo);
198 0 : uint8_t subType = sqeInfo.subType;
199 :
200 0 : std::stringstream ss;
201 0 : std::string taskName = "UN";
202 0 : switch (sqeType) {
203 0 : case RT_STARS_SQE_TYPE_NOTIFY_RECORD:
204 0 : taskName = "NR"; // Notify Record
205 0 : break;
206 0 : case RT_STARS_SQE_TYPE_WRITE_VALUE:
207 0 : if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE) {
208 0 : taskName = "NR";
209 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_EVENT_RESET) {
210 0 : taskName = "NW"; // Notify Wait
211 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_RDMA_DB_SEND) {
212 0 : taskName = "RS"; // Rdma Send
213 : }
214 0 : break;
215 0 : case RT_STARS_SQE_TYPE_NOTIFY_WAIT:
216 0 : taskName = "NW";
217 0 : break;
218 0 : case RT_STARS_SQE_TYPE_EVENT_WAIT:
219 0 : taskName = "NW";
220 0 : break;
221 0 : case RT_STARS_SQE_TYPE_SDMA:
222 0 : taskName = "SD"; // SDMA
223 0 : break;
224 0 : case RT_STARS_SQE_TYPE_PLACE_HOLDER:
225 0 : taskName = "PH";
226 0 : break;
227 0 : default:
228 0 : break;
229 : }
230 :
231 0 : ss << taskName << "(";
232 0 : if (sqeContextBuffer->rtsDfxInfo[idx].remoteRank != INVALID_VALUE_RANKID) {
233 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].remoteRank;
234 : } else {
235 0 : ss << "/";
236 : }
237 0 : ss << ",";
238 0 : if (sqeContextBuffer->rtsDfxInfo[idx].notifyId != INVALID_VALUE_RANKID) {
239 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].notifyId;
240 : } else {
241 0 : ss << "/";
242 : }
243 0 : ss << ")";
244 0 : return ss.str();
245 0 : }
246 : } // namespace dfx
|
