Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "utils/hccl_aicpu_utils.h"
12 : #include <sstream>
13 : #include <dlog_pub.h>
14 :
15 : #include "algorithm/task_orchestrator.h"
16 : #include "common/aicpu_sqe_context.h"
17 : #include "common/aicpu_hccl_common.h"
18 : #include "profiling_manager_device.h"
19 : #include "framework/aicpu_communicator.h"
20 : #include "log.h"
21 : #include "hccl_types.h"
22 : #include "transport_pub.h"
23 :
24 46902 : int32_t HcclAicpuUtils::GetCpuId()
25 : {
26 : static thread_local int32_t curCpu = -1;
27 46902 : if (curCpu < 0) {
28 2 : curCpu = sched_getcpu();
29 : }
30 46902 : return curCpu;
31 : }
32 :
33 46980 : int32_t HcclAicpuUtils::GetCurClusterId() { return !!(GetCpuId() & (AICPU_CNT / CLUSTER_CNT)); }
34 :
35 81 : void HcclAicpuUtils::PrintHcclCombinOpParam(const HccCommResParamTask& commParam)
36 : {
37 81 : if (!HcclCheckLogLevel(HCCL_LOG_INFO)) {
38 0 : return;
39 : }
40 81 : HCCL_INFO("HccCommResParamTask.workSpace %p", commParam.mc2WorkSpace.workSpace);
41 81 : HCCL_INFO("HccCommResParamTask.workSpaceSize %lu", commParam.mc2WorkSpace.workSpaceSize);
42 81 : HCCL_INFO("HccCommResParamTask.rankId %u", commParam.rankId);
43 81 : HCCL_INFO("HccCommResParamTask.rankNum %u", commParam.rankNum);
44 81 : HCCL_INFO("HccCommResParamTask.winSize %lu", commParam.winSize);
45 2673 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM; i++) {
46 2592 : HCCL_INFO(
47 : "HccCommResParamTask.windowsIn[%u] %p, windowsOut[%u] %p", i, commParam.windowsIn[i], i,
48 : commParam.windowsOut[i]);
49 : }
50 2673 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM; i++) {
51 2592 : const HcclStreamInfo& sinfo = commParam.streamInfo[i];
52 2592 : HCCL_INFO(
53 : "HccCommResParamTask.streamInfo[%u] streamId %d, sqId %u, cqId %lu logicCqid %u", i, sinfo.streamIds,
54 : sinfo.sqIds, sinfo.cqIds, sinfo.logicCqids);
55 : }
56 5265 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM * 2; i++) { // 2 is number of noIpcNotify
57 5184 : const HcclSignalInfo& sinfo = commParam.signalInfo.noIpcNotifys[i];
58 5184 : HCCL_INFO(
59 : "HccCommResParamTask.noIpcNotifys[%u] resId %lu, addr %p, devId %u, tsId %u, rankId %u", i, sinfo.resId,
60 : sinfo.addr, sinfo.devId, sinfo.tsId, sinfo.rankId);
61 : }
62 10449 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM * 4; i++) { // 4 is number of ipcNotifys
63 10368 : const HcclSignalInfo& sinfo = commParam.signalInfo.ipcNotifys[i];
64 10368 : HCCL_INFO(
65 : "HccCommResParamTask.ipcNotifys[%u] resId %lu, addr %p, devId %u, tsId %u, rankId %u", i, sinfo.resId,
66 : sinfo.addr, sinfo.devId, sinfo.tsId, sinfo.rankId);
67 : }
68 2673 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM; i++) {
69 2592 : const HcclSignalInfo& sinfo = commParam.signalInfo.noIpcEvents[i];
70 2592 : HCCL_INFO(
71 : "HccCommResParamTask.noIpcEvents[%u] resId %lu, addr %p, devId %u, tsId %u, rankId %u", i, sinfo.resId,
72 : sinfo.addr, sinfo.devId, sinfo.tsId, sinfo.rankId);
73 : }
74 :
75 243 : for (uint32_t i = 0; i < AICPU_OP_NOTIFY_NUM; i++) {
76 162 : const HcclSignalInfo& sinfo = commParam.signalInfo.aicpuOpNotify[i];
77 162 : HCCL_INFO(
78 : "HccCommResParamTask.aicpuOpNotify[%u] resId %lu, addr %p, devId %u, tsId %u, rankId %u", i, sinfo.resId,
79 : sinfo.addr, sinfo.devId, sinfo.tsId, sinfo.rankId);
80 : }
81 81 : const auto& sigInfo = commParam.signalInfo.aicpuNotify;
82 81 : HCCL_INFO(
83 : "HccCommResParamTask.aicpuNotify resId %lu, addr %p, devId %u, tsId %u, rankId %u", sigInfo.resId, sigInfo.addr,
84 : sigInfo.devId, sigInfo.tsId, sigInfo.rankId);
85 81 : HCCL_INFO("HccCommResParamTask.determinism %u", commParam.config.deterministic);
86 81 : HCCL_INFO("HccCommResParamTask.overflowAddr %p", commParam.overFlowAddr);
87 81 : HCCL_INFO("HccCommResParamTask.retryParams: retryEnable %u", commParam.config.retryEnable);
88 : }
89 :
90 14 : void HcclAicpuUtils::PrintHcclOpResParam(const HcclOpResParam* resParam)
91 : {
92 14 : if (resParam == nullptr) {
93 1 : HCCL_ERROR("[%s] resParam is nullptr", __func__);
94 1 : return;
95 : }
96 13 : HCCL_INFO("HcclOpResParam.rankId %u", resParam->localUsrRankId);
97 13 : HCCL_INFO("HcclOpResParam.rankNum %u", resParam->rankSize);
98 :
99 13 : HCCL_INFO("HcclOpResParam.windowSize %lu", resParam->winSize);
100 13 : HCCL_INFO("HcclOpResParam.workSpaceAddr %p", resParam->mc2WorkSpace.workSpace);
101 13 : HCCL_INFO("HcclOpResParam.workSpaceSize %lu", resParam->mc2WorkSpace.workSpaceSize);
102 373 : for (uint32_t i = 0; i < resParam->localRes.streamNum; i++) {
103 360 : const auto streamInfo = resParam->localRes.streamParam[i].streamInfo;
104 360 : HCCL_INFO(
105 : "HcclOpResParam.streamInfo[%u] streamId %d sqId %u cqId %u logicCqid %u", i, streamInfo.streamIds,
106 : streamInfo.sqIds, streamInfo.cqIds, streamInfo.logicCqids);
107 : }
108 13 : const auto mainStreamInfo = resParam->localRes.mainStreamParam.streamInfo;
109 13 : HCCL_INFO(
110 : "HcclOpResParam.mainStreamInfo streamId %d sqId %u cqId %u logicCqid %u", mainStreamInfo.streamIds,
111 : mainStreamInfo.sqIds, mainStreamInfo.cqIds, mainStreamInfo.logicCqids);
112 589 : for (uint32_t i = 0; i < resParam->localRes.signalNum; i++) {
113 576 : const auto& signals = resParam->localRes.localSignals[i];
114 576 : HCCL_INFO(
115 : "HcclOpResParam.localSignals[%u] resId %p addr %p devId %u tsId %u rankId %u", i, signals.resId,
116 : signals.addr, signals.devId, signals.tsId, signals.rankId);
117 : }
118 :
119 39 : for (uint32_t i = 0; i < AICPU_OP_NOTIFY_MAX_NUM; i++) {
120 26 : const auto& aicpuOpNotify = resParam->localRes.aicpuOpNotify[i];
121 26 : HCCL_INFO(
122 : "HcclOpResParam.aicpuOpNotify[%u] resId %p addr %p devId %u tsId %u rankId %u", i, aicpuOpNotify.resId,
123 : aicpuOpNotify.addr, aicpuOpNotify.devId, aicpuOpNotify.tsId, aicpuOpNotify.rankId);
124 : }
125 13 : HCCL_INFO("HcclOpResParam.determinism %u", resParam->config.deterministic);
126 : }
127 :
128 1 : HcclResult HcclAicpuUtils::Getkey(
129 : const AicpuComContext& ctx, u32 remoteRankId, const void* userAddr, u64 length, u32& outKey, int32_t keyType)
130 : {
131 1 : CHK_PTR_NULL(userAddr);
132 0 : HCCL_INFO("[HcclAicpuUtils][Getkey] addr[%p] len[%llu]", userAddr, length);
133 0 : u64 inAddr = reinterpret_cast<u64>(userAddr);
134 : MemDetails inputMem
135 0 : = (keyType == LOCAL) ? ctx.ibversData[remoteRankId].localInputMem : ctx.ibversData[remoteRankId].remoteInputMem;
136 0 : MemDetails outputMem = (keyType == LOCAL) ? ctx.ibversData[remoteRankId].localOutputMem :
137 0 : ctx.ibversData[remoteRankId].remoteOutputMem;
138 :
139 0 : u64 inputStartAddr = inputMem.addr;
140 0 : u64 inputCCLSize = inputMem.size;
141 0 : u32 inputKey = inputMem.key;
142 :
143 0 : u64 outputStartAddr = outputMem.addr;
144 0 : u64 outputCCLSize = outputMem.size;
145 0 : u32 outputKey = outputMem.key;
146 0 : if (inAddr >= inputStartAddr && inAddr < inputStartAddr + inputCCLSize) {
147 0 : outKey = inputKey;
148 0 : } else if (inAddr >= outputStartAddr && inAddr <= outputStartAddr + outputCCLSize) {
149 0 : outKey = outputKey;
150 : } else {
151 0 : HCCL_ERROR(
152 : "[HcclAicpuUtils][Getkey]src_ptr=%p is out of range, inputmem src[%p], size[%llu];"
153 : " outputmem src[%p] size[%llu]",
154 : userAddr, inputStartAddr, inputCCLSize, outputStartAddr, outputCCLSize);
155 0 : return HCCL_E_INTERNAL;
156 : }
157 0 : HCCL_INFO(
158 : "[HcclAicpuUtils][Getkey] addr[%p] length[%llu] outKey[%u], keyType:[%s]", userAddr, length, outKey,
159 : (keyType == LOCAL) ? "local" : "remote");
160 :
161 0 : return HCCL_SUCCESS;
162 : }
163 :
164 : std::mutex g_mtxForDoorbell;
165 0 : HcclResult HcclAicpuUtils::PostSend(
166 : const AicpuComContext& ctx, u32 remoteRankId, struct std::vector<hccl::Transport::Buffer>& remoteBuf,
167 : struct std::vector<hccl::Transport::Buffer>& localBuf, bool isWrite)
168 : {
169 0 : if (UNLIKELY(remoteRankId >= ctx.rankNum)) {
170 0 : HCCL_ERROR("[AicpuIbverbs][PostSend] remoteRankId %u is out of range, ranknum %u", remoteRankId, ctx.rankNum);
171 0 : return HCCL_E_PARA;
172 : }
173 0 : CHK_PRT_RET(
174 : remoteBuf.size() != localBuf.size(),
175 : HCCL_ERROR(
176 : "[AicpuIbverbs][PostSend] remoteBuf list size %u is not equal localBuffer list size %u ", remoteBuf.size(),
177 : localBuf.size()),
178 : HCCL_E_PARA);
179 :
180 0 : uint32_t len = remoteBuf.size();
181 0 : const uint32_t MAX_MEM_NUM = 8;
182 0 : CHK_PRT_RET(
183 : len > MAX_MEM_NUM,
184 : HCCL_ERROR("[AicpuIbverbs][PostSend] buffer size is:%u over MAX_MEM_NUM: %u", len, MAX_MEM_NUM), HCCL_E_PARA);
185 :
186 0 : MemDetails localMems[MAX_MEM_NUM];
187 0 : MemDetails remoteMems[MAX_MEM_NUM];
188 0 : u32 lkey = 0;
189 0 : u32 rkey = 0;
190 0 : for (uint32_t index = 0; index < len; index++) {
191 0 : u64 remBuffSize = remoteBuf[index].size;
192 0 : u64 locBuffSize = localBuf[index].size;
193 0 : CHK_PRT_RET(
194 : remBuffSize != locBuffSize,
195 : HCCL_ERROR(
196 : "[AicpuIbverbs][PostSend] remoteBuf size %u is not equal localBuffer size %u ", remBuffSize,
197 : locBuffSize),
198 : HCCL_E_PARA);
199 : // 获取WR的lkey和rkey
200 0 : CHK_RET(Getkey(ctx, remoteRankId, localBuf[index].addr, locBuffSize, lkey, LOCAL));
201 0 : CHK_RET(Getkey(ctx, remoteRankId, remoteBuf[index].addr, remBuffSize, rkey, REMOTE));
202 : // 设置MemDetails
203 0 : localMems[index].addr = reinterpret_cast<u64>(localBuf[index].addr);
204 0 : localMems[index].size = locBuffSize;
205 0 : localMems[index].key = lkey;
206 :
207 0 : remoteMems[index].addr = reinterpret_cast<u64>(remoteBuf[index].addr);
208 0 : remoteMems[index].size = remBuffSize;
209 0 : remoteMems[index].key = rkey;
210 : }
211 :
212 0 : u64 db_info = 0;
213 0 : u32 memNum = (ctx.ibversData[remoteRankId].qpMode != QPMode::NORMAL) ? 1 : len;
214 0 : CHK_RET(
215 : LIKELY(isWrite) ? hccl::Transport::HcclBatchWrite(
216 : ctx.ibversData[remoteRankId], &localMems[0], &remoteMems[0], memNum, db_info) :
217 : hccl::Transport::HcclBatchRead(
218 : ctx.ibversData[remoteRankId], &localMems[0], &remoteMems[0], memNum, db_info));
219 :
220 0 : if (UNLIKELY(ctx.ibversData[remoteRankId].qpMode != QPMode::NORMAL)) {
221 0 : for (u32 i = 1; i < len; i++) {
222 0 : CHK_RET(
223 : LIKELY(isWrite) ? hccl::Transport::HcclBatchWrite(
224 : ctx.ibversData[remoteRankId], &localMems[i], &remoteMems[i], 1, db_info) :
225 : hccl::Transport::HcclBatchRead(
226 : ctx.ibversData[remoteRankId], &localMems[i], &remoteMems[i], 1, db_info));
227 : }
228 0 : u64 roceBaseAddr = 0x2000000000ULL;
229 0 : u64 roceVfDbCfg0Reg = 0x230ULL;
230 0 : u64 chipAddrOffset = 0x80000000000ULL;
231 0 : u64 dieAddrOffset = 0x10000000000ULL;
232 0 : u64 dbDieIdMask = 0x00ff0000;
233 0 : u64 dbDieIdShift = 16; // 16 is dbDieIdShift
234 0 : u64 dbAddr = roceBaseAddr + roceVfDbCfg0Reg + chipAddrOffset * ctx.chipId
235 0 : + dieAddrOffset * ((ctx.ibversData[remoteRankId].qpInfo.dbIndex & dbDieIdMask) >> dbDieIdShift);
236 0 : HCCL_DEBUG("chipId : %llu", ctx.chipId);
237 0 : std::lock_guard<std::mutex> lock(g_mtxForDoorbell);
238 0 : CHK_RET(AicpuDispatcher::RdmaSend(0, db_info, dbAddr, remoteRankId));
239 0 : CHK_RET(AicpuDispatcher::LaunchTask(0));
240 0 : }
241 0 : return HCCL_SUCCESS;
242 : }
243 :
244 0 : HcclResult HcclAicpuUtils::PostSend(
245 : const u32 lKey, const u32 rKey, const struct HcclQpInfoV2& qpInfo, const struct hccl::Transport::Buffer& remoteBuf,
246 : const struct hccl::Transport::Buffer& localBuf, const bool isWrite)
247 : {
248 0 : MemDetails localMems;
249 0 : MemDetails remoteMems;
250 0 : u64 remainDataSize = localBuf.size;
251 0 : u64 remBuffSize = remainDataSize;
252 0 : u64 locBuffSize = remainDataSize;
253 :
254 : // 设置MemDetails
255 0 : localMems.addr = reinterpret_cast<u64>(localBuf.addr);
256 0 : localMems.size = locBuffSize;
257 0 : localMems.key = lKey;
258 :
259 0 : remoteMems.addr = reinterpret_cast<u64>(remoteBuf.addr);
260 0 : remoteMems.size = remBuffSize;
261 0 : remoteMems.key = rKey;
262 :
263 0 : u64 db_info = 0;
264 0 : u32 memNum = 1;
265 0 : struct hccl::TransportDeviceNormalData ibversDataforRemoteRank;
266 0 : ibversDataforRemoteRank.qpInfo = qpInfo;
267 0 : HCCL_INFO(
268 : "remBuffSize is [%u], locBuffSize is [%u], localMems.addr is [%p], localMems.size is [%u],"
269 : "localMems.key is [%u], remoteMems.addr is [%p], remoteMems.size is [%u], remoteMems.key is [%u],"
270 : "ibversDataforRemoteRank.qpInfo.qpPtr is [%p]",
271 : remBuffSize, locBuffSize, localMems.addr, localMems.size, localMems.key, remoteMems.addr, remoteMems.size,
272 : remoteMems.key, ibversDataforRemoteRank.qpInfo.qpPtr);
273 0 : while (remainDataSize > 0) {
274 0 : u64 chunkBytes = (remainDataSize > MAX_RDMA_WQE_SIZE) ? MAX_RDMA_WQE_SIZE : remainDataSize;
275 0 : localMems.size = chunkBytes;
276 0 : remoteMems.size = chunkBytes;
277 0 : CHK_RET(
278 : LIKELY(isWrite) ?
279 : hccl::Transport::HcclBatchWrite(ibversDataforRemoteRank, &localMems, &remoteMems, memNum, db_info) :
280 : hccl::Transport::HcclBatchRead(ibversDataforRemoteRank, &localMems, &remoteMems, memNum, db_info));
281 0 : localMems.addr += chunkBytes;
282 0 : remoteMems.addr += chunkBytes;
283 0 : remainDataSize -= chunkBytes;
284 : }
285 0 : return HCCL_SUCCESS;
286 : }
287 :
288 114 : u32 HcclAicpuUtils::GetBlockNum(u32 defaultVal)
289 : {
290 114 : if (AicpuGetBlockNum != nullptr) {
291 0 : return AicpuGetBlockNum();
292 114 : } else if (aicpu::GetBlockNum != nullptr) {
293 0 : return aicpu::GetBlockNum();
294 : } else {
295 114 : return defaultVal;
296 : }
297 : }
298 :
299 53 : u32 HcclAicpuUtils::GetBlockIdx()
300 : {
301 53 : u32 res = 0U;
302 53 : if (AicpuGetBlockIdx != nullptr) {
303 0 : res = AicpuGetBlockIdx();
304 53 : } else if (aicpu::GetBlockIdx != nullptr) {
305 0 : res = aicpu::GetBlockIdx();
306 : }
307 53 : return res;
308 : }
|
