Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "utils/hccl_aicpu_utils.h"
12 : #include <sstream>
13 : #include <dlog_pub.h>
14 :
15 : #include "algorithm/task_orchestrator.h"
16 : #include "common/aicpu_sqe_context.h"
17 : #include "common/aicpu_hccl_common.h"
18 : #include "profiling_manager_device.h"
19 : #include "framework/aicpu_communicator.h"
20 : #include "log.h"
21 : #include "hccl_types.h"
22 : #include "transport_pub.h"
23 :
24 46902 : int32_t HcclAicpuUtils::GetCpuId()
25 : {
26 : static thread_local int32_t curCpu = -1;
27 46902 : if (curCpu < 0) {
28 2 : curCpu = sched_getcpu();
29 : }
30 46902 : return curCpu;
31 : }
32 :
33 46980 : int32_t HcclAicpuUtils::GetCurClusterId()
34 : {
35 46980 : return !!(GetCpuId() & (AICPU_CNT / CLUSTER_CNT));
36 : }
37 :
38 81 : void HcclAicpuUtils::PrintHcclCombinOpParam(const HccCommResParamTask &commParam)
39 : {
40 81 : if (!HcclCheckLogLevel(HCCL_LOG_INFO)) {
41 0 : return;
42 : }
43 81 : HCCL_INFO("HccCommResParamTask.workSpace %p", commParam.mc2WorkSpace.workSpace);
44 81 : HCCL_INFO("HccCommResParamTask.workSpaceSize %lu", commParam.mc2WorkSpace.workSpaceSize);
45 81 : HCCL_INFO("HccCommResParamTask.rankId %u", commParam.rankId);
46 81 : HCCL_INFO("HccCommResParamTask.rankNum %u", commParam.rankNum);
47 81 : HCCL_INFO("HccCommResParamTask.winSize %lu", commParam.winSize);
48 2673 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM; i++) {
49 2592 : HCCL_INFO("HccCommResParamTask.windowsIn[%u] %p, windowsOut[%u] %p",
50 : i, commParam.windowsIn[i], i, commParam.windowsOut[i]);
51 : }
52 2673 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM; i++) {
53 2592 : const HcclStreamInfo &sinfo = commParam.streamInfo[i];
54 2592 : HCCL_INFO("HccCommResParamTask.streamInfo[%u] streamId %d, sqId %u, cqId %lu logicCqid %u",
55 : i,
56 : sinfo.streamIds,
57 : sinfo.sqIds,
58 : sinfo.cqIds,
59 : sinfo.logicCqids);
60 : }
61 5265 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM * 2; i++) { // 2 is number of noIpcNotify
62 5184 : const HcclSignalInfo &sinfo = commParam.signalInfo.noIpcNotifys[i];
63 5184 : HCCL_INFO("HccCommResParamTask.noIpcNotifys[%u] resId %lu, addr %p, devId %u, tsId %u, rankId %u", i,
64 : sinfo.resId, sinfo.addr, sinfo.devId, sinfo.tsId, sinfo.rankId);
65 : }
66 10449 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM * 4; i++) { // 4 is number of ipcNotifys
67 10368 : const HcclSignalInfo &sinfo = commParam.signalInfo.ipcNotifys[i];
68 10368 : HCCL_INFO("HccCommResParamTask.ipcNotifys[%u] resId %lu, addr %p, devId %u, tsId %u, rankId %u", i,
69 : sinfo.resId, sinfo.addr, sinfo.devId, sinfo.tsId, sinfo.rankId);
70 : }
71 2673 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM; i++) {
72 2592 : const HcclSignalInfo &sinfo = commParam.signalInfo.noIpcEvents[i];
73 2592 : HCCL_INFO("HccCommResParamTask.noIpcEvents[%u] resId %lu, addr %p, devId %u, tsId %u, rankId %u", i,
74 : sinfo.resId, sinfo.addr, sinfo.devId, sinfo.tsId, sinfo.rankId);
75 : }
76 :
77 243 : for (uint32_t i = 0; i < AICPU_OP_NOTIFY_NUM; i++) {
78 162 : const HcclSignalInfo &sinfo = commParam.signalInfo.aicpuOpNotify[i];
79 162 : HCCL_INFO("HccCommResParamTask.aicpuOpNotify[%u] resId %lu, addr %p, devId %u, tsId %u, rankId %u", i,
80 : sinfo.resId, sinfo.addr, sinfo.devId, sinfo.tsId, sinfo.rankId);
81 : }
82 81 : const auto &sigInfo = commParam.signalInfo.aicpuNotify;
83 81 : HCCL_INFO("HccCommResParamTask.aicpuNotify resId %lu, addr %p, devId %u, tsId %u, rankId %u", sigInfo.resId,
84 : sigInfo.addr, sigInfo.devId, sigInfo.tsId, sigInfo.rankId);
85 81 : HCCL_INFO("HccCommResParamTask.determinism %u", commParam.config.deterministic);
86 81 : HCCL_INFO("HccCommResParamTask.overflowAddr %p", commParam.overFlowAddr);
87 81 : HCCL_INFO("HccCommResParamTask.retryParams: retryEnable %u", commParam.config.retryEnable);
88 : }
89 :
90 14 : void HcclAicpuUtils::PrintHcclOpResParam(const HcclOpResParam *resParam)
91 : {
92 14 : if (resParam == nullptr) {
93 1 : HCCL_ERROR("[%s] resParam is nullptr", __func__);
94 1 : return;
95 : }
96 13 : HCCL_INFO("HcclOpResParam.rankId %u", resParam->localUsrRankId);
97 13 : HCCL_INFO("HcclOpResParam.rankNum %u", resParam->rankSize);
98 :
99 13 : HCCL_INFO("HcclOpResParam.windowSize %lu", resParam->winSize);
100 13 : HCCL_INFO("HcclOpResParam.workSpaceAddr %p", resParam->mc2WorkSpace.workSpace);
101 13 : HCCL_INFO("HcclOpResParam.workSpaceSize %lu", resParam->mc2WorkSpace.workSpaceSize);
102 373 : for (uint32_t i = 0; i < resParam->localRes.streamNum; i++) {
103 360 : const auto streamInfo = resParam->localRes.streamParam[i].streamInfo;
104 360 : HCCL_INFO("HcclOpResParam.streamInfo[%u] streamId %d sqId %u cqId %u logicCqid %u", i, streamInfo.streamIds,
105 : streamInfo.sqIds, streamInfo.cqIds, streamInfo.logicCqids);
106 : }
107 13 : const auto mainStreamInfo = resParam->localRes.mainStreamParam.streamInfo;
108 13 : HCCL_INFO("HcclOpResParam.mainStreamInfo streamId %d sqId %u cqId %u logicCqid %u", mainStreamInfo.streamIds,
109 : mainStreamInfo.sqIds, mainStreamInfo.cqIds, mainStreamInfo.logicCqids);
110 589 : for (uint32_t i = 0; i < resParam->localRes.signalNum; i++) {
111 576 : const auto &signals = resParam->localRes.localSignals[i];
112 576 : HCCL_INFO("HcclOpResParam.localSignals[%u] resId %p addr %p devId %u tsId %u rankId %u", i, signals.resId,
113 : signals.addr, signals.devId, signals.tsId, signals.rankId);
114 : }
115 :
116 39 : for (uint32_t i = 0; i < AICPU_OP_NOTIFY_MAX_NUM; i++) {
117 26 : const auto &aicpuOpNotify = resParam->localRes.aicpuOpNotify[i];
118 26 : HCCL_INFO("HcclOpResParam.aicpuOpNotify[%u] resId %p addr %p devId %u tsId %u rankId %u", i, aicpuOpNotify.resId,
119 : aicpuOpNotify.addr, aicpuOpNotify.devId, aicpuOpNotify.tsId, aicpuOpNotify.rankId);
120 : }
121 13 : HCCL_INFO("HcclOpResParam.determinism %u", resParam->config.deterministic);
122 : }
123 :
124 1 : HcclResult HcclAicpuUtils::Getkey(const AicpuComContext &ctx, u32 remoteRankId, const void *userAddr,
125 : u64 length, u32 &outKey, int32_t keyType)
126 : {
127 1 : CHK_PTR_NULL(userAddr);
128 0 : HCCL_INFO("[HcclAicpuUtils][Getkey] addr[%p] len[%llu]", userAddr, length);
129 0 : u64 inAddr = reinterpret_cast<u64>(userAddr);
130 0 : MemDetails inputMem = (keyType == LOCAL) ? ctx.ibversData[remoteRankId].localInputMem : ctx.ibversData[remoteRankId].remoteInputMem;
131 0 : MemDetails outputMem = (keyType == LOCAL) ? ctx.ibversData[remoteRankId].localOutputMem : ctx.ibversData[remoteRankId].remoteOutputMem;
132 :
133 0 : u64 inputStartAddr = inputMem.addr;
134 0 : u64 inputCCLSize = inputMem.size;
135 0 : u32 inputKey = inputMem.key;
136 :
137 0 : u64 outputStartAddr = outputMem.addr;
138 0 : u64 outputCCLSize = outputMem.size;
139 0 : u32 outputKey = outputMem.key;
140 0 : if (inAddr >= inputStartAddr && inAddr < inputStartAddr + inputCCLSize) {
141 0 : outKey = inputKey;
142 0 : } else if (inAddr >= outputStartAddr && inAddr <= outputStartAddr + outputCCLSize) {
143 0 : outKey = outputKey;
144 : } else {
145 0 : HCCL_ERROR("[HcclAicpuUtils][Getkey]src_ptr=%p is out of range, inputmem src[%p], size[%llu];"
146 : " outputmem src[%p] size[%llu]",
147 : userAddr, inputStartAddr, inputCCLSize, outputStartAddr, outputCCLSize);
148 0 : return HCCL_E_INTERNAL;
149 : }
150 0 : HCCL_INFO("[HcclAicpuUtils][Getkey] addr[%p] length[%llu] outKey[%u], keyType:[%s]",
151 : userAddr, length, outKey, (keyType == LOCAL) ? "local" : "remote");
152 :
153 0 : return HCCL_SUCCESS;
154 : }
155 :
156 : std::mutex g_mtxForDoorbell;
157 0 : HcclResult HcclAicpuUtils::PostSend(const AicpuComContext &ctx, u32 remoteRankId, struct std::vector<hccl::Transport::Buffer> &remoteBuf,
158 : struct std::vector<hccl::Transport::Buffer> &localBuf, bool isWrite)
159 : {
160 0 : if (UNLIKELY(remoteRankId >= ctx.rankNum)) {
161 0 : HCCL_ERROR("[AicpuIbverbs][PostSend] remoteRankId %u is out of range, ranknum %u",remoteRankId, ctx.rankNum);
162 0 : return HCCL_E_PARA;
163 : }
164 0 : CHK_PRT_RET(remoteBuf.size() != localBuf.size(),
165 : HCCL_ERROR("[AicpuIbverbs][PostSend] remoteBuf list size %u is not equal localBuffer list size %u ",
166 : remoteBuf.size(), localBuf.size()), HCCL_E_PARA);
167 :
168 0 : uint32_t len = remoteBuf.size();
169 0 : const uint32_t MAX_MEM_NUM = 8;
170 0 : CHK_PRT_RET(len > MAX_MEM_NUM,
171 : HCCL_ERROR("[AicpuIbverbs][PostSend] buffer size is:%u over MAX_MEM_NUM: %u", len, MAX_MEM_NUM), HCCL_E_PARA);
172 :
173 0 : MemDetails localMems[MAX_MEM_NUM];
174 0 : MemDetails remoteMems[MAX_MEM_NUM];
175 0 : u32 lkey = 0;
176 0 : u32 rkey = 0;
177 0 : for (uint32_t index = 0; index < len; index++) {
178 0 : u64 remBuffSize = remoteBuf[index].size;
179 0 : u64 locBuffSize = localBuf[index].size;
180 0 : CHK_PRT_RET(remBuffSize != locBuffSize,
181 : HCCL_ERROR("[AicpuIbverbs][PostSend] remoteBuf size %u is not equal localBuffer size %u ",
182 : remBuffSize, locBuffSize), HCCL_E_PARA);
183 : // 获取WR的lkey和rkey
184 0 : CHK_RET(Getkey(ctx, remoteRankId, localBuf[index].addr, locBuffSize, lkey, LOCAL));
185 0 : CHK_RET(Getkey(ctx, remoteRankId, remoteBuf[index].addr, remBuffSize, rkey, REMOTE));
186 : // 设置MemDetails
187 0 : localMems[index].addr = reinterpret_cast<u64>(localBuf[index].addr);
188 0 : localMems[index].size = locBuffSize;
189 0 : localMems[index].key = lkey;
190 :
191 0 : remoteMems[index].addr = reinterpret_cast<u64>(remoteBuf[index].addr);
192 0 : remoteMems[index].size = remBuffSize;
193 0 : remoteMems[index].key = rkey;
194 : }
195 :
196 0 : u64 db_info = 0;
197 0 : u32 memNum = (ctx.ibversData[remoteRankId].qpMode != QPMode::NORMAL) ? 1 : len;
198 0 : CHK_RET(LIKELY(isWrite) ?
199 : hccl::Transport::HcclBatchWrite(ctx.ibversData[remoteRankId], &localMems[0], &remoteMems[0], memNum, db_info) :
200 : hccl::Transport::HcclBatchRead(ctx.ibversData[remoteRankId], &localMems[0], &remoteMems[0], memNum, db_info));
201 :
202 0 : if (UNLIKELY(ctx.ibversData[remoteRankId].qpMode != QPMode::NORMAL)) {
203 0 : for (u32 i = 1; i < len; i++) {
204 0 : CHK_RET(LIKELY(isWrite) ?
205 : hccl::Transport::HcclBatchWrite(ctx.ibversData[remoteRankId], &localMems[i],
206 : &remoteMems[i], 1, db_info) :
207 : hccl::Transport::HcclBatchRead(ctx.ibversData[remoteRankId], &localMems[i],
208 : &remoteMems[i], 1, db_info));
209 : }
210 0 : u64 roceBaseAddr = 0x2000000000ULL;
211 0 : u64 roceVfDbCfg0Reg = 0x230ULL;
212 0 : u64 chipAddrOffset = 0x80000000000ULL;
213 0 : u64 dieAddrOffset = 0x10000000000ULL;
214 0 : u64 dbDieIdMask = 0x00ff0000;
215 0 : u64 dbDieIdShift = 16; // 16 is dbDieIdShift
216 0 : u64 dbAddr = roceBaseAddr + roceVfDbCfg0Reg + chipAddrOffset * ctx.chipId +
217 0 : dieAddrOffset * ((ctx.ibversData[remoteRankId].qpInfo.dbIndex & dbDieIdMask) >> dbDieIdShift);
218 0 : HCCL_DEBUG("chipId : %llu", ctx.chipId);
219 0 : std::lock_guard<std::mutex> lock(g_mtxForDoorbell);
220 0 : CHK_RET(AicpuDispatcher::RdmaSend(0, db_info, dbAddr, remoteRankId));
221 0 : CHK_RET(AicpuDispatcher::LaunchTask(0));
222 0 : }
223 0 : return HCCL_SUCCESS;
224 : }
225 :
226 0 : HcclResult HcclAicpuUtils::PostSend(const u32 lKey, const u32 rKey, const struct HcclQpInfoV2 &qpInfo,
227 : const struct hccl::Transport::Buffer &remoteBuf, const struct hccl::Transport::Buffer &localBuf, const bool isWrite)
228 : {
229 0 : MemDetails localMems;
230 0 : MemDetails remoteMems;
231 0 : u64 remainDataSize = localBuf.size;
232 0 : u64 remBuffSize = remainDataSize;
233 0 : u64 locBuffSize = remainDataSize;
234 :
235 : // 设置MemDetails
236 0 : localMems.addr = reinterpret_cast<u64>(localBuf.addr);
237 0 : localMems.size = locBuffSize;
238 0 : localMems.key = lKey;
239 :
240 0 : remoteMems.addr = reinterpret_cast<u64>(remoteBuf.addr);
241 0 : remoteMems.size = remBuffSize;
242 0 : remoteMems.key = rKey;
243 :
244 0 : u64 db_info = 0;
245 0 : u32 memNum = 1;
246 0 : struct hccl::TransportDeviceNormalData ibversDataforRemoteRank;
247 0 : ibversDataforRemoteRank.qpInfo = qpInfo;
248 0 : HCCL_INFO("remBuffSize is [%u], locBuffSize is [%u], localMems.addr is [%p], localMems.size is [%u],"
249 : "localMems.key is [%u], remoteMems.addr is [%p], remoteMems.size is [%u], remoteMems.key is [%u],"
250 : "ibversDataforRemoteRank.qpInfo.qpPtr is [%p]", remBuffSize, locBuffSize, localMems.addr, localMems.size,
251 : localMems.key, remoteMems.addr, remoteMems.size, remoteMems.key, ibversDataforRemoteRank.qpInfo.qpPtr);
252 0 : while (remainDataSize > 0) {
253 0 : u64 chunkBytes = (remainDataSize > MAX_RDMA_WQE_SIZE) ? MAX_RDMA_WQE_SIZE : remainDataSize;
254 0 : localMems.size = chunkBytes;
255 0 : remoteMems.size = chunkBytes;
256 0 : CHK_RET(LIKELY(isWrite) ?
257 : hccl::Transport::HcclBatchWrite(ibversDataforRemoteRank, &localMems, &remoteMems, memNum, db_info) :
258 : hccl::Transport::HcclBatchRead(ibversDataforRemoteRank, &localMems, &remoteMems, memNum, db_info));
259 0 : localMems.addr += chunkBytes;
260 0 : remoteMems.addr += chunkBytes;
261 0 : remainDataSize -= chunkBytes;
262 : }
263 0 : return HCCL_SUCCESS;
264 : }
265 :
266 114 : u32 HcclAicpuUtils::GetBlockNum(u32 defaultVal) {
267 114 : if (AicpuGetBlockNum != nullptr) {
268 0 : return AicpuGetBlockNum();
269 114 : } else if (aicpu::GetBlockNum != nullptr) {
270 0 : return aicpu::GetBlockNum();
271 : } else {
272 114 : return defaultVal;
273 : }
274 : }
275 :
276 53 : u32 HcclAicpuUtils::GetBlockIdx() {
277 53 : u32 res = 0U;
278 53 : if (AicpuGetBlockIdx != nullptr) {
279 0 : res = AicpuGetBlockIdx();
280 53 : } else if (aicpu::GetBlockIdx != nullptr) {
281 0 : res = aicpu::GetBlockIdx();
282 : }
283 53 : return res;
284 : }
|
