Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hcomm_team_mgr.h"
12 :
13 : #include <cstdlib>
14 :
15 : #include "adapter_rts_common.h"
16 : #include "hcomm_res_entity_defs.h"
17 : #include "hcomm_result_defs.h"
18 : #include "hcomm_team_entity_defs.h"
19 : #include "log.h"
20 : #include "securec.h"
21 :
22 : namespace hcomm {
23 :
24 227 : HcommTeamMgr& HcommTeamMgr::GetInstance()
25 : {
26 : static std::once_flag instanceFlag;
27 : static HcommTeamMgr* instance = nullptr;
28 227 : std::call_once(instanceFlag, [&] {
29 1 : instance = new HcommTeamMgr();
30 1 : });
31 227 : return *instance;
32 : }
33 :
34 0 : HcommTeamMgr::~HcommTeamMgr()
35 : {
36 : {
37 0 : std::unique_lock<std::shared_mutex> lock(teamsRwMutex_);
38 0 : for (auto& pair : teams_) {
39 0 : if (pair.second != nullptr) {
40 0 : FreeTeamResources(pair.second.get());
41 : }
42 : }
43 0 : teams_.clear();
44 0 : }
45 : {
46 0 : std::unique_lock<std::shared_mutex> lock(windowsRwMutex_);
47 0 : for (auto& pair : windows_) {
48 0 : if (pair.second != nullptr) {
49 0 : FreeWindowResources(pair.second.get());
50 : }
51 : }
52 0 : windows_.clear();
53 0 : }
54 : {
55 0 : std::unique_lock<std::shared_mutex> lock(windowToTeamRwMutex_);
56 0 : windowToTeamMap_.clear();
57 0 : }
58 0 : }
59 :
60 73 : TeamEntry* HcommTeamMgr::FindTeamByHandleLocked(HcommTeamHandle handle)
61 : {
62 73 : auto it = teams_.find(handle);
63 73 : if (it == teams_.end()) {
64 6 : HCCL_ERROR("[FindTeamByHandleLocked] team handle[%p] not found", handle);
65 6 : return nullptr;
66 : }
67 67 : return it->second.get();
68 : }
69 :
70 19 : WindowEntry* HcommTeamMgr::FindWindowByHandleLocked(HcommWindowHandle handle)
71 : {
72 19 : auto it = windows_.find(handle);
73 19 : if (it == windows_.end()) {
74 1 : HCCL_ERROR("[FindWindowByHandleLocked] window handle[%p] not found", handle);
75 1 : return nullptr;
76 : }
77 18 : return it->second.get();
78 : }
79 :
80 65 : HcommResult HcommTeamMgr::SyncTeamToDevice(TeamEntry* entry)
81 : {
82 65 : if (entry->devTeam == nullptr) {
83 0 : HCCL_ERROR("[SyncTeamToDevice] devTeam is null");
84 0 : return HCOMM_E_PTR;
85 : }
86 130 : return static_cast<HcommResult>(hrtMemSyncCopy(
87 65 : entry->devTeam, sizeof(HcommTeam), &entry->hostTeam, sizeof(HcommTeam),
88 65 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
89 : }
90 :
91 29 : HcommResult HcommTeamMgr::SyncWindowToDevice(WindowEntry* entry)
92 : {
93 29 : if (entry->devWindow == nullptr) {
94 0 : HCCL_ERROR("[SyncWindowToDevice] devWindow is null");
95 0 : return HCOMM_E_PTR;
96 : }
97 58 : return static_cast<HcommResult>(hrtMemSyncCopy(
98 29 : entry->devWindow, sizeof(HcommWindow), &entry->hostWindow, sizeof(HcommWindow),
99 29 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
100 : }
101 :
102 54 : HcommResult HcommTeamMgr::AllocAndCopyWorldTeamIds(TeamEntry* entry, const uint32_t* src, uint32_t memberNum)
103 : {
104 : // worldTeam（src==nullptr）：worldTeamIds 置为 [0,memberNum) 连续序列（memberId 自身）
105 54 : std::vector<uint32_t> sequentialIds;
106 54 : const uint32_t* srcPtr = src;
107 54 : if (src == nullptr) {
108 0 : sequentialIds.resize(memberNum);
109 0 : for (uint32_t i = 0; i < memberNum; i++) {
110 0 : sequentialIds[i] = i;
111 : }
112 0 : srcPtr = sequentialIds.data();
113 : }
114 :
115 54 : void* devPtr = nullptr;
116 54 : HcommResult ret = static_cast<HcommResult>(hrtMalloc(&devPtr, static_cast<uint64_t>(memberNum * sizeof(uint32_t))));
117 54 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[AllocAndCopyWorldTeamIds] hrtMalloc failed, ret[%d]", ret), ret);
118 :
119 106 : ret = static_cast<HcommResult>(hrtMemSyncCopy(
120 53 : devPtr, static_cast<uint64_t>(memberNum * sizeof(uint32_t)), srcPtr,
121 53 : static_cast<uint64_t>(memberNum * sizeof(uint32_t)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
122 53 : if (ret != HCOMM_SUCCESS) {
123 1 : HCCL_ERROR("[AllocAndCopyWorldTeamIds] hrtMemSyncCopy failed, ret[%d]", ret);
124 1 : (void)hrtFree(devPtr);
125 1 : return ret;
126 : }
127 :
128 52 : entry->devWorldTeamIds = devPtr;
129 52 : entry->hostTeam.worldTeamIds = static_cast<uint32_t*>(devPtr);
130 :
131 52 : entry->hostWorldTeamIds = static_cast<uint32_t*>(malloc(memberNum * sizeof(uint32_t)));
132 52 : if (entry->hostWorldTeamIds == nullptr) {
133 0 : HCCL_ERROR("[AllocAndCopyWorldTeamIds] malloc hostWorldTeamIds failed");
134 0 : (void)hrtFree(devPtr);
135 0 : entry->devWorldTeamIds = nullptr;
136 0 : entry->hostTeam.worldTeamIds = nullptr;
137 0 : return HCOMM_E_MEMORY;
138 : }
139 : errno_t memRet
140 52 : = memcpy_s(entry->hostWorldTeamIds, memberNum * sizeof(uint32_t), srcPtr, memberNum * sizeof(uint32_t));
141 52 : if (memRet != EOK) {
142 0 : HCCL_ERROR("[AllocAndCopyWorldTeamIds] memcpy_s hostWorldTeamIds failed, ret[%d]", memRet);
143 0 : free(entry->hostWorldTeamIds);
144 0 : entry->hostWorldTeamIds = nullptr;
145 0 : (void)hrtFree(devPtr);
146 0 : entry->devWorldTeamIds = nullptr;
147 0 : entry->hostTeam.worldTeamIds = nullptr;
148 0 : return HCOMM_E_MEMORY;
149 : }
150 :
151 52 : return HCOMM_SUCCESS;
152 54 : }
153 :
154 6 : HcommResult HcommTeamMgr::AllocChannelEntities(TeamEntry* entry)
155 : {
156 6 : uint32_t memberNum = entry->hostTeam.memberNum;
157 6 : entry->hostChannelNums.resize(memberNum, 0);
158 6 : uint32_t totalChannels = 0;
159 30 : for (uint32_t i = 0; i < memberNum; i++) {
160 24 : uint32_t chNum = static_cast<uint32_t>(entry->channelsList[i].size());
161 24 : entry->hostChannelNums[i] = chNum;
162 24 : totalChannels += chNum;
163 : }
164 :
165 6 : if (totalChannels == 0) {
166 0 : entry->devChannels = nullptr;
167 0 : entry->hostTeam.channelsBaseAddr = 0;
168 0 : return HCOMM_SUCCESS;
169 : }
170 :
171 6 : void* devPtr = nullptr;
172 : HcommResult ret
173 6 : = static_cast<HcommResult>(hrtMalloc(&devPtr, static_cast<uint64_t>(totalChannels * sizeof(ChannelEntity))));
174 6 : CHK_PRT_RET(
175 : ret != HCOMM_SUCCESS,
176 : HCCL_ERROR("[AllocChannelEntities] hrtMalloc failed, totalChannels[%u] ret[%d]", totalChannels, ret), ret);
177 :
178 : // 逐个 channel 做 D2D 拷贝：把 ChannelHandle 指向的 ChannelEntity 本体拷贝到连续内存对应偏移
179 : // 偏移按前缀和 sum(channelNumPerMember[0..peer-1]) + channelIdx 计算
180 5 : uint32_t offset = 0;
181 25 : for (uint32_t peer = 0; peer < memberNum; peer++) {
182 48 : for (uint32_t idx = 0; idx < entry->channelsList[peer].size(); idx++) {
183 28 : void* dst = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(devPtr) + offset * sizeof(ChannelEntity));
184 28 : void* src = reinterpret_cast<void*>(static_cast<uintptr_t>(entry->channelsList[peer][idx]));
185 28 : ret = static_cast<HcommResult>(hrtMemSyncCopy(
186 : dst, sizeof(ChannelEntity), src, sizeof(ChannelEntity),
187 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_DEVICE));
188 28 : if (ret != HCOMM_SUCCESS) {
189 0 : HCCL_ERROR(
190 : "[AllocChannelEntities] hrtMemSyncCopy D2D failed, peer[%u] idx[%u] ret[%d]", peer, idx, ret);
191 0 : (void)hrtFree(devPtr);
192 0 : return ret;
193 : }
194 28 : offset++;
195 : }
196 : }
197 :
198 5 : entry->devChannels = devPtr;
199 5 : entry->hostTeam.channelsBaseAddr = reinterpret_cast<uint64_t>(devPtr);
200 5 : return HCOMM_SUCCESS;
201 : }
202 :
203 5 : HcommResult HcommTeamMgr::AllocChannelNumsArray(TeamEntry* entry)
204 : {
205 5 : uint32_t memberNum = entry->hostTeam.memberNum;
206 5 : void* devPtr = nullptr;
207 5 : HcommResult ret = static_cast<HcommResult>(hrtMalloc(&devPtr, static_cast<uint64_t>(memberNum * sizeof(uint32_t))));
208 5 : if (ret != HCOMM_SUCCESS) {
209 0 : HCCL_ERROR("[AllocChannelNumsArray] hrtMalloc channelNums failed, ret[%d]", ret);
210 0 : return ret;
211 : }
212 :
213 10 : ret = static_cast<HcommResult>(hrtMemSyncCopy(
214 5 : devPtr, static_cast<uint64_t>(memberNum * sizeof(uint32_t)), entry->hostChannelNums.data(),
215 5 : static_cast<uint64_t>(memberNum * sizeof(uint32_t)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
216 5 : if (ret != HCOMM_SUCCESS) {
217 0 : HCCL_ERROR("[AllocChannelNumsArray] hrtMemSyncCopy channelNums failed, ret[%d]", ret);
218 0 : (void)hrtFree(devPtr);
219 0 : return ret;
220 : }
221 5 : entry->devChannelNums = devPtr;
222 5 : entry->hostTeam.channelNumPerMember = static_cast<uint32_t*>(devPtr);
223 5 : return HCOMM_SUCCESS;
224 : }
225 :
226 6 : HcommResult HcommTeamMgr::AllocAndCopyChannels(TeamEntry* entry)
227 : {
228 6 : uint32_t memberNum = entry->hostTeam.memberNum;
229 6 : if (memberNum == 0 || entry->channelsList.empty()) {
230 0 : HCCL_ERROR("[AllocAndCopyChannels] memberNum[%u] or channelsList is empty", memberNum);
231 0 : return HCOMM_E_PARA;
232 : }
233 :
234 6 : FreeDeviceChannels(entry);
235 :
236 6 : HcommResult ret = AllocChannelEntities(entry);
237 6 : if (ret != HCOMM_SUCCESS) {
238 1 : HCCL_ERROR("[AllocAndCopyChannels] AllocChannelEntities failed, ret[%d]", ret);
239 1 : return ret;
240 : }
241 :
242 5 : ret = AllocChannelNumsArray(entry);
243 5 : if (ret != HCOMM_SUCCESS) {
244 0 : HCCL_ERROR("[AllocAndCopyChannels] AllocChannelNumsArray failed, ret[%d]", ret);
245 0 : FreeDeviceChannels(entry);
246 0 : return ret;
247 : }
248 :
249 5 : return HCOMM_SUCCESS;
250 : }
251 :
252 10 : HcommResult HcommTeamMgr::AllocAndCopyRemoteMems(TeamEntry* entry, const CommMem* src, uint32_t memberNum)
253 : {
254 10 : if (entry->hostRemoteMems == nullptr) {
255 8 : return AllocRemoteMems(entry, src, memberNum);
256 : }
257 2 : return UpdateRemoteMems(entry, src, memberNum);
258 : }
259 :
260 8 : HcommResult HcommTeamMgr::AllocRemoteMems(TeamEntry* entry, const CommMem* src, uint32_t memberNum)
261 : {
262 : // 首次分配：calloc hostRemoteMems + memcpy 拷入；hrtMalloc devRemoteMems + hrtMemSyncCopy 到 device
263 8 : entry->hostRemoteMems = static_cast<CommMem*>(calloc(memberNum, sizeof(CommMem)));
264 8 : if (entry->hostRemoteMems == nullptr) {
265 0 : HCCL_ERROR("[AllocRemoteMems] calloc hostRemoteMems failed");
266 0 : return HCOMM_E_MEMORY;
267 : }
268 8 : errno_t memRet = memcpy_s(entry->hostRemoteMems, memberNum * sizeof(CommMem), src, memberNum * sizeof(CommMem));
269 8 : if (memRet != EOK) {
270 0 : HCCL_ERROR("[AllocRemoteMems] memcpy_s failed, ret[%d]", memRet);
271 0 : free(entry->hostRemoteMems);
272 0 : entry->hostRemoteMems = nullptr;
273 0 : return HCOMM_E_MEMORY;
274 : }
275 :
276 8 : void* devPtr = nullptr;
277 8 : HcommResult ret = static_cast<HcommResult>(hrtMalloc(&devPtr, static_cast<uint64_t>(memberNum * sizeof(CommMem))));
278 8 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[AllocRemoteMems] hrtMalloc failed, ret[%d]", ret), ret);
279 :
280 14 : ret = static_cast<HcommResult>(hrtMemSyncCopy(
281 7 : devPtr, static_cast<uint64_t>(memberNum * sizeof(CommMem)), src,
282 7 : static_cast<uint64_t>(memberNum * sizeof(CommMem)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
283 7 : if (ret != HCOMM_SUCCESS) {
284 0 : HCCL_ERROR("[AllocRemoteMems] hrtMemSyncCopy failed, ret[%d]", ret);
285 0 : (void)hrtFree(devPtr);
286 0 : return ret;
287 : }
288 :
289 7 : entry->devRemoteMems = devPtr;
290 7 : entry->hostTeam.syncMem.remoteMems = static_cast<CommMem*>(devPtr);
291 7 : entry->hostTeam.syncMem.remoteMemsNum = memberNum;
292 7 : return HCOMM_SUCCESS;
293 : }
294 :
295 2 : HcommResult HcommTeamMgr::UpdateRemoteMems(TeamEntry* entry, const CommMem* src, uint32_t memberNum)
296 : {
297 : // 重复 bind：校验维度一致后更新 hostRemoteMems 并重新 sync 到 devRemoteMems
298 2 : CHK_PRT_RET(
299 : memberNum != entry->hostTeam.syncMem.remoteMemsNum,
300 : HCCL_ERROR(
301 : "[UpdateRemoteMems] memberNum[%u] != remoteMemNum[%u], dimension mismatch", memberNum,
302 : entry->hostTeam.syncMem.remoteMemsNum),
303 : HCOMM_E_PARA);
304 :
305 2 : errno_t memRet = memcpy_s(entry->hostRemoteMems, memberNum * sizeof(CommMem), src, memberNum * sizeof(CommMem));
306 2 : CHK_PRT_RET(memRet != EOK, HCCL_ERROR("[UpdateRemoteMems] memcpy_s failed, ret[%d]", memRet), HCOMM_E_MEMORY);
307 :
308 4 : HcclResult hrtRet = hrtMemSyncCopy(
309 2 : entry->devRemoteMems, static_cast<uint64_t>(memberNum * sizeof(CommMem)), entry->hostRemoteMems,
310 2 : static_cast<uint64_t>(memberNum * sizeof(CommMem)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE);
311 2 : HcommResult ret = static_cast<HcommResult>(hrtRet);
312 2 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[UpdateRemoteMems] hrtMemSyncCopy failed, ret[%d]", ret), ret);
313 2 : return HCOMM_SUCCESS;
314 : }
315 :
316 60 : void HcommTeamMgr::FreeDeviceChannels(TeamEntry* entry)
317 : {
318 60 : if (entry->devChannels != nullptr) {
319 5 : (void)hrtFree(entry->devChannels);
320 5 : entry->devChannels = nullptr;
321 : }
322 60 : if (entry->devChannelNums != nullptr) {
323 5 : (void)hrtFree(entry->devChannelNums);
324 5 : entry->devChannelNums = nullptr;
325 : }
326 60 : entry->hostTeam.channelsBaseAddr = 0;
327 60 : entry->hostTeam.channelNumPerMember = nullptr;
328 60 : }
329 :
330 54 : void HcommTeamMgr::FreeTeamResources(TeamEntry* entry)
331 : {
332 54 : if (entry->devRemoteMems != nullptr) {
333 7 : (void)hrtFree(entry->devRemoteMems);
334 7 : entry->devRemoteMems = nullptr;
335 : }
336 54 : if (entry->hostRemoteMems != nullptr) {
337 8 : free(entry->hostRemoteMems);
338 8 : entry->hostRemoteMems = nullptr;
339 : }
340 :
341 54 : FreeDeviceChannels(entry);
342 :
343 54 : if (entry->devWorldTeamIds != nullptr) {
344 52 : (void)hrtFree(entry->devWorldTeamIds);
345 52 : entry->devWorldTeamIds = nullptr;
346 : }
347 54 : if (entry->hostWorldTeamIds != nullptr) {
348 52 : free(entry->hostWorldTeamIds);
349 52 : entry->hostWorldTeamIds = nullptr;
350 : }
351 :
352 54 : if (entry->hostTeam.syncMem.shadowMem.addr != nullptr) {
353 7 : (void)hrtFree(entry->hostTeam.syncMem.shadowMem.addr);
354 7 : entry->hostTeam.syncMem.shadowMem.addr = nullptr;
355 7 : entry->hostTeam.syncMem.shadowMem.size = 0;
356 : }
357 :
358 54 : if (entry->devTeam != nullptr) {
359 51 : (void)hrtFree(entry->devTeam);
360 51 : entry->devTeam = nullptr;
361 : }
362 :
363 54 : entry->channelsList.clear();
364 54 : entry->hostChannelNums.clear();
365 54 : }
366 :
367 19 : void HcommTeamMgr::FreeWindowResources(WindowEntry* entry)
368 : {
369 19 : if (entry->devMems != nullptr) {
370 8 : (void)hrtFree(entry->devMems);
371 8 : entry->devMems = nullptr;
372 : }
373 19 : if (entry->hostMems != nullptr) {
374 8 : free(entry->hostMems);
375 8 : entry->hostMems = nullptr;
376 : }
377 19 : if (entry->devWindow != nullptr) {
378 19 : (void)hrtFree(entry->devWindow);
379 19 : entry->devWindow = nullptr;
380 : }
381 19 : }
382 :
383 54 : HcommResult HcommTeamMgr::ValidateSubTeam(TeamEntry* worldEntry, const HcommTeamCreateDesc* desc)
384 : {
385 : /* worldEntry 为 nullptr 表示创建 world team（无父 team），无需 sub team 成员校验。 */
386 54 : CHK_PRT_RET(
387 : worldEntry == nullptr, HCCL_INFO("[TeamCreate] worldEntry is null, skip sub team validation"), HCOMM_SUCCESS);
388 3 : CHK_PRT_RET(
389 : desc->memberNum > worldEntry->hostTeam.memberNum,
390 : HCCL_ERROR(
391 : "[TeamCreate] sub team memberNum[%u] > world team memberNum[%u]", desc->memberNum,
392 : worldEntry->hostTeam.memberNum),
393 : HCOMM_E_PARA);
394 :
395 3 : uint32_t worldMemberNum = worldEntry->hostTeam.memberNum;
396 11 : for (uint32_t i = 0; i < desc->memberNum; i++) {
397 8 : CHK_PRT_RET(
398 : desc->worldMemberIds[i] >= worldMemberNum,
399 : HCCL_ERROR(
400 : "[TeamCreate] worldMemberIds[%u]=%u >= world team memberNum[%u]", i, desc->worldMemberIds[i],
401 : worldMemberNum),
402 : HCOMM_E_PARA);
403 : }
404 3 : return HCOMM_SUCCESS;
405 : }
406 :
407 54 : void HcommTeamMgr::InitTeamEntry(TeamEntry* entry, const HcommTeamCreateDesc* desc, HcommTeamHandle worldTeam)
408 : {
409 54 : entry->hostTeam.header.version = HCOMM_TEAM_VERSION;
410 54 : entry->hostTeam.header.magicWord = HCOMM_TEAM_MAGIC_WORD;
411 54 : entry->hostTeam.header.size = sizeof(HcommTeam);
412 54 : entry->hostTeam.header.reserved = 0;
413 54 : entry->hostTeam.memberNum = desc->memberNum;
414 54 : entry->hostTeam.selfMemberId = desc->selfMemberId;
415 54 : entry->hostTeam.netLayer = desc->netLayer;
416 54 : entry->hostTeam.engine = COMM_ENGINE_RESERVED;
417 54 : entry->syncMemReq = desc->requirement;
418 : entry->syncMemSize
419 54 : = static_cast<uint64_t>(
420 54 : desc->requirement.signalCount + desc->requirement.counterCount + desc->requirement.barrierCount)
421 54 : * sizeof(uint64_t) * desc->memberNum;
422 54 : entry->hostTeam.syncMem.syncMemReq = desc->requirement;
423 54 : entry->hostTeam.syncMem.syncMemSize = entry->syncMemSize;
424 :
425 54 : if (worldTeam != nullptr) {
426 3 : entry->worldTeamHandle = worldTeam;
427 3 : entry->isSubTeam = true;
428 : }
429 54 : }
430 :
431 54 : HcommResult HcommTeamMgr::AllocAndSyncTeam(TeamEntry* entry, const HcommTeamCreateDesc* desc)
432 : {
433 54 : HcommResult ret = AllocAndCopyWorldTeamIds(entry, desc->worldMemberIds, desc->memberNum);
434 54 : if (ret != HCOMM_SUCCESS) {
435 2 : HCCL_ERROR("[AllocAndSyncTeam] AllocAndCopyWorldTeamIds failed");
436 2 : FreeTeamResources(entry);
437 2 : return ret;
438 : }
439 :
440 52 : void* devTeamPtr = nullptr;
441 52 : ret = static_cast<HcommResult>(hrtMalloc(&devTeamPtr, static_cast<uint64_t>(sizeof(HcommTeam))));
442 52 : if (ret != HCOMM_SUCCESS) {
443 1 : HCCL_ERROR("[AllocAndSyncTeam] hrtMalloc devTeam failed, ret[%d]", ret);
444 1 : FreeTeamResources(entry);
445 1 : return ret;
446 : }
447 51 : entry->devTeam = static_cast<HcommTeamHandle>(devTeamPtr);
448 :
449 51 : ret = SyncTeamToDevice(entry);
450 51 : if (ret != HCOMM_SUCCESS) {
451 1 : HCCL_ERROR("[AllocAndSyncTeam] SyncTeamToDevice failed, ret[%d]", ret);
452 1 : FreeTeamResources(entry);
453 1 : return ret;
454 : }
455 50 : return HCOMM_SUCCESS;
456 : }
457 :
458 55 : HcommResult HcommTeamMgr::TeamCreate(
459 : HcommTeamHandle worldTeam, const HcommTeamCreateDesc* desc, HcommTeamHandle* team, uint64_t* outSyncMemSize)
460 : {
461 55 : TeamEntry* worldEntry = nullptr;
462 55 : if (worldTeam != nullptr) {
463 4 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
464 4 : worldEntry = FindTeamByHandleLocked(worldTeam);
465 4 : CHK_PRT_RET(
466 : worldEntry == nullptr, HCCL_ERROR("[TeamCreate] worldTeam handle[%p] not found", worldTeam),
467 : HCOMM_E_NOT_FOUND);
468 4 : }
469 :
470 54 : HcommResult ret = ValidateSubTeam(worldEntry, desc);
471 54 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[TeamCreate] ValidateSubTeam failed"), ret);
472 :
473 54 : auto entry = std::make_unique<TeamEntry>();
474 54 : InitTeamEntry(entry.get(), desc, worldTeam);
475 :
476 54 : ret = AllocAndSyncTeam(entry.get(), desc);
477 54 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[TeamCreate] AllocAndSyncTeam failed"), ret);
478 :
479 50 : *outSyncMemSize = entry->syncMemSize;
480 50 : *team = entry->devTeam;
481 :
482 : {
483 50 : std::unique_lock<std::shared_mutex> lock(teamsRwMutex_);
484 50 : teams_[*team] = std::move(entry);
485 50 : }
486 :
487 50 : HCCL_INFO(
488 : "[TeamCreate] team created, memberNum[%u], selfMemberId[%u], "
489 : "netLayer[%u], protocol[%d], "
490 : "signalCount[%u], counterCount[%u], barrierCount[%u], "
491 : "syncMemSize[%llu], isSubTeam[%d], devTeam[%p]",
492 : desc->memberNum, desc->selfMemberId, desc->netLayer, static_cast<int32_t>(desc->protocol),
493 : desc->requirement.signalCount, desc->requirement.counterCount, desc->requirement.barrierCount, *outSyncMemSize,
494 : static_cast<int32_t>(worldTeam != nullptr), *team);
495 :
496 50 : return HCOMM_SUCCESS;
497 54 : }
498 :
499 14 : HcommResult HcommTeamMgr::TeamDestroy(HcommTeamHandle team)
500 : {
501 14 : std::unique_lock<std::shared_mutex> lock(teamsRwMutex_);
502 14 : auto it = teams_.find(team);
503 14 : CHK_PRT_RET(
504 : it == teams_.end(), HCCL_WARNING("[TeamDestroy] team handle[%p] not found, maybe already destroyed", team),
505 : HCOMM_SUCCESS);
506 :
507 13 : bool isSubTeam = it->second->isSubTeam;
508 13 : FreeTeamResources(it->second.get());
509 13 : teams_.erase(it);
510 :
511 13 : HCCL_INFO("[TeamDestroy] team[%p] destroyed, isSubTeam[%d]", team, static_cast<int32_t>(isSubTeam));
512 13 : return HCOMM_SUCCESS;
513 14 : }
514 :
515 20 : HcommResult HcommTeamMgr::WindowRegister(HcommTeamHandle team, HcommWindowHandle* handle)
516 : {
517 : // 校验 team 存在且为 worldTeam：shared_lock 仅覆盖查找/校验，不可延伸到 unique_lock 区间，否则自死锁。
518 : {
519 20 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
520 20 : TeamEntry* entry = FindTeamByHandleLocked(team);
521 20 : CHK_PRT_RET(
522 : entry == nullptr, HCCL_ERROR("[WindowRegister] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
523 20 : CHK_PRT_RET(
524 : entry->isSubTeam, HCCL_ERROR("[WindowRegister] subteam cannot register window, use worldTeam"),
525 : HCOMM_E_PARA);
526 20 : }
527 :
528 20 : auto winEntry = std::make_unique<WindowEntry>();
529 20 : winEntry->teamHandle = team;
530 20 : winEntry->hostWindow.header.version = HCOMM_WINDOW_VERSION;
531 20 : winEntry->hostWindow.header.magicWord = HCOMM_WINDOW_MAGIC_WORD;
532 20 : winEntry->hostWindow.header.size = sizeof(HcommWindow);
533 20 : winEntry->hostWindow.header.reserved = 0;
534 20 : winEntry->hostWindow.worldTeam = team;
535 :
536 20 : void* devWindowPtr = nullptr;
537 20 : HcommResult ret = static_cast<HcommResult>(hrtMalloc(&devWindowPtr, static_cast<uint64_t>(sizeof(HcommWindow))));
538 20 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[WindowRegister] hrtMalloc devWindow failed, ret[%d]", ret), ret);
539 19 : winEntry->devWindow = static_cast<HcommWindowHandle>(devWindowPtr);
540 :
541 19 : ret = SyncWindowToDevice(winEntry.get());
542 19 : if (ret != HCOMM_SUCCESS) {
543 1 : HCCL_ERROR("[WindowRegister] SyncWindowToDevice failed, ret[%d]", ret);
544 1 : FreeWindowResources(winEntry.get());
545 1 : return ret;
546 : }
547 :
548 18 : *handle = winEntry->devWindow;
549 :
550 : {
551 : // 同时持有 windows_ 与 windowToTeamMap_ 写锁，保证 window 登记原子性（锁顺序：windows→windowToTeam）
552 18 : std::unique_lock<std::shared_mutex> winLock(windowsRwMutex_);
553 18 : std::unique_lock<std::shared_mutex> mapLock(windowToTeamRwMutex_);
554 18 : windowToTeamMap_[*handle] = team;
555 18 : windows_[*handle] = std::move(winEntry);
556 18 : }
557 :
558 18 : HCCL_INFO("[WindowRegister] window created, team[%p], devWindow[%p]", team, devWindowPtr);
559 18 : return HCOMM_SUCCESS;
560 20 : }
561 :
562 9 : HcommResult HcommTeamMgr::AllocAndCopyWindowMems(WindowEntry* winEntry, uint64_t memberNum, const CommMem* src)
563 : {
564 9 : winEntry->hostMems = static_cast<CommMem*>(calloc(memberNum, sizeof(CommMem)));
565 9 : if (winEntry->hostMems == nullptr) {
566 0 : HCCL_ERROR("[AllocAndCopyWindowMems] calloc hostMems failed");
567 0 : return HCOMM_E_PTR;
568 : }
569 9 : errno_t memRet = memcpy_s(winEntry->hostMems, memberNum * sizeof(CommMem), src, memberNum * sizeof(CommMem));
570 9 : if (memRet != EOK) {
571 0 : HCCL_ERROR("[AllocAndCopyWindowMems] memcpy_s hostMems failed, ret[%d]", memRet);
572 0 : free(winEntry->hostMems);
573 0 : winEntry->hostMems = nullptr;
574 0 : return HCOMM_E_MEMORY;
575 : }
576 :
577 9 : void* devMemsPtr = nullptr;
578 : HcommResult ret
579 9 : = static_cast<HcommResult>(hrtMalloc(&devMemsPtr, static_cast<uint64_t>(memberNum * sizeof(CommMem))));
580 9 : if (ret != HCOMM_SUCCESS) {
581 1 : HCCL_ERROR("[AllocAndCopyWindowMems] hrtMalloc devMems failed, ret[%d]", ret);
582 1 : free(winEntry->hostMems);
583 1 : winEntry->hostMems = nullptr;
584 1 : return ret;
585 : }
586 :
587 8 : ret = static_cast<HcommResult>(hrtMemSyncCopy(
588 : devMemsPtr, static_cast<uint64_t>(memberNum * sizeof(CommMem)), src,
589 : static_cast<uint64_t>(memberNum * sizeof(CommMem)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
590 8 : if (ret != HCOMM_SUCCESS) {
591 0 : HCCL_ERROR("[AllocAndCopyWindowMems] hrtMemSyncCopy devMems failed, ret[%d]", ret);
592 0 : (void)hrtFree(devMemsPtr);
593 0 : free(winEntry->hostMems);
594 0 : winEntry->hostMems = nullptr;
595 0 : return ret;
596 : }
597 8 : winEntry->devMems = devMemsPtr;
598 8 : winEntry->hostWindow.mems = static_cast<CommMem*>(devMemsPtr);
599 8 : winEntry->hostWindow.memsNum = memberNum;
600 8 : return HCOMM_SUCCESS;
601 : }
602 :
603 3 : HcommResult HcommTeamMgr::MergeWindowMems(WindowEntry* winEntry, uint64_t memberNum, const CommMem* src)
604 : {
605 3 : CHK_PRT_RET(
606 : winEntry->hostMems == nullptr || winEntry->devMems == nullptr,
607 : HCCL_ERROR("[MergeWindowMems] hostMems or devMems is null, not allocated yet"), HCOMM_E_PTR);
608 3 : CHK_PRT_RET(
609 : memberNum != winEntry->hostWindow.memsNum,
610 : HCCL_ERROR(
611 : "[MergeWindowMems] memberNum[%llu] != memsNum[%llu], dimension mismatch", memberNum,
612 : winEntry->hostWindow.memsNum),
613 : HCOMM_E_PARA);
614 :
615 11 : for (uint64_t i = 0; i < memberNum; i++) {
616 9 : if (src[i].addr != nullptr) {
617 4 : CHK_PRT_RET(
618 : winEntry->hostMems[i].addr != nullptr,
619 : HCCL_ERROR("[MergeWindowMems] member[%llu] already bound, register a new window to rebind", i),
620 : HCOMM_E_PARA);
621 3 : winEntry->hostMems[i] = src[i];
622 : }
623 : }
624 :
625 : // 重新把 hostMems 整块 sync 到 devMems
626 4 : HcclResult hrtRet = hrtMemSyncCopy(
627 2 : winEntry->devMems, static_cast<uint64_t>(memberNum * sizeof(CommMem)), winEntry->hostMems,
628 : static_cast<uint64_t>(memberNum * sizeof(CommMem)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE);
629 2 : HcommResult ret = static_cast<HcommResult>(hrtRet);
630 2 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[MergeWindowMems] hrtMemSyncCopy devMems failed, ret[%d]", ret), ret);
631 2 : return HCOMM_SUCCESS;
632 : }
633 :
634 13 : HcommResult HcommTeamMgr::BindWindow(HcommTeamHandle team, HcommWindowHandle handle, const HcommTeamWindowDesc* desc)
635 : {
636 : // 同时读 windows_ 与 teams_ 校验存在性（锁顺序：teams→windows）
637 13 : std::shared_lock<std::shared_mutex> teamLock(teamsRwMutex_);
638 13 : std::shared_lock<std::shared_mutex> winLock(windowsRwMutex_);
639 13 : WindowEntry* winEntry = FindWindowByHandleLocked(handle);
640 13 : CHK_PRT_RET(winEntry == nullptr, HCCL_ERROR("[BindWindow] window handle[%p] not found", handle), HCOMM_E_NOT_FOUND);
641 12 : TeamEntry* teamEntry = FindTeamByHandleLocked(team);
642 12 : CHK_PRT_RET(teamEntry == nullptr, HCCL_ERROR("[BindWindow] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
643 :
644 : (void)teamEntry;
645 : HcommResult ret;
646 12 : if (winEntry->hostMems == nullptr) {
647 9 : ret = AllocAndCopyWindowMems(winEntry, desc->memberNum, desc->mems);
648 9 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[BindWindow] AllocAndCopyWindowMems failed"), ret);
649 : } else {
650 3 : ret = MergeWindowMems(winEntry, desc->memberNum, desc->mems);
651 3 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[BindWindow] MergeWindowMems failed"), ret);
652 : }
653 :
654 10 : ret = SyncWindowToDevice(winEntry);
655 10 : if (ret != HCOMM_SUCCESS) {
656 1 : HCCL_ERROR("[BindWindow] SyncWindowToDevice failed, ret[%d]", ret);
657 1 : return ret;
658 : }
659 :
660 9 : HCCL_INFO(
661 : "[BindWindow] window bound, team[%p], handle[%p], devWindow[%p], mems[%p], memberNum[%u], devMems[%p]", team,
662 : handle, winEntry->devWindow, desc->mems, desc->memberNum, winEntry->devMems);
663 9 : return HCOMM_SUCCESS;
664 13 : }
665 :
666 6 : HcommResult HcommTeamMgr::WindowDeregister(HcommTeamHandle team, HcommWindowHandle handle)
667 : {
668 : {
669 6 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
670 6 : TeamEntry* entry = FindTeamByHandleLocked(team);
671 6 : CHK_PRT_RET(
672 : entry == nullptr, HCCL_ERROR("[WindowDeregister] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
673 4 : CHK_PRT_RET(
674 : entry->isSubTeam, HCCL_ERROR("[WindowDeregister] subteam cannot deregister window, use worldTeam"),
675 : HCOMM_E_PARA);
676 6 : }
677 : // 同时持有 windows_ 与 windowToTeamMap_ 写锁，保证销毁原子性（锁顺序：windows→windowToTeam）
678 4 : std::unique_lock<std::shared_mutex> winLock(windowsRwMutex_);
679 4 : std::unique_lock<std::shared_mutex> mapLock(windowToTeamRwMutex_);
680 4 : auto it = windows_.find(handle);
681 4 : CHK_PRT_RET(
682 : it == windows_.end(),
683 : HCCL_WARNING("[WindowDeregister] window handle[%p] not found, maybe already destroyed", handle), HCOMM_SUCCESS);
684 :
685 4 : FreeWindowResources(it->second.get());
686 4 : windowToTeamMap_.erase(handle);
687 4 : windows_.erase(it);
688 :
689 4 : HCCL_INFO("[WindowDeregister] window[%p] destroyed, team[%p]", handle, team);
690 4 : return HCOMM_SUCCESS;
691 4 : }
692 :
693 6 : void HcommTeamMgr::MergeChannelLists(
694 : TeamEntry* entry, const HcommTeamBindChannelsDesc* desc, std::vector<std::vector<uint64_t>>& newChannels)
695 : {
696 6 : if (entry->channelsList.empty()) {
697 4 : newChannels.resize(entry->hostTeam.memberNum);
698 : } else {
699 2 : newChannels = entry->channelsList;
700 : }
701 :
702 30 : for (uint32_t i = 0; i < entry->hostTeam.memberNum; i++) {
703 24 : uint64_t chNum = desc->channelNumPerMember[i];
704 24 : uint64_t* chSrc = desc->channelsByMemberId[i];
705 52 : for (uint64_t j = 0; j < chNum; j++) {
706 28 : newChannels[i].push_back(chSrc[j]);
707 : }
708 : }
709 6 : }
710 :
711 8 : HcommResult HcommTeamMgr::BindChannels(HcommTeamHandle team, const HcommTeamBindChannelsDesc* desc)
712 : {
713 8 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
714 8 : TeamEntry* entry = FindTeamByHandleLocked(team);
715 8 : CHK_PRT_RET(entry == nullptr, HCCL_ERROR("[BindChannels] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
716 :
717 7 : CHK_PRT_RET(
718 : desc->memberNum != entry->hostTeam.memberNum,
719 : HCCL_ERROR("[BindChannels] memberNum[%u] != team memberNum[%u]", desc->memberNum, entry->hostTeam.memberNum),
720 : HCOMM_E_PARA);
721 :
722 30 : for (uint32_t i = 0; i < entry->hostTeam.memberNum; i++) {
723 24 : CHK_PRT_RET(
724 : desc->channelNumPerMember[i] != 0 && desc->channelsByMemberId[i] == nullptr,
725 : HCCL_ERROR(
726 : "[BindChannels] member[%u] channelNum[%u] > 0 but channels is null", i, desc->channelNumPerMember[i]),
727 : HCOMM_E_PARA);
728 : }
729 :
730 : // 二维数组，需要填入HcommTeam中的channels字段
731 6 : std::vector<std::vector<uint64_t>> newChannels;
732 6 : MergeChannelLists(entry, desc, newChannels);
733 :
734 6 : std::vector<std::vector<uint64_t>> oldChannels = std::move(entry->channelsList);
735 6 : entry->channelsList = std::move(newChannels);
736 :
737 6 : HcommResult ret = AllocAndCopyChannels(entry);
738 6 : if (ret != HCOMM_SUCCESS) {
739 1 : HCCL_ERROR("[BindChannels] AllocAndCopyChannels failed, this bind does not take effect, "
740 : "previously bound channels are not affected");
741 1 : entry->channelsList = std::move(oldChannels);
742 1 : return ret;
743 : }
744 :
745 5 : ret = SyncTeamToDevice(entry);
746 5 : if (ret != HCOMM_SUCCESS) {
747 0 : HCCL_ERROR("[BindChannels] SyncTeamToDevice failed");
748 0 : return ret;
749 : }
750 :
751 5 : HCCL_INFO(
752 : "[BindChannels] channels appended and synced, team[%p], channelNumPerMember[%p], "
753 : "channelsByMemberId[%p], memberNum[%u], isSubTeam[%d]",
754 : team, desc->channelNumPerMember, desc->channelsByMemberId, entry->hostTeam.memberNum, entry->isSubTeam);
755 5 : return HCOMM_SUCCESS;
756 8 : }
757 :
758 12 : HcommResult HcommTeamMgr::BindSyncMem(HcommTeamHandle team, const HcommTeamBindSyncMemDesc* desc)
759 : {
760 12 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
761 12 : TeamEntry* entry = FindTeamByHandleLocked(team);
762 12 : CHK_PRT_RET(entry == nullptr, HCCL_ERROR("[BindSyncMem] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
763 :
764 11 : CHK_PRT_RET(
765 : desc->remoteMemNum != entry->hostTeam.memberNum,
766 : HCCL_ERROR("[BindSyncMem] remoteMemNum[%u] != memberNum[%u]", desc->remoteMemNum, entry->hostTeam.memberNum),
767 : HCOMM_E_PARA);
768 :
769 10 : HcommResult ret = AllocAndCopyRemoteMems(entry, desc->remoteMems, desc->remoteMemNum);
770 10 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[BindSyncMem] AllocAndCopyRemoteMems failed"), ret);
771 :
772 9 : if (entry->hostTeam.syncMem.shadowMem.addr != nullptr) {
773 2 : (void)hrtFree(entry->hostTeam.syncMem.shadowMem.addr);
774 2 : entry->hostTeam.syncMem.shadowMem.addr = nullptr;
775 2 : entry->hostTeam.syncMem.shadowMem.size = 0;
776 : }
777 9 : void* devShadowMemPtr = nullptr;
778 9 : uint64_t shadowMemSize = entry->hostTeam.syncMem.syncMemSize;
779 9 : ret = static_cast<HcommResult>(hrtMalloc(&devShadowMemPtr, shadowMemSize));
780 9 : if (ret != HCOMM_SUCCESS) {
781 0 : HCCL_ERROR("[BindSyncMem] hrtMalloc shadowMem failed, size[%llu] ret[%d]", shadowMemSize, ret);
782 0 : return ret;
783 : }
784 9 : entry->hostTeam.syncMem.shadowMem.type = COMM_MEM_TYPE_DEVICE;
785 9 : entry->hostTeam.syncMem.shadowMem.addr = devShadowMemPtr;
786 9 : entry->hostTeam.syncMem.shadowMem.size = shadowMemSize;
787 :
788 9 : ret = SyncTeamToDevice(entry);
789 9 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[BindSyncMem] SyncTeamToDevice failed"), ret);
790 :
791 8 : HCCL_INFO("[BindSyncMem] syncMem bound, remoteMems[%p], remoteMemNum[%u]", desc->remoteMems, desc->remoteMemNum);
792 8 : return HCOMM_SUCCESS;
793 12 : }
794 :
795 2 : HcommResult HcommTeamMgr::GetNetLayer(HcommTeamHandle team, uint32_t* netLayer)
796 : {
797 2 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
798 2 : TeamEntry* entry = FindTeamByHandleLocked(team);
799 2 : CHK_PRT_RET(entry == nullptr, HCCL_ERROR("[GetNetLayer] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
800 :
801 1 : *netLayer = entry->hostTeam.netLayer;
802 1 : return HCOMM_SUCCESS;
803 2 : }
804 :
805 : } // namespace hcomm
|
