Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hcomm_team_mgr.h"
12 :
13 : #include <cstdlib>
14 :
15 : #include "adapter_rts_common.h"
16 : #include "hcomm_res_entity_defs.h"
17 : #include "hcomm_team_entity_defs.h"
18 : #include "log.h"
19 : #include "securec.h"
20 :
21 : namespace hcomm {
22 :
23 227 : HcommTeamMgr& HcommTeamMgr::GetInstance()
24 : {
25 : static std::once_flag instanceFlag;
26 : static HcommTeamMgr* instance = nullptr;
27 227 : std::call_once(instanceFlag, [&] {
28 1 : instance = new HcommTeamMgr();
29 1 : });
30 227 : return *instance;
31 : }
32 :
33 0 : HcommTeamMgr::~HcommTeamMgr()
34 : {
35 : {
36 0 : std::unique_lock<std::shared_mutex> lock(teamsRwMutex_);
37 0 : for (auto& pair : teams_) {
38 0 : if (pair.second != nullptr) {
39 0 : FreeTeamResources(pair.second.get());
40 : }
41 : }
42 0 : teams_.clear();
43 0 : }
44 : {
45 0 : std::unique_lock<std::shared_mutex> lock(windowsRwMutex_);
46 0 : for (auto& pair : windows_) {
47 0 : if (pair.second != nullptr) {
48 0 : FreeWindowResources(pair.second.get());
49 : }
50 : }
51 0 : windows_.clear();
52 0 : }
53 : {
54 0 : std::unique_lock<std::shared_mutex> lock(windowToTeamRwMutex_);
55 0 : windowToTeamMap_.clear();
56 0 : }
57 0 : }
58 :
59 73 : TeamEntry* HcommTeamMgr::FindTeamByHandleLocked(HcommTeamHandle handle)
60 : {
61 73 : auto it = teams_.find(handle);
62 73 : if (it == teams_.end()) {
63 6 : HCCL_ERROR("[FindTeamByHandleLocked] team handle[%p] not found", handle);
64 6 : return nullptr;
65 : }
66 67 : return it->second.get();
67 : }
68 :
69 19 : WindowEntry* HcommTeamMgr::FindWindowByHandleLocked(HcommWindowHandle handle)
70 : {
71 19 : auto it = windows_.find(handle);
72 19 : if (it == windows_.end()) {
73 1 : HCCL_ERROR("[FindWindowByHandleLocked] window handle[%p] not found", handle);
74 1 : return nullptr;
75 : }
76 18 : return it->second.get();
77 : }
78 :
79 65 : HcommResult HcommTeamMgr::SyncTeamToDevice(TeamEntry* entry)
80 : {
81 65 : if (entry->devTeam == nullptr) {
82 0 : HCCL_ERROR("[SyncTeamToDevice] devTeam is null");
83 0 : return HCOMM_E_PTR;
84 : }
85 130 : return static_cast<HcommResult>(hrtMemSyncCopy(
86 65 : entry->devTeam, sizeof(HcommTeam), &entry->hostTeam, sizeof(HcommTeam),
87 65 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
88 : }
89 :
90 29 : HcommResult HcommTeamMgr::SyncWindowToDevice(WindowEntry* entry)
91 : {
92 29 : if (entry->devWindow == nullptr) {
93 0 : HCCL_ERROR("[SyncWindowToDevice] devWindow is null");
94 0 : return HCOMM_E_PTR;
95 : }
96 58 : return static_cast<HcommResult>(hrtMemSyncCopy(
97 29 : entry->devWindow, sizeof(HcommWindow), &entry->hostWindow, sizeof(HcommWindow),
98 29 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
99 : }
100 :
101 54 : HcommResult HcommTeamMgr::AllocAndCopyWorldTeamIds(TeamEntry* entry, const uint32_t* src, uint32_t memberNum)
102 : {
103 : // worldTeam（src==nullptr）：worldTeamIds 置为 [0,memberNum) 连续序列（memberId 自身）
104 54 : std::vector<uint32_t> sequentialIds;
105 54 : const uint32_t* srcPtr = src;
106 54 : if (src == nullptr) {
107 0 : sequentialIds.resize(memberNum);
108 0 : for (uint32_t i = 0; i < memberNum; i++) {
109 0 : sequentialIds[i] = i;
110 : }
111 0 : srcPtr = sequentialIds.data();
112 : }
113 :
114 54 : void* devPtr = nullptr;
115 54 : HcommResult ret = static_cast<HcommResult>(hrtMalloc(&devPtr, static_cast<uint64_t>(memberNum * sizeof(uint32_t))));
116 54 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[AllocAndCopyWorldTeamIds] hrtMalloc failed, ret[%d]", ret), ret);
117 :
118 106 : ret = static_cast<HcommResult>(hrtMemSyncCopy(
119 53 : devPtr, static_cast<uint64_t>(memberNum * sizeof(uint32_t)), srcPtr,
120 53 : static_cast<uint64_t>(memberNum * sizeof(uint32_t)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
121 53 : if (ret != HCOMM_SUCCESS) {
122 1 : HCCL_ERROR("[AllocAndCopyWorldTeamIds] hrtMemSyncCopy failed, ret[%d]", ret);
123 1 : (void)hrtFree(devPtr);
124 1 : return ret;
125 : }
126 :
127 52 : entry->devWorldTeamIds = devPtr;
128 52 : entry->hostTeam.worldTeamIds = static_cast<uint32_t*>(devPtr);
129 :
130 52 : entry->hostWorldTeamIds = static_cast<uint32_t*>(malloc(memberNum * sizeof(uint32_t)));
131 52 : if (entry->hostWorldTeamIds == nullptr) {
132 0 : HCCL_ERROR("[AllocAndCopyWorldTeamIds] malloc hostWorldTeamIds failed");
133 0 : (void)hrtFree(devPtr);
134 0 : entry->devWorldTeamIds = nullptr;
135 0 : entry->hostTeam.worldTeamIds = nullptr;
136 0 : return HCOMM_E_MEMORY;
137 : }
138 : errno_t memRet
139 52 : = memcpy_s(entry->hostWorldTeamIds, memberNum * sizeof(uint32_t), srcPtr, memberNum * sizeof(uint32_t));
140 52 : if (memRet != EOK) {
141 0 : HCCL_ERROR("[AllocAndCopyWorldTeamIds] memcpy_s hostWorldTeamIds failed, ret[%d]", memRet);
142 0 : free(entry->hostWorldTeamIds);
143 0 : entry->hostWorldTeamIds = nullptr;
144 0 : (void)hrtFree(devPtr);
145 0 : entry->devWorldTeamIds = nullptr;
146 0 : entry->hostTeam.worldTeamIds = nullptr;
147 0 : return HCOMM_E_MEMORY;
148 : }
149 :
150 52 : return HCOMM_SUCCESS;
151 54 : }
152 :
153 6 : HcommResult HcommTeamMgr::AllocChannelEntities(TeamEntry* entry)
154 : {
155 6 : uint32_t memberNum = entry->hostTeam.memberNum;
156 6 : entry->hostChannelNums.resize(memberNum, 0);
157 6 : uint32_t totalChannels = 0;
158 30 : for (uint32_t i = 0; i < memberNum; i++) {
159 24 : uint32_t chNum = static_cast<uint32_t>(entry->channelsList[i].size());
160 24 : entry->hostChannelNums[i] = chNum;
161 24 : totalChannels += chNum;
162 : }
163 :
164 6 : if (totalChannels == 0) {
165 0 : entry->devChannels = nullptr;
166 0 : entry->hostTeam.channelsBaseAddr = 0;
167 0 : return HCOMM_SUCCESS;
168 : }
169 :
170 6 : void* devPtr = nullptr;
171 : HcommResult ret
172 6 : = static_cast<HcommResult>(hrtMalloc(&devPtr, static_cast<uint64_t>(totalChannels * sizeof(ChannelEntity))));
173 6 : CHK_PRT_RET(
174 : ret != HCOMM_SUCCESS,
175 : HCCL_ERROR("[AllocChannelEntities] hrtMalloc failed, totalChannels[%u] ret[%d]", totalChannels, ret), ret);
176 :
177 : // 逐个 channel 做 D2D 拷贝：把 ChannelHandle 指向的 ChannelEntity 本体拷贝到连续内存对应偏移
178 : // 偏移按前缀和 sum(channelNumPerMember[0..peer-1]) + channelIdx 计算
179 5 : uint32_t offset = 0;
180 25 : for (uint32_t peer = 0; peer < memberNum; peer++) {
181 48 : for (uint32_t idx = 0; idx < entry->channelsList[peer].size(); idx++) {
182 28 : void* dst = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(devPtr) + offset * sizeof(ChannelEntity));
183 28 : void* src = reinterpret_cast<void*>(static_cast<uintptr_t>(entry->channelsList[peer][idx]));
184 28 : ret = static_cast<HcommResult>(hrtMemSyncCopy(
185 : dst, sizeof(ChannelEntity), src, sizeof(ChannelEntity),
186 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_DEVICE));
187 28 : if (ret != HCOMM_SUCCESS) {
188 0 : HCCL_ERROR(
189 : "[AllocChannelEntities] hrtMemSyncCopy D2D failed, peer[%u] idx[%u] ret[%d]", peer, idx, ret);
190 0 : (void)hrtFree(devPtr);
191 0 : return ret;
192 : }
193 28 : offset++;
194 : }
195 : }
196 :
197 5 : entry->devChannels = devPtr;
198 5 : entry->hostTeam.channelsBaseAddr = reinterpret_cast<uint64_t>(devPtr);
199 5 : return HCOMM_SUCCESS;
200 : }
201 :
202 5 : HcommResult HcommTeamMgr::AllocChannelNumsArray(TeamEntry* entry)
203 : {
204 5 : uint32_t memberNum = entry->hostTeam.memberNum;
205 5 : void* devPtr = nullptr;
206 5 : HcommResult ret = static_cast<HcommResult>(hrtMalloc(&devPtr, static_cast<uint64_t>(memberNum * sizeof(uint32_t))));
207 5 : if (ret != HCOMM_SUCCESS) {
208 0 : HCCL_ERROR("[AllocChannelNumsArray] hrtMalloc channelNums failed, ret[%d]", ret);
209 0 : return ret;
210 : }
211 :
212 10 : ret = static_cast<HcommResult>(hrtMemSyncCopy(
213 5 : devPtr, static_cast<uint64_t>(memberNum * sizeof(uint32_t)), entry->hostChannelNums.data(),
214 5 : static_cast<uint64_t>(memberNum * sizeof(uint32_t)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
215 5 : if (ret != HCOMM_SUCCESS) {
216 0 : HCCL_ERROR("[AllocChannelNumsArray] hrtMemSyncCopy channelNums failed, ret[%d]", ret);
217 0 : (void)hrtFree(devPtr);
218 0 : return ret;
219 : }
220 5 : entry->devChannelNums = devPtr;
221 5 : entry->hostTeam.channelNumPerMember = static_cast<uint32_t*>(devPtr);
222 5 : return HCOMM_SUCCESS;
223 : }
224 :
225 6 : HcommResult HcommTeamMgr::AllocAndCopyChannels(TeamEntry* entry)
226 : {
227 6 : uint32_t memberNum = entry->hostTeam.memberNum;
228 6 : if (memberNum == 0 || entry->channelsList.empty()) {
229 0 : HCCL_ERROR("[AllocAndCopyChannels] memberNum[%u] or channelsList is empty", memberNum);
230 0 : return HCOMM_E_PARA;
231 : }
232 :
233 6 : FreeDeviceChannels(entry);
234 :
235 6 : HcommResult ret = AllocChannelEntities(entry);
236 6 : if (ret != HCOMM_SUCCESS) {
237 1 : HCCL_ERROR("[AllocAndCopyChannels] AllocChannelEntities failed, ret[%d]", ret);
238 1 : return ret;
239 : }
240 :
241 5 : ret = AllocChannelNumsArray(entry);
242 5 : if (ret != HCOMM_SUCCESS) {
243 0 : HCCL_ERROR("[AllocAndCopyChannels] AllocChannelNumsArray failed, ret[%d]", ret);
244 0 : FreeDeviceChannels(entry);
245 0 : return ret;
246 : }
247 :
248 5 : return HCOMM_SUCCESS;
249 : }
250 :
251 10 : HcommResult HcommTeamMgr::AllocAndCopyRemoteMems(TeamEntry* entry, const CommMem* src, uint32_t memberNum)
252 : {
253 10 : if (entry->hostRemoteMems == nullptr) {
254 8 : return AllocRemoteMems(entry, src, memberNum);
255 : }
256 2 : return UpdateRemoteMems(entry, src, memberNum);
257 : }
258 :
259 8 : HcommResult HcommTeamMgr::AllocRemoteMems(TeamEntry* entry, const CommMem* src, uint32_t memberNum)
260 : {
261 : // 首次分配：calloc hostRemoteMems + memcpy 拷入；hrtMalloc devRemoteMems + hrtMemSyncCopy 到 device
262 8 : entry->hostRemoteMems = static_cast<CommMem*>(calloc(memberNum, sizeof(CommMem)));
263 8 : if (entry->hostRemoteMems == nullptr) {
264 0 : HCCL_ERROR("[AllocRemoteMems] calloc hostRemoteMems failed");
265 0 : return HCOMM_E_MEMORY;
266 : }
267 8 : errno_t memRet = memcpy_s(entry->hostRemoteMems, memberNum * sizeof(CommMem), src, memberNum * sizeof(CommMem));
268 8 : if (memRet != EOK) {
269 0 : HCCL_ERROR("[AllocRemoteMems] memcpy_s failed, ret[%d]", memRet);
270 0 : free(entry->hostRemoteMems);
271 0 : entry->hostRemoteMems = nullptr;
272 0 : return HCOMM_E_MEMORY;
273 : }
274 :
275 8 : void* devPtr = nullptr;
276 8 : HcommResult ret = static_cast<HcommResult>(hrtMalloc(&devPtr, static_cast<uint64_t>(memberNum * sizeof(CommMem))));
277 8 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[AllocRemoteMems] hrtMalloc failed, ret[%d]", ret), ret);
278 :
279 14 : ret = static_cast<HcommResult>(hrtMemSyncCopy(
280 7 : devPtr, static_cast<uint64_t>(memberNum * sizeof(CommMem)), src,
281 7 : static_cast<uint64_t>(memberNum * sizeof(CommMem)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
282 7 : if (ret != HCOMM_SUCCESS) {
283 0 : HCCL_ERROR("[AllocRemoteMems] hrtMemSyncCopy failed, ret[%d]", ret);
284 0 : (void)hrtFree(devPtr);
285 0 : return ret;
286 : }
287 :
288 7 : entry->devRemoteMems = devPtr;
289 7 : entry->hostTeam.syncMem.remoteMems = static_cast<CommMem*>(devPtr);
290 7 : entry->hostTeam.syncMem.remoteMemsNum = memberNum;
291 7 : return HCOMM_SUCCESS;
292 : }
293 :
294 2 : HcommResult HcommTeamMgr::UpdateRemoteMems(TeamEntry* entry, const CommMem* src, uint32_t memberNum)
295 : {
296 : // 重复 bind：校验维度一致后更新 hostRemoteMems 并重新 sync 到 devRemoteMems
297 2 : CHK_PRT_RET(
298 : memberNum != entry->hostTeam.syncMem.remoteMemsNum,
299 : HCCL_ERROR(
300 : "[UpdateRemoteMems] memberNum[%u] != remoteMemNum[%u], dimension mismatch", memberNum,
301 : entry->hostTeam.syncMem.remoteMemsNum),
302 : HCOMM_E_PARA);
303 :
304 2 : errno_t memRet = memcpy_s(entry->hostRemoteMems, memberNum * sizeof(CommMem), src, memberNum * sizeof(CommMem));
305 2 : CHK_PRT_RET(memRet != EOK, HCCL_ERROR("[UpdateRemoteMems] memcpy_s failed, ret[%d]", memRet), HCOMM_E_MEMORY);
306 :
307 4 : HcclResult hrtRet = hrtMemSyncCopy(
308 2 : entry->devRemoteMems, static_cast<uint64_t>(memberNum * sizeof(CommMem)), entry->hostRemoteMems,
309 2 : static_cast<uint64_t>(memberNum * sizeof(CommMem)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE);
310 2 : HcommResult ret = static_cast<HcommResult>(hrtRet);
311 2 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[UpdateRemoteMems] hrtMemSyncCopy failed, ret[%d]", ret), ret);
312 2 : return HCOMM_SUCCESS;
313 : }
314 :
315 60 : void HcommTeamMgr::FreeDeviceChannels(TeamEntry* entry)
316 : {
317 60 : if (entry->devChannels != nullptr) {
318 5 : (void)hrtFree(entry->devChannels);
319 5 : entry->devChannels = nullptr;
320 : }
321 60 : if (entry->devChannelNums != nullptr) {
322 5 : (void)hrtFree(entry->devChannelNums);
323 5 : entry->devChannelNums = nullptr;
324 : }
325 60 : entry->hostTeam.channelsBaseAddr = 0;
326 60 : entry->hostTeam.channelNumPerMember = nullptr;
327 60 : }
328 :
329 54 : void HcommTeamMgr::FreeTeamResources(TeamEntry* entry)
330 : {
331 54 : if (entry->devRemoteMems != nullptr) {
332 7 : (void)hrtFree(entry->devRemoteMems);
333 7 : entry->devRemoteMems = nullptr;
334 : }
335 54 : if (entry->hostRemoteMems != nullptr) {
336 8 : free(entry->hostRemoteMems);
337 8 : entry->hostRemoteMems = nullptr;
338 : }
339 :
340 54 : FreeDeviceChannels(entry);
341 :
342 54 : if (entry->devWorldTeamIds != nullptr) {
343 52 : (void)hrtFree(entry->devWorldTeamIds);
344 52 : entry->devWorldTeamIds = nullptr;
345 : }
346 54 : if (entry->hostWorldTeamIds != nullptr) {
347 52 : free(entry->hostWorldTeamIds);
348 52 : entry->hostWorldTeamIds = nullptr;
349 : }
350 :
351 54 : if (entry->hostTeam.syncMem.shadowMem.addr != nullptr) {
352 7 : (void)hrtFree(entry->hostTeam.syncMem.shadowMem.addr);
353 7 : entry->hostTeam.syncMem.shadowMem.addr = nullptr;
354 7 : entry->hostTeam.syncMem.shadowMem.size = 0;
355 : }
356 :
357 54 : if (entry->devTeam != nullptr) {
358 51 : (void)hrtFree(entry->devTeam);
359 51 : entry->devTeam = nullptr;
360 : }
361 :
362 54 : entry->channelsList.clear();
363 54 : entry->hostChannelNums.clear();
364 54 : }
365 :
366 19 : void HcommTeamMgr::FreeWindowResources(WindowEntry* entry)
367 : {
368 19 : if (entry->devMems != nullptr) {
369 8 : (void)hrtFree(entry->devMems);
370 8 : entry->devMems = nullptr;
371 : }
372 19 : if (entry->hostMems != nullptr) {
373 8 : free(entry->hostMems);
374 8 : entry->hostMems = nullptr;
375 : }
376 19 : if (entry->devWindow != nullptr) {
377 19 : (void)hrtFree(entry->devWindow);
378 19 : entry->devWindow = nullptr;
379 : }
380 19 : }
381 :
382 54 : HcommResult HcommTeamMgr::ValidateSubTeam(TeamEntry* worldEntry, const HcommTeamCreateDesc* desc)
383 : {
384 : /* worldEntry 为 nullptr 表示创建 world team（无父 team），无需 sub team 成员校验。 */
385 54 : CHK_PRT_RET(
386 : worldEntry == nullptr, HCCL_INFO("[TeamCreate] worldEntry is null, skip sub team validation"), HCOMM_SUCCESS);
387 3 : CHK_PRT_RET(
388 : desc->memberNum > worldEntry->hostTeam.memberNum,
389 : HCCL_ERROR(
390 : "[TeamCreate] sub team memberNum[%u] > world team memberNum[%u]", desc->memberNum,
391 : worldEntry->hostTeam.memberNum),
392 : HCOMM_E_PARA);
393 :
394 3 : uint32_t worldMemberNum = worldEntry->hostTeam.memberNum;
395 11 : for (uint32_t i = 0; i < desc->memberNum; i++) {
396 8 : CHK_PRT_RET(
397 : desc->worldMemberIds[i] >= worldMemberNum,
398 : HCCL_ERROR(
399 : "[TeamCreate] worldMemberIds[%u]=%u >= world team memberNum[%u]", i, desc->worldMemberIds[i],
400 : worldMemberNum),
401 : HCOMM_E_PARA);
402 : }
403 3 : return HCOMM_SUCCESS;
404 : }
405 :
406 54 : void HcommTeamMgr::InitTeamEntry(TeamEntry* entry, const HcommTeamCreateDesc* desc, HcommTeamHandle worldTeam)
407 : {
408 54 : entry->hostTeam.header.version = HCOMM_TEAM_VERSION;
409 54 : entry->hostTeam.header.magicWord = HCOMM_TEAM_MAGIC_WORD;
410 54 : entry->hostTeam.header.size = sizeof(HcommTeam);
411 54 : entry->hostTeam.header.reserved = 0;
412 54 : entry->hostTeam.memberNum = desc->memberNum;
413 54 : entry->hostTeam.selfMemberId = desc->selfMemberId;
414 54 : entry->hostTeam.netLayer = desc->netLayer;
415 54 : entry->hostTeam.engine = COMM_ENGINE_RESERVED;
416 54 : entry->syncMemReq = desc->requirement;
417 : entry->syncMemSize
418 54 : = static_cast<uint64_t>(
419 54 : desc->requirement.signalCount + desc->requirement.counterCount + desc->requirement.barrierCount)
420 54 : * sizeof(uint64_t) * desc->memberNum;
421 54 : entry->hostTeam.syncMem.syncMemReq = desc->requirement;
422 54 : entry->hostTeam.syncMem.syncMemSize = entry->syncMemSize;
423 :
424 54 : if (worldTeam != nullptr) {
425 3 : entry->worldTeamHandle = worldTeam;
426 3 : entry->isSubTeam = true;
427 : }
428 54 : }
429 :
430 54 : HcommResult HcommTeamMgr::AllocAndSyncTeam(TeamEntry* entry, const HcommTeamCreateDesc* desc)
431 : {
432 54 : HcommResult ret = AllocAndCopyWorldTeamIds(entry, desc->worldMemberIds, desc->memberNum);
433 54 : if (ret != HCOMM_SUCCESS) {
434 2 : HCCL_ERROR("[AllocAndSyncTeam] AllocAndCopyWorldTeamIds failed");
435 2 : FreeTeamResources(entry);
436 2 : return ret;
437 : }
438 :
439 52 : void* devTeamPtr = nullptr;
440 52 : ret = static_cast<HcommResult>(hrtMalloc(&devTeamPtr, static_cast<uint64_t>(sizeof(HcommTeam))));
441 52 : if (ret != HCOMM_SUCCESS) {
442 1 : HCCL_ERROR("[AllocAndSyncTeam] hrtMalloc devTeam failed, ret[%d]", ret);
443 1 : FreeTeamResources(entry);
444 1 : return ret;
445 : }
446 51 : entry->devTeam = static_cast<HcommTeamHandle>(devTeamPtr);
447 :
448 51 : ret = SyncTeamToDevice(entry);
449 51 : if (ret != HCOMM_SUCCESS) {
450 1 : HCCL_ERROR("[AllocAndSyncTeam] SyncTeamToDevice failed, ret[%d]", ret);
451 1 : FreeTeamResources(entry);
452 1 : return ret;
453 : }
454 50 : return HCOMM_SUCCESS;
455 : }
456 :
457 55 : HcommResult HcommTeamMgr::TeamCreate(
458 : HcommTeamHandle worldTeam, const HcommTeamCreateDesc* desc, HcommTeamHandle* team, uint64_t* outSyncMemSize)
459 : {
460 55 : TeamEntry* worldEntry = nullptr;
461 55 : if (worldTeam != nullptr) {
462 4 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
463 4 : worldEntry = FindTeamByHandleLocked(worldTeam);
464 4 : CHK_PRT_RET(
465 : worldEntry == nullptr, HCCL_ERROR("[TeamCreate] worldTeam handle[%p] not found", worldTeam),
466 : HCOMM_E_NOT_FOUND);
467 4 : }
468 :
469 54 : HcommResult ret = ValidateSubTeam(worldEntry, desc);
470 54 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[TeamCreate] ValidateSubTeam failed"), ret);
471 :
472 54 : auto entry = std::make_unique<TeamEntry>();
473 54 : InitTeamEntry(entry.get(), desc, worldTeam);
474 :
475 54 : ret = AllocAndSyncTeam(entry.get(), desc);
476 54 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[TeamCreate] AllocAndSyncTeam failed"), ret);
477 :
478 50 : *outSyncMemSize = entry->syncMemSize;
479 50 : *team = entry->devTeam;
480 :
481 : {
482 50 : std::unique_lock<std::shared_mutex> lock(teamsRwMutex_);
483 50 : teams_[*team] = std::move(entry);
484 50 : }
485 :
486 50 : HCCL_INFO(
487 : "[TeamCreate] team created, memberNum[%u], selfMemberId[%u], "
488 : "netLayer[%u], protocol[%d], "
489 : "signalCount[%u], counterCount[%u], barrierCount[%u], "
490 : "syncMemSize[%llu], isSubTeam[%d], devTeam[%p]",
491 : desc->memberNum, desc->selfMemberId, desc->netLayer, static_cast<int32_t>(desc->protocol),
492 : desc->requirement.signalCount, desc->requirement.counterCount, desc->requirement.barrierCount, *outSyncMemSize,
493 : static_cast<int32_t>(worldTeam != nullptr), *team);
494 :
495 50 : return HCOMM_SUCCESS;
496 54 : }
497 :
498 14 : HcommResult HcommTeamMgr::TeamDestroy(HcommTeamHandle team)
499 : {
500 14 : std::unique_lock<std::shared_mutex> lock(teamsRwMutex_);
501 14 : auto it = teams_.find(team);
502 14 : CHK_PRT_RET(
503 : it == teams_.end(), HCCL_WARNING("[TeamDestroy] team handle[%p] not found, maybe already destroyed", team),
504 : HCOMM_SUCCESS);
505 :
506 13 : bool isSubTeam = it->second->isSubTeam;
507 13 : FreeTeamResources(it->second.get());
508 13 : teams_.erase(it);
509 :
510 13 : HCCL_INFO("[TeamDestroy] team[%p] destroyed, isSubTeam[%d]", team, static_cast<int32_t>(isSubTeam));
511 13 : return HCOMM_SUCCESS;
512 14 : }
513 :
514 20 : HcommResult HcommTeamMgr::WindowRegister(HcommTeamHandle team, HcommWindowHandle* handle)
515 : {
516 : // 校验 team 存在且为 worldTeam：shared_lock 仅覆盖查找/校验，不可延伸到 unique_lock 区间，否则自死锁。
517 : {
518 20 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
519 20 : TeamEntry* entry = FindTeamByHandleLocked(team);
520 20 : CHK_PRT_RET(
521 : entry == nullptr, HCCL_ERROR("[WindowRegister] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
522 20 : CHK_PRT_RET(
523 : entry->isSubTeam, HCCL_ERROR("[WindowRegister] subteam cannot register window, use worldTeam"),
524 : HCOMM_E_PARA);
525 20 : }
526 :
527 20 : auto winEntry = std::make_unique<WindowEntry>();
528 20 : winEntry->teamHandle = team;
529 20 : winEntry->hostWindow.header.version = HCOMM_WINDOW_VERSION;
530 20 : winEntry->hostWindow.header.magicWord = HCOMM_WINDOW_MAGIC_WORD;
531 20 : winEntry->hostWindow.header.size = sizeof(HcommWindow);
532 20 : winEntry->hostWindow.header.reserved = 0;
533 20 : winEntry->hostWindow.worldTeam = team;
534 :
535 20 : void* devWindowPtr = nullptr;
536 20 : HcommResult ret = static_cast<HcommResult>(hrtMalloc(&devWindowPtr, static_cast<uint64_t>(sizeof(HcommWindow))));
537 20 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[WindowRegister] hrtMalloc devWindow failed, ret[%d]", ret), ret);
538 19 : winEntry->devWindow = static_cast<HcommWindowHandle>(devWindowPtr);
539 :
540 19 : ret = SyncWindowToDevice(winEntry.get());
541 19 : if (ret != HCOMM_SUCCESS) {
542 1 : HCCL_ERROR("[WindowRegister] SyncWindowToDevice failed, ret[%d]", ret);
543 1 : FreeWindowResources(winEntry.get());
544 1 : return ret;
545 : }
546 :
547 18 : *handle = winEntry->devWindow;
548 :
549 : {
550 : // 同时持有 windows_ 与 windowToTeamMap_ 写锁，保证 window 登记原子性（锁顺序：windows→windowToTeam）
551 18 : std::unique_lock<std::shared_mutex> winLock(windowsRwMutex_);
552 18 : std::unique_lock<std::shared_mutex> mapLock(windowToTeamRwMutex_);
553 18 : windowToTeamMap_[*handle] = team;
554 18 : windows_[*handle] = std::move(winEntry);
555 18 : }
556 :
557 18 : HCCL_INFO("[WindowRegister] window created, team[%p], devWindow[%p]", team, devWindowPtr);
558 18 : return HCOMM_SUCCESS;
559 20 : }
560 :
561 9 : HcommResult HcommTeamMgr::AllocAndCopyWindowMems(WindowEntry* winEntry, uint64_t memberNum, const CommMem* src)
562 : {
563 9 : winEntry->hostMems = static_cast<CommMem*>(calloc(memberNum, sizeof(CommMem)));
564 9 : if (winEntry->hostMems == nullptr) {
565 0 : HCCL_ERROR("[AllocAndCopyWindowMems] calloc hostMems failed");
566 0 : return HCOMM_E_PTR;
567 : }
568 9 : errno_t memRet = memcpy_s(winEntry->hostMems, memberNum * sizeof(CommMem), src, memberNum * sizeof(CommMem));
569 9 : if (memRet != EOK) {
570 0 : HCCL_ERROR("[AllocAndCopyWindowMems] memcpy_s hostMems failed, ret[%d]", memRet);
571 0 : free(winEntry->hostMems);
572 0 : winEntry->hostMems = nullptr;
573 0 : return HCOMM_E_MEMORY;
574 : }
575 :
576 9 : void* devMemsPtr = nullptr;
577 : HcommResult ret
578 9 : = static_cast<HcommResult>(hrtMalloc(&devMemsPtr, static_cast<uint64_t>(memberNum * sizeof(CommMem))));
579 9 : if (ret != HCOMM_SUCCESS) {
580 1 : HCCL_ERROR("[AllocAndCopyWindowMems] hrtMalloc devMems failed, ret[%d]", ret);
581 1 : free(winEntry->hostMems);
582 1 : winEntry->hostMems = nullptr;
583 1 : return ret;
584 : }
585 :
586 8 : ret = static_cast<HcommResult>(hrtMemSyncCopy(
587 : devMemsPtr, static_cast<uint64_t>(memberNum * sizeof(CommMem)), src,
588 : static_cast<uint64_t>(memberNum * sizeof(CommMem)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
589 8 : if (ret != HCOMM_SUCCESS) {
590 0 : HCCL_ERROR("[AllocAndCopyWindowMems] hrtMemSyncCopy devMems failed, ret[%d]", ret);
591 0 : (void)hrtFree(devMemsPtr);
592 0 : free(winEntry->hostMems);
593 0 : winEntry->hostMems = nullptr;
594 0 : return ret;
595 : }
596 8 : winEntry->devMems = devMemsPtr;
597 8 : winEntry->hostWindow.mems = static_cast<CommMem*>(devMemsPtr);
598 8 : winEntry->hostWindow.memsNum = memberNum;
599 8 : return HCOMM_SUCCESS;
600 : }
601 :
602 3 : HcommResult HcommTeamMgr::MergeWindowMems(WindowEntry* winEntry, uint64_t memberNum, const CommMem* src)
603 : {
604 3 : CHK_PRT_RET(
605 : winEntry->hostMems == nullptr || winEntry->devMems == nullptr,
606 : HCCL_ERROR("[MergeWindowMems] hostMems or devMems is null, not allocated yet"), HCOMM_E_PTR);
607 3 : CHK_PRT_RET(
608 : memberNum != winEntry->hostWindow.memsNum,
609 : HCCL_ERROR(
610 : "[MergeWindowMems] memberNum[%llu] != memsNum[%llu], dimension mismatch", memberNum,
611 : winEntry->hostWindow.memsNum),
612 : HCOMM_E_PARA);
613 :
614 11 : for (uint64_t i = 0; i < memberNum; i++) {
615 9 : if (src[i].addr != nullptr) {
616 4 : CHK_PRT_RET(
617 : winEntry->hostMems[i].addr != nullptr,
618 : HCCL_ERROR("[MergeWindowMems] member[%llu] already bound, register a new window to rebind", i),
619 : HCOMM_E_PARA);
620 3 : winEntry->hostMems[i] = src[i];
621 : }
622 : }
623 :
624 : // 重新把 hostMems 整块 sync 到 devMems
625 4 : HcclResult hrtRet = hrtMemSyncCopy(
626 2 : winEntry->devMems, static_cast<uint64_t>(memberNum * sizeof(CommMem)), winEntry->hostMems,
627 : static_cast<uint64_t>(memberNum * sizeof(CommMem)), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE);
628 2 : HcommResult ret = static_cast<HcommResult>(hrtRet);
629 2 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[MergeWindowMems] hrtMemSyncCopy devMems failed, ret[%d]", ret), ret);
630 2 : return HCOMM_SUCCESS;
631 : }
632 :
633 13 : HcommResult HcommTeamMgr::BindWindow(HcommTeamHandle team, HcommWindowHandle handle, const HcommTeamWindowDesc* desc)
634 : {
635 : // 同时读 windows_ 与 teams_ 校验存在性（锁顺序：teams→windows）
636 13 : std::shared_lock<std::shared_mutex> teamLock(teamsRwMutex_);
637 13 : std::shared_lock<std::shared_mutex> winLock(windowsRwMutex_);
638 13 : WindowEntry* winEntry = FindWindowByHandleLocked(handle);
639 13 : CHK_PRT_RET(winEntry == nullptr, HCCL_ERROR("[BindWindow] window handle[%p] not found", handle), HCOMM_E_NOT_FOUND);
640 12 : TeamEntry* teamEntry = FindTeamByHandleLocked(team);
641 12 : CHK_PRT_RET(teamEntry == nullptr, HCCL_ERROR("[BindWindow] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
642 :
643 : (void)teamEntry;
644 : HcommResult ret;
645 12 : if (winEntry->hostMems == nullptr) {
646 9 : ret = AllocAndCopyWindowMems(winEntry, desc->memberNum, desc->mems);
647 9 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[BindWindow] AllocAndCopyWindowMems failed"), ret);
648 : } else {
649 3 : ret = MergeWindowMems(winEntry, desc->memberNum, desc->mems);
650 3 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[BindWindow] MergeWindowMems failed"), ret);
651 : }
652 :
653 10 : ret = SyncWindowToDevice(winEntry);
654 10 : if (ret != HCOMM_SUCCESS) {
655 1 : HCCL_ERROR("[BindWindow] SyncWindowToDevice failed, ret[%d]", ret);
656 1 : return ret;
657 : }
658 :
659 9 : HCCL_INFO(
660 : "[BindWindow] window bound, team[%p], handle[%p], devWindow[%p], mems[%p], memberNum[%u], devMems[%p]", team,
661 : handle, winEntry->devWindow, desc->mems, desc->memberNum, winEntry->devMems);
662 9 : return HCOMM_SUCCESS;
663 13 : }
664 :
665 6 : HcommResult HcommTeamMgr::WindowDeregister(HcommTeamHandle team, HcommWindowHandle handle)
666 : {
667 : {
668 6 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
669 6 : TeamEntry* entry = FindTeamByHandleLocked(team);
670 6 : CHK_PRT_RET(
671 : entry == nullptr, HCCL_ERROR("[WindowDeregister] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
672 4 : CHK_PRT_RET(
673 : entry->isSubTeam, HCCL_ERROR("[WindowDeregister] subteam cannot deregister window, use worldTeam"),
674 : HCOMM_E_PARA);
675 6 : }
676 : // 同时持有 windows_ 与 windowToTeamMap_ 写锁，保证销毁原子性（锁顺序：windows→windowToTeam）
677 4 : std::unique_lock<std::shared_mutex> winLock(windowsRwMutex_);
678 4 : std::unique_lock<std::shared_mutex> mapLock(windowToTeamRwMutex_);
679 4 : auto it = windows_.find(handle);
680 4 : CHK_PRT_RET(
681 : it == windows_.end(),
682 : HCCL_WARNING("[WindowDeregister] window handle[%p] not found, maybe already destroyed", handle), HCOMM_SUCCESS);
683 :
684 4 : FreeWindowResources(it->second.get());
685 4 : windowToTeamMap_.erase(handle);
686 4 : windows_.erase(it);
687 :
688 4 : HCCL_INFO("[WindowDeregister] window[%p] destroyed, team[%p]", handle, team);
689 4 : return HCOMM_SUCCESS;
690 4 : }
691 :
692 6 : void HcommTeamMgr::MergeChannelLists(
693 : TeamEntry* entry, const HcommTeamBindChannelsDesc* desc, std::vector<std::vector<uint64_t>>& newChannels)
694 : {
695 6 : if (entry->channelsList.empty()) {
696 4 : newChannels.resize(entry->hostTeam.memberNum);
697 : } else {
698 2 : newChannels = entry->channelsList;
699 : }
700 :
701 30 : for (uint32_t i = 0; i < entry->hostTeam.memberNum; i++) {
702 24 : uint64_t chNum = desc->channelNumPerMember[i];
703 24 : uint64_t* chSrc = desc->channelsByMemberId[i];
704 52 : for (uint64_t j = 0; j < chNum; j++) {
705 28 : newChannels[i].push_back(chSrc[j]);
706 : }
707 : }
708 6 : }
709 :
710 8 : HcommResult HcommTeamMgr::BindChannels(HcommTeamHandle team, const HcommTeamBindChannelsDesc* desc)
711 : {
712 8 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
713 8 : TeamEntry* entry = FindTeamByHandleLocked(team);
714 8 : CHK_PRT_RET(entry == nullptr, HCCL_ERROR("[BindChannels] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
715 :
716 7 : CHK_PRT_RET(
717 : desc->memberNum != entry->hostTeam.memberNum,
718 : HCCL_ERROR("[BindChannels] memberNum[%u] != team memberNum[%u]", desc->memberNum, entry->hostTeam.memberNum),
719 : HCOMM_E_PARA);
720 :
721 30 : for (uint32_t i = 0; i < entry->hostTeam.memberNum; i++) {
722 24 : CHK_PRT_RET(
723 : desc->channelNumPerMember[i] != 0 && desc->channelsByMemberId[i] == nullptr,
724 : HCCL_ERROR(
725 : "[BindChannels] member[%u] channelNum[%u] > 0 but channels is null", i, desc->channelNumPerMember[i]),
726 : HCOMM_E_PARA);
727 : }
728 :
729 : // 二维数组，需要填入HcommTeam中的channels字段
730 6 : std::vector<std::vector<uint64_t>> newChannels;
731 6 : MergeChannelLists(entry, desc, newChannels);
732 :
733 6 : std::vector<std::vector<uint64_t>> oldChannels = std::move(entry->channelsList);
734 6 : entry->channelsList = std::move(newChannels);
735 :
736 6 : HcommResult ret = AllocAndCopyChannels(entry);
737 6 : if (ret != HCOMM_SUCCESS) {
738 1 : HCCL_ERROR("[BindChannels] AllocAndCopyChannels failed, this bind does not take effect, "
739 : "previously bound channels are not affected");
740 1 : entry->channelsList = std::move(oldChannels);
741 1 : return ret;
742 : }
743 :
744 5 : ret = SyncTeamToDevice(entry);
745 5 : if (ret != HCOMM_SUCCESS) {
746 0 : HCCL_ERROR("[BindChannels] SyncTeamToDevice failed");
747 0 : return ret;
748 : }
749 :
750 5 : HCCL_INFO(
751 : "[BindChannels] channels appended and synced, team[%p], channelNumPerMember[%p], "
752 : "channelsByMemberId[%p], memberNum[%u], isSubTeam[%d]",
753 : team, desc->channelNumPerMember, desc->channelsByMemberId, entry->hostTeam.memberNum, entry->isSubTeam);
754 5 : return HCOMM_SUCCESS;
755 8 : }
756 :
757 12 : HcommResult HcommTeamMgr::BindSyncMem(HcommTeamHandle team, const HcommTeamBindSyncMemDesc* desc)
758 : {
759 12 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
760 12 : TeamEntry* entry = FindTeamByHandleLocked(team);
761 12 : CHK_PRT_RET(entry == nullptr, HCCL_ERROR("[BindSyncMem] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
762 :
763 11 : CHK_PRT_RET(
764 : desc->remoteMemNum != entry->hostTeam.memberNum,
765 : HCCL_ERROR("[BindSyncMem] remoteMemNum[%u] != memberNum[%u]", desc->remoteMemNum, entry->hostTeam.memberNum),
766 : HCOMM_E_PARA);
767 :
768 10 : HcommResult ret = AllocAndCopyRemoteMems(entry, desc->remoteMems, desc->remoteMemNum);
769 10 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[BindSyncMem] AllocAndCopyRemoteMems failed"), ret);
770 :
771 9 : if (entry->hostTeam.syncMem.shadowMem.addr != nullptr) {
772 2 : (void)hrtFree(entry->hostTeam.syncMem.shadowMem.addr);
773 2 : entry->hostTeam.syncMem.shadowMem.addr = nullptr;
774 2 : entry->hostTeam.syncMem.shadowMem.size = 0;
775 : }
776 9 : void* devShadowMemPtr = nullptr;
777 9 : uint64_t shadowMemSize = entry->hostTeam.syncMem.syncMemSize;
778 9 : ret = static_cast<HcommResult>(hrtMalloc(&devShadowMemPtr, shadowMemSize));
779 9 : if (ret != HCOMM_SUCCESS) {
780 0 : HCCL_ERROR("[BindSyncMem] hrtMalloc shadowMem failed, size[%llu] ret[%d]", shadowMemSize, ret);
781 0 : return ret;
782 : }
783 9 : entry->hostTeam.syncMem.shadowMem.type = COMM_MEM_TYPE_DEVICE;
784 9 : entry->hostTeam.syncMem.shadowMem.addr = devShadowMemPtr;
785 9 : entry->hostTeam.syncMem.shadowMem.size = shadowMemSize;
786 :
787 9 : ret = SyncTeamToDevice(entry);
788 9 : CHK_PRT_RET(ret != HCOMM_SUCCESS, HCCL_ERROR("[BindSyncMem] SyncTeamToDevice failed"), ret);
789 :
790 8 : HCCL_INFO("[BindSyncMem] syncMem bound, remoteMems[%p], remoteMemNum[%u]", desc->remoteMems, desc->remoteMemNum);
791 8 : return HCOMM_SUCCESS;
792 12 : }
793 :
794 2 : HcommResult HcommTeamMgr::GetNetLayer(HcommTeamHandle team, uint32_t* netLayer)
795 : {
796 2 : std::shared_lock<std::shared_mutex> lock(teamsRwMutex_);
797 2 : TeamEntry* entry = FindTeamByHandleLocked(team);
798 2 : CHK_PRT_RET(entry == nullptr, HCCL_ERROR("[GetNetLayer] team handle[%p] not found", team), HCOMM_E_NOT_FOUND);
799 :
800 1 : *netLayer = entry->hostTeam.netLayer;
801 1 : return HCOMM_SUCCESS;
802 2 : }
803 :
804 : } // namespace hcomm
|
