Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "p2p_mgmt.h"
12 :
13 : #include <chrono>
14 : #include <thread>
15 : #include "adapter_error_manager.h"
16 : #include "externalinput_pub.h"
17 : #include "device_capacity.h"
18 : #include "mem_name_repository_pub.h"
19 : #include "sal_pub.h"
20 : #include "driver/ascend_hal.h"
21 : #include "workflow_pub.h"
22 :
23 : namespace hccl {
24 : const u32 DEVICE_PER_MODULE = 8; // A+X 一个mesh卡数
25 : const u32 DIE_PER_MODULE = 16; // 910_93 16die
26 : std::atomic<bool> P2PMgmt::initFlag_ = {false};
27 554 : P2PMgmt& P2PMgmt::Instance()
28 : {
29 554 : static P2PMgmt mgmt;
30 554 : return mgmt;
31 : }
32 :
33 9 : P2PMgmt::P2PMgmt() : deviceType_(DevType::DEV_TYPE_COUNT) { initFlag_ = true; }
34 :
35 9 : P2PMgmt::~P2PMgmt() { initFlag_ = false; }
36 :
37 161 : HcclResult P2PMgmt::EnableP2P(std::vector<uint32_t> remoteDevices)
38 : {
39 161 : if (initFlag_) {
40 160 : isStandardCardFor910B_ = IsStandardCardFor910B(remoteDevices);
41 963 : for (auto& remoteDevicePhysicID : remoteDevices) {
42 800 : if (IsNeedEstablishP2Pconnection(remoteDevicePhysicID)) {
43 800 : CHK_RET(EnableP2P(remoteDevicePhysicID));
44 : }
45 : }
46 : }
47 162 : return HCCL_SUCCESS;
48 : }
49 :
50 800 : HcclResult P2PMgmt::EnableP2P(uint32_t remoteDevicePhysicID)
51 : {
52 800 : if (Is310PDevice()) {
53 0 : return HCCL_SUCCESS;
54 : }
55 : int32_t localDeviceLogicID;
56 800 : CHK_RET(hrtGetDevice(&localDeviceLogicID));
57 : u32 maxDeviceNum;
58 799 : CHK_RET(GetMaxDevNum(maxDeviceNum));
59 801 : CHK_PRT_RET(
60 : static_cast<u32>(localDeviceLogicID) >= maxDeviceNum,
61 : HCCL_ERROR(
62 : "[EnableP2P]localDeviceLogicID[%d] is bigger than maxDeviceNum[%u]", localDeviceLogicID, maxDeviceNum),
63 : HCCL_E_INTERNAL);
64 :
65 801 : std::unique_lock<std::mutex> lock(connectionsLock_[localDeviceLogicID]);
66 801 : auto& iterLocalDevice = connectionsInfo_[localDeviceLogicID];
67 801 : auto iterRemoteDevice = iterLocalDevice.find(remoteDevicePhysicID);
68 800 : if ((iterRemoteDevice == iterLocalDevice.end()) || (iterRemoteDevice->second.reference == 0)) {
69 798 : bool isMarsterIdDiff = false;
70 798 : u32 localDevicePhysicID = 0;
71 798 : CHK_RET(hrtGetDevicePhyIdByIndex(localDeviceLogicID, localDevicePhysicID));
72 798 : CHK_RET(CheckMarsterId(remoteDevicePhysicID, localDevicePhysicID, isMarsterIdDiff));
73 799 : HCCL_INFO(
74 : "[EnableP2P][CheckMarsterId]localDevicePhysicID[%u], remoteDevicePhysicID[%u], isMarsterIdDiff[%s]",
75 : localDevicePhysicID, remoteDevicePhysicID, isMarsterIdDiff ? "true" : "false");
76 799 : if (isMarsterIdDiff) {
77 0 : CHK_RET(hrtEnableP2P(localDeviceLogicID, remoteDevicePhysicID));
78 0 : HCCL_INFO(
79 : "[EnableP2P]enable p2p: local logic id:%d, local physic id:%u, remote physic id:%u.",
80 : localDeviceLogicID, localDevicePhysicID, remoteDevicePhysicID);
81 : }
82 799 : iterLocalDevice[remoteDevicePhysicID].status = P2PStatus::P2P_STATUS_ENABLING;
83 797 : iterLocalDevice[remoteDevicePhysicID].reference++;
84 798 : return HCCL_SUCCESS;
85 : } else {
86 : // 使已执行过 enable，且未执行过 disable，不重复执行 enable p2p。
87 2 : iterLocalDevice[remoteDevicePhysicID].reference++;
88 : }
89 2 : return HCCL_SUCCESS;
90 800 : }
91 :
92 0 : HcclResult P2PMgmt::DisableAllP2P()
93 : {
94 : int32_t localDeviceLogicID;
95 0 : CHK_RET(hrtGetDevice(&localDeviceLogicID));
96 : u32 maxDeviceNum;
97 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
98 0 : CHK_PRT_RET(
99 : static_cast<u32>(localDeviceLogicID) >= maxDeviceNum,
100 : HCCL_ERROR(
101 : "[DisableAllP2P]localDeviceLogicID[%d] is bigger than maxDeviceNum[%u]", localDeviceLogicID, maxDeviceNum),
102 : HCCL_E_INTERNAL);
103 :
104 0 : std::map<uint32_t, P2PConnectionInfo> localP2PInfo;
105 0 : std::unique_lock<std::mutex> lock(connectionsLock_[localDeviceLogicID]);
106 0 : auto& iterLocalDevice = connectionsInfo_[localDeviceLogicID];
107 0 : if (iterLocalDevice.empty()) {
108 0 : return HCCL_SUCCESS;
109 : } else {
110 0 : localP2PInfo = iterLocalDevice;
111 : }
112 0 : lock.unlock();
113 :
114 0 : for (auto& iterRemoteDevice : localP2PInfo) {
115 0 : if (iterRemoteDevice.second.reference == 0) {
116 0 : continue;
117 : }
118 :
119 0 : bool isMarsterIdDiff = false;
120 0 : u32 localDevicePhysicID = 0;
121 0 : CHK_RET(hrtGetDevicePhyIdByIndex(localDeviceLogicID, localDevicePhysicID));
122 0 : HcclResult ret = CheckMarsterId(iterRemoteDevice.first, localDevicePhysicID, isMarsterIdDiff);
123 0 : CHK_PRT_RET(
124 : ret != HCCL_SUCCESS,
125 : HCCL_ERROR(
126 : "[Disable][AllP2P]check pcie connection failed. device info: local logic id:%d, "
127 : "remote physic id:%u.",
128 : localDeviceLogicID, iterRemoteDevice.first),
129 : ret);
130 0 : if (isMarsterIdDiff) {
131 0 : HCCL_INFO(
132 : "there is active p2p connections. in P2PMgmt disable all p2p, it is forced to disable p2p. "
133 : "device info: local logic id:%d, remote physic id:%u.",
134 : localDeviceLogicID, iterRemoteDevice.first);
135 0 : CHK_RET(hrtDisableP2P(localDeviceLogicID, iterRemoteDevice.first));
136 : }
137 : }
138 :
139 0 : lock.lock();
140 0 : connectionsInfo_[localDeviceLogicID].clear();
141 0 : lock.unlock();
142 0 : return HCCL_SUCCESS;
143 0 : }
144 :
145 313 : HcclResult P2PMgmt::DisableP2P(std::vector<uint32_t> remoteDevices)
146 : {
147 313 : if (initFlag_) {
148 313 : if (Is310PDevice()) {
149 0 : return HCCL_SUCCESS;
150 : }
151 : int32_t localDeviceLogicID;
152 313 : CHK_RET(hrtGetDevice(&localDeviceLogicID));
153 : u32 maxDeviceNum;
154 313 : CHK_RET(GetMaxDevNum(maxDeviceNum));
155 313 : CHK_PRT_RET(
156 : static_cast<u32>(localDeviceLogicID) >= maxDeviceNum,
157 : HCCL_ERROR(
158 : "[DisableP2P]localDeviceLogicID[%d] is bigger than maxDeviceNum[%u]", localDeviceLogicID, maxDeviceNum),
159 : HCCL_E_INTERNAL);
160 :
161 313 : HcclWorkflowMode mode = GetWorkflowMode();
162 313 : if (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
163 92 : MemNameRepository::GetInstance(localDeviceLogicID)->ClearMemNameRepository();
164 : }
165 :
166 1282 : for (auto& remoteDevicePhysicID : remoteDevices) {
167 969 : if (IsNeedEstablishP2Pconnection(remoteDevicePhysicID)) {
168 971 : CHK_RET(DisableP2P(localDeviceLogicID, remoteDevicePhysicID));
169 : }
170 : }
171 : }
172 313 : return HCCL_SUCCESS;
173 : }
174 :
175 971 : HcclResult P2PMgmt::DisableP2P(uint32_t localDeviceLogicID, uint32_t remoteDevicePhysicID)
176 : {
177 971 : std::unique_lock<std::mutex> lock(connectionsLock_[localDeviceLogicID]);
178 971 : auto& iterLocalDevice = connectionsInfo_[localDeviceLogicID];
179 970 : auto iterRemoteDevice = iterLocalDevice.find(remoteDevicePhysicID);
180 967 : if ((iterRemoteDevice == iterLocalDevice.end()) || (iterRemoteDevice->second.reference == 0)) {
181 170 : HCCL_WARNING(
182 : "there is no p2p connections, no need to disable p2p. "
183 : "device info: local logic id:%d, remote physic id:%u.",
184 : localDeviceLogicID, remoteDevicePhysicID);
185 170 : return HCCL_SUCCESS;
186 : }
187 :
188 799 : iterRemoteDevice->second.reference--;
189 798 : if (iterRemoteDevice->second.reference == 0) {
190 796 : bool isMarsterIdDiff = false;
191 796 : u32 localDevicePhysicID = 0;
192 796 : CHK_RET(hrtGetDevicePhyIdByIndex(localDeviceLogicID, localDevicePhysicID, true));
193 799 : HCCL_INFO("local logic id:%d, local physic id:%u.", localDeviceLogicID, localDevicePhysicID);
194 799 : HcclResult ret = CheckMarsterId(remoteDevicePhysicID, localDevicePhysicID, isMarsterIdDiff);
195 799 : CHK_PRT_RET(
196 : ret != HCCL_SUCCESS,
197 : HCCL_ERROR(
198 : "[Disable][P2P]check pcie connection failed. device info: local logic id:%d, "
199 : "remote physic id:%u.",
200 : localDeviceLogicID, remoteDevicePhysicID),
201 : ret);
202 799 : if (isMarsterIdDiff) {
203 0 : HCCL_INFO("disable p2p: local logic id:%d, remote physic id:%u.", localDeviceLogicID, remoteDevicePhysicID);
204 0 : CHK_RET(hrtDisableP2P(localDeviceLogicID, remoteDevicePhysicID));
205 : }
206 799 : iterLocalDevice[remoteDevicePhysicID].status = P2PStatus::P2P_STATUS_DISABLED;
207 : }
208 :
209 798 : return HCCL_SUCCESS;
210 968 : }
211 :
212 79 : HcclResult P2PMgmt::WaitP2PEnabled(std::vector<uint32_t> remoteDevices, std::function<bool()> needStop)
213 : {
214 79 : if (initFlag_) {
215 172 : for (auto& remoteDevicePhysicID : remoteDevices) {
216 92 : if (IsNeedEstablishP2Pconnection(remoteDevicePhysicID)) {
217 93 : CHK_RET(WaitP2PEnabled(remoteDevicePhysicID, needStop));
218 : }
219 : }
220 : }
221 79 : return HCCL_SUCCESS;
222 : }
223 :
224 1690 : HcclResult P2PMgmt::CheckMarsterId(uint32_t remoteDevicePhysicID, uint32_t localDevicePhysicID, bool& isMarsterIdDiff)
225 : {
226 1690 : if (localDevicePhysicID == remoteDevicePhysicID) {
227 316 : isMarsterIdDiff = false;
228 316 : return HCCL_SUCCESS;
229 : }
230 1374 : if (deviceType_ == DevType::DEV_TYPE_910B || deviceType_ == DevType::DEV_TYPE_910_93) {
231 : s64 localDevicePhysicValue, remoteDevicePhysicValue;
232 1019 : CHK_RET(hrtGetPhyDeviceInfo(
233 : localDevicePhysicID, MODULE_TYPE_SYSTEM, RT_PHY_INFO_TYPE_MASTER_ID, localDevicePhysicValue));
234 1019 : CHK_RET(hrtGetPhyDeviceInfo(
235 : remoteDevicePhysicID, MODULE_TYPE_SYSTEM, RT_PHY_INFO_TYPE_MASTER_ID, remoteDevicePhysicValue));
236 :
237 1020 : isMarsterIdDiff = (localDevicePhysicValue == remoteDevicePhysicValue) ? false : true;
238 1020 : return HCCL_SUCCESS;
239 : }
240 : LinkTypeInServer linkType;
241 355 : CHK_RET(hrtGetPairDeviceLinkType(localDevicePhysicID, remoteDevicePhysicID, linkType));
242 :
243 0 : isMarsterIdDiff = ((linkType != LinkTypeInServer::HCCS_TYPE) && (linkType != LinkTypeInServer::SIO_TYPE)
244 355 : && (linkType != LinkTypeInServer::HCCS_SW_TYPE)) ?
245 : true :
246 : false;
247 355 : return HCCL_SUCCESS;
248 : }
249 :
250 93 : HcclResult P2PMgmt::WaitP2PEnabled(uint32_t remoteDevicePhysicID, std::function<bool()> needStop)
251 : {
252 93 : if (Is310PDevice()) {
253 0 : return HCCL_SUCCESS;
254 : }
255 : int32_t localDeviceLogicID;
256 93 : CHK_RET(hrtGetDevice(&localDeviceLogicID));
257 : u32 maxDeviceNum;
258 93 : CHK_RET(GetMaxDevNum(maxDeviceNum));
259 93 : CHK_PRT_RET(
260 : static_cast<u32>(localDeviceLogicID) >= maxDeviceNum,
261 : HCCL_ERROR(
262 : "[WaitP2PEnabled]localDeviceLogicID[%d] is bigger than maxDeviceNum[%u]", localDeviceLogicID, maxDeviceNum),
263 : HCCL_E_INTERNAL);
264 93 : std::unique_lock<std::mutex> lock(connectionsLock_[localDeviceLogicID]);
265 :
266 93 : auto& iterLocalDevice = connectionsInfo_[localDeviceLogicID];
267 93 : auto iterRemoteDevice = iterLocalDevice.find(remoteDevicePhysicID);
268 186 : bool bErr = (iterRemoteDevice == iterLocalDevice.end()) || (iterRemoteDevice->second.reference == 0)
269 186 : || (iterRemoteDevice->second.status == P2PStatus::P2P_STATUS_DISABLED);
270 93 : CHK_PRT_RET(
271 : bErr,
272 : HCCL_ERROR(
273 : "[Wait][P2PEnabled]wait p2p enabled failed. enable operation has not been executed, "
274 : "ret[%u]. device info: local logic id:%d, remote physic id:%u.",
275 : HCCL_E_INTERNAL, localDeviceLogicID, remoteDevicePhysicID),
276 : HCCL_E_INTERNAL);
277 :
278 93 : if (iterRemoteDevice->second.status == P2PStatus::P2P_STATUS_ENABLED) {
279 0 : return HCCL_SUCCESS;
280 : } else {
281 93 : bool isMarsterIdDiff = false;
282 93 : u32 localDevicePhysicID = 0;
283 93 : CHK_RET(hrtGetDevicePhyIdByIndex(localDeviceLogicID, localDevicePhysicID));
284 93 : HcclResult ret = CheckMarsterId(remoteDevicePhysicID, localDevicePhysicID, isMarsterIdDiff);
285 93 : HCCL_INFO(
286 : "[WaitP2PEnabled][CheckMarsterId]localDevicePhysicID[%u], remoteDevicePhysicID[%u], isMarsterIdDiff[%s]",
287 : localDevicePhysicID, remoteDevicePhysicID, isMarsterIdDiff ? "true" : "false");
288 93 : CHK_PRT_RET(
289 : ret != HCCL_SUCCESS,
290 : HCCL_ERROR(
291 : "[Wait][P2PEnabled]check pcie connection failed. device info: local logic id:%d, "
292 : "remote physic id:%u.",
293 : localDeviceLogicID, remoteDevicePhysicID),
294 : ret);
295 93 : if (isMarsterIdDiff) {
296 0 : CHK_RET(WaitP2PConnected(localDeviceLogicID, remoteDevicePhysicID, needStop));
297 0 : HCCL_INFO(
298 : "[Wait]enable p2p: local logic id:%d, local physic id:%u, remote physic id:%u.", localDeviceLogicID,
299 : localDevicePhysicID, remoteDevicePhysicID);
300 : }
301 93 : iterLocalDevice[remoteDevicePhysicID].status = P2PStatus::P2P_STATUS_ENABLED;
302 : }
303 93 : return HCCL_SUCCESS;
304 93 : }
305 :
306 : HcclResult
307 0 : P2PMgmt::WaitP2PConnected(int32_t localDeviceLogicID, uint32_t remoteDevicePhysicID, std::function<bool()> needStop)
308 : {
309 : // 读取P2P状态超时时间
310 0 : const std::chrono::seconds timeout(GetExternalInputHcclLinkTimeOut());
311 0 : const std::chrono::milliseconds checkP2PTimeInterval(1); // 轮询P2P状态时间 1ms
312 0 : const auto start = TIME_NOW();
313 :
314 : while (true) {
315 0 : CHK_PRT_RET(needStop(), HCCL_ERROR("Terminating operation due to external request"), HCCL_E_INTERNAL);
316 :
317 0 : bool enabled = false;
318 0 : CHK_RET(CheckP2P(remoteDevicePhysicID, enabled));
319 :
320 0 : if (enabled) {
321 0 : HCCL_INFO(
322 : "connected p2p success, take time [%lld]us. device info: local logic id:%d, remote physic id:%u.",
323 : DURATION_US(TIME_NOW() - start), localDeviceLogicID, remoteDevicePhysicID);
324 0 : return HCCL_SUCCESS;
325 : }
326 0 : std::this_thread::sleep_for(checkP2PTimeInterval);
327 : /* 获取当前时间，如果耗时超过timeout，则返回错误 */
328 0 : const auto elapsed = std::chrono::duration_cast<std::chrono::seconds>(TIME_NOW() - start);
329 0 : if (elapsed > timeout) {
330 0 : RPT_INNER_ERR_PRT(
331 : "connected p2p timeout, timeout:%d s.local logicDevid:%d,"
332 : "remote physic id:%u The possible causes are as follows:1.the connection "
333 : "between this device and the target device is abnormal 2.an exception occurred "
334 : "at the target devices 3.The ranktable is not matched.",
335 : GetExternalInputHcclLinkTimeOut(), localDeviceLogicID, remoteDevicePhysicID);
336 :
337 0 : HCCL_ERROR(
338 : "[Wait][P2PConnected]connected p2p timeout, timeout:%d s. local logicDevid:%d, "
339 : "remote physic id:%u.",
340 : GetExternalInputHcclLinkTimeOut(), localDeviceLogicID, remoteDevicePhysicID);
341 0 : return HCCL_E_DRV;
342 : }
343 0 : }
344 : return HCCL_SUCCESS;
345 : }
346 :
347 0 : HcclResult P2PMgmt::CheckP2P(uint32_t remoteDevicePhysicID, bool& enabled)
348 : {
349 0 : uint32_t status = DRV_P2P_STATUS_DISABLE;
350 : int32_t localDeviceLogicID;
351 0 : CHK_RET(hrtGetDevice(&localDeviceLogicID));
352 :
353 0 : CHK_RET(hrtGetP2PStatus(localDeviceLogicID, remoteDevicePhysicID, &status));
354 :
355 0 : enabled = (status == DRV_P2P_STATUS_ENABLE);
356 0 : return HCCL_SUCCESS;
357 : }
358 :
359 : /*
360 : * ****************************************************************************
361 : * 判断localdevice和remotedevice是否在相同平面，在相同平面内的device间需要做P2P
362 : * *****************************************************************************
363 : */
364 1863 : bool P2PMgmt::IsNeedEstablishP2Pconnection(uint32_t remoteDevicePhysicID)
365 : {
366 1863 : if (static_cast<s32>(remoteDevicePhysicID) == HOST_DEVICE_ID)
367 0 : return false;
368 :
369 : int32_t localDeviceLogicID;
370 1863 : CHK_RET(hrtGetDevice(&localDeviceLogicID));
371 1865 : u32 localDevicePhysicID = 0;
372 1865 : CHK_RET(hrtGetDevicePhyIdByIndex(localDeviceLogicID, localDevicePhysicID));
373 1864 : CHK_RET(hrtGetDeviceType(deviceType_));
374 1864 : if (deviceType_ == DevType::DEV_TYPE_310P3 || isStandardCardFor910B_) {
375 0 : return true;
376 : }
377 1864 : u32 deviceNum = (deviceType_ == DevType::DEV_TYPE_910_93) ? DIE_PER_MODULE : DEVICE_PER_MODULE;
378 : return (
379 1864 : (localDevicePhysicID % deviceNum == remoteDevicePhysicID % deviceNum)
380 1864 : || (localDevicePhysicID / deviceNum == remoteDevicePhysicID / deviceNum));
381 : }
382 :
383 162 : bool P2PMgmt::IsStandardCardFor910B(std::vector<uint32_t>& remoteDevicePhysicIDs)
384 : {
385 : // 非910B场景返回false
386 162 : CHK_RET(hrtGetDeviceType(deviceType_));
387 160 : if (deviceType_ != DevType::DEV_TYPE_910B) {
388 44 : return false;
389 : }
390 :
391 : int32_t localDeviceLogicID;
392 116 : CHK_RET(hrtGetDevice(&localDeviceLogicID));
393 117 : u32 localDevicePhysicID = 0;
394 117 : CHK_RET(hrtGetDevicePhyIdByIndex(localDeviceLogicID, localDevicePhysicID));
395 : LinkTypeInServer linkType;
396 118 : for (auto remoteDevicePhysicID : remoteDevicePhysicIDs) {
397 234 : CHK_RET(hrtGetPairDeviceLinkType(localDevicePhysicID, remoteDevicePhysicID, linkType));
398 : // 两卡之间的链路是HCCS或者SIO时，返回false
399 118 : if (linkType == LinkTypeInServer::HCCS_TYPE || linkType == LinkTypeInServer::SIO_TYPE
400 0 : || linkType == LinkTypeInServer::HCCS_SW_TYPE) {
401 118 : return false;
402 : }
403 : }
404 0 : HCCL_INFO("[IsStandardCardFor910B] isStandardCardFor910B_[true]");
405 0 : return true;
406 : }
407 : } // namespace hccl
|
