Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "transport_heterog_roce.h"
12 : #include "log.h"
13 : #include "externalinput_pub.h"
14 : #include "mr_manager.h"
15 : #include "adapter_hccp.h"
16 : #include "adapter_rts.h"
17 : #include "network_manager_pub.h"
18 : #include "transport_ibverbs_pub.h"
19 : #include "hccl_socket.h"
20 :
21 : using namespace std;
22 : namespace hccl {
23 : constexpr u32 MAX_COSTTIME_COUNT = 1000;
24 : constexpr u32 MAX_TOTALCOST_COUNT = 10000; // 总耗时预警门槛 10ms
25 : constexpr u32 BLOCK_ALLOCATOR_POOL_SIZE = 4096;
26 : constexpr s32 PROTOCOL_TYPE = 0;
27 : constexpr s32 LINK_NUM = 3;
28 : constexpr u32 SOCKET_FOR_TAG_QP = 0;
29 : constexpr u32 SOCKET_FOR_DATA_QP = 1;
30 : constexpr u32 SOCKET_FOR_SENDRECV_QP = 2;
31 :
32 : constexpr u32 RECV_WQE_HDC_BATCH_NUM = 128;
33 : constexpr u32 RECV_WQE_HDC_BATCH_SUPPLEMENT = 1;
34 : constexpr u32 RECV_WQE_NUM_THRESHOLD = 96;
35 : constexpr u32 RECV_WQE_BATCH_SUPPLEMENT = 96;
36 : constexpr u32 SMALL_PAGE_SIZE = 4096;
37 : constexpr u32 EIGHE_BIT = 8;
38 : constexpr u32 WAIT_SLEEP_TIME_US = 50;
39 :
40 : enum class RdmaOp { OP_WRITE = 0, OP_SEND = 2, OP_READ = 4 };
41 :
42 0 : TransportHeterogRoce::TransportHeterogRoce(
43 : const std::string& transTag, HcclIpAddress& selfIp, HcclIpAddress& peerIp, u32 peerPort, u32 selfPort,
44 0 : const TransportResourceInfo& transportResourceInfo)
45 : : TransportHeterog(transTag, selfIp, peerIp, peerPort, selfPort, transportResourceInfo),
46 0 : nicRdmaHandle_(nullptr),
47 0 : mrManager_(transportResourceInfo.mrManager.get()),
48 0 : blockMemLkey_(transportResourceInfo.lkey),
49 0 : recvWqeBatchNum_(RECV_WQE_BATCH_NUM),
50 0 : recvWqeBatchThreshold_(RECV_WQE_NUM_THRESHOLD),
51 0 : recvWqeBatchSupplement_(RECV_WQE_BATCH_SUPPLEMENT),
52 0 : access_(RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_WRITE),
53 0 : tagRecvWqeNum_(0),
54 0 : dataRecvWqeNum_(0),
55 0 : dataRecvWqeExpNum_(0),
56 0 : memBlocksManager_(transportResourceInfo.memBlocksManager),
57 0 : pRecvWrInfosMem_(transportResourceInfo.pRecvWrInfosMem),
58 0 : deviceEvePtr_(nullptr),
59 0 : deviceEveLkey_(0),
60 0 : useDevMem_(true),
61 0 : isRawConn_(transportResourceInfo.isRawConn)
62 : {
63 0 : GetTransportResourceInfo(transportResourceInfo);
64 0 : }
65 :
66 0 : TransportHeterogRoce::TransportHeterogRoce(const TransportResourceInfo& transportResourceInfo)
67 : : TransportHeterog(transportResourceInfo),
68 0 : nicRdmaHandle_(nullptr),
69 0 : mrManager_(transportResourceInfo.mrManager.get()),
70 0 : blockMemLkey_(transportResourceInfo.lkey),
71 0 : recvWqeBatchNum_(RECV_WQE_BATCH_NUM),
72 0 : recvWqeBatchThreshold_(RECV_WQE_NUM_THRESHOLD),
73 0 : recvWqeBatchSupplement_(RECV_WQE_BATCH_SUPPLEMENT),
74 0 : access_(RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_WRITE),
75 0 : tagRecvWqeNum_(0),
76 0 : dataRecvWqeNum_(0),
77 0 : dataRecvWqeExpNum_(0),
78 0 : memBlocksManager_(transportResourceInfo.memBlocksManager),
79 0 : pRecvWrInfosMem_(transportResourceInfo.pRecvWrInfosMem),
80 0 : deviceEvePtr_(nullptr),
81 0 : deviceEveLkey_(0),
82 0 : useDevMem_(true),
83 0 : isRawConn_(transportResourceInfo.isRawConn)
84 : {
85 0 : GetTransportResourceInfo(transportResourceInfo);
86 0 : }
87 :
88 0 : TransportHeterogRoce::~TransportHeterogRoce() {}
89 :
90 0 : u64 HostAddrToDev(const u64& hostAddr, u64 hostAddrBegin, u64 devAddrBegin)
91 : {
92 0 : u64 devAddr = hostAddr - hostAddrBegin + devAddrBegin;
93 0 : return devAddr;
94 : }
95 :
96 0 : HcclResult TransportHeterogRoce::Init()
97 : {
98 0 : if (!isHdcMode_ && (remoteIsHdc_ && (deviceLogicId_ == HOST_DEVICE_ID))) {
99 0 : HCCL_INFO("TransportHeterogRoce no useDevMem_");
100 0 : useDevMem_ = false;
101 : }
102 :
103 0 : CHK_RET(CheckRecvMsgAndRequestBuffer());
104 :
105 0 : CHK_RET(GetNetworkResource());
106 :
107 0 : CHK_RET(PreQpConnect());
108 :
109 0 : CHK_RET(InitTransportConnect(PROTOCOL_TYPE, LINK_NUM));
110 :
111 0 : CHK_RET(ConnectAsync());
112 :
113 0 : return HCCL_SUCCESS;
114 : }
115 :
116 0 : HcclResult TransportHeterogRoce::Deinit()
117 : {
118 0 : if (isDeinited_ == true) {
119 0 : return HCCL_SUCCESS;
120 : }
121 :
122 0 : if (isHdcMode_) {
123 0 : if (deviceLogicId_ == HOST_DEVICE_ID) {
124 0 : CHK_PRT(MemBlocksManagerDeInit());
125 : }
126 0 : CHK_PRT(MrManagerDeInit());
127 0 : if (deviceEvePtr_ != nullptr) {
128 : #ifndef CCL_KERNEL
129 : CHK_RET(HrtDevFree(deviceEvePtr_));
130 : #endif
131 : }
132 : }
133 :
134 0 : CHK_RET(DeleteNotifyValueBuffer());
135 :
136 0 : CHK_RET(DestroyCqAndQp());
137 :
138 0 : CHK_RET(SocketClose());
139 :
140 0 : isDeinited_ = true;
141 0 : return HCCL_SUCCESS;
142 : }
143 :
144 : HcclResult
145 0 : TransportHeterogRoce::Isend(const TransData& sendData, const TransportEndPointParam& epParam, HcclRequestInfo*& request)
146 : {
147 0 : CHK_RET(GenerateSendRequest(sendData, epParam, request));
148 :
149 0 : u32 lkey = 0;
150 0 : CHK_RET(RegMr(
151 : reinterpret_cast<void*>(sendData.srcBuf), static_cast<u64>(sendData.count * SIZE_TABLE[sendData.dataType]),
152 : lkey));
153 0 : HCCL_DEBUG(
154 : "addr[%llu] count[%d] datatype[%s]", sendData.srcBuf, sendData.count,
155 : GetDataTypeEnumStr(sendData.dataType).c_str());
156 :
157 : HcclEnvelope envelope(
158 0 : request->transportRequest.protocol, request->transportRequest.transData, request->transportRequest.epParam,
159 0 : lkey, request->transportRequest.msn);
160 :
161 : // 如果建链未完成，或者积压的信封未发送完成，则Isend不进行信封发送。
162 : // Test接口中推动积压信封发送完成后，Isend接口才启动信封发送。
163 0 : std::unique_lock<std::mutex> lock(envelopeBacklogQueueLock_);
164 0 : if (GetState() != ConnState::CONN_STATE_COMPLETE) {
165 0 : envelopeBacklogQueue_.push(envelope);
166 0 : return HCCL_SUCCESS;
167 : }
168 :
169 0 : return SendEnvelope(envelope);
170 0 : }
171 :
172 0 : HcclResult TransportHeterogRoce::Send(
173 : [[maybe_unused]] const TransData& sendData, [[maybe_unused]] const TransportEndPointParam& epParam)
174 : {
175 0 : HCCL_ERROR("TransportHeterogRoce::Send is not supported.");
176 0 : return HCCL_E_NOT_SUPPORT;
177 : }
178 :
179 0 : HcclResult TransportHeterogRoce::Improbe(
180 : const TransportEndPointParam& epParam, s32& matched, HcclMessageInfo*& msg, HcclStatus& status,
181 : [[maybe_unused]] bool& flag)
182 : {
183 0 : return Improbe(epParam, matched, msg, status);
184 : }
185 :
186 0 : HcclResult TransportHeterogRoce::Improbe(
187 : [[maybe_unused]] const TransportEndPointParam& epParam, s32& matched, HcclMessageInfo*& msg, HcclStatus& status)
188 : {
189 : // 建链未完成时，返回未匹配到
190 0 : if (GetState() != ConnState::CONN_STATE_COMPLETE) {
191 0 : CHK_RET(ConnectAsync());
192 0 : return ProbeNothing(matched, msg, status);
193 : }
194 :
195 : // 先检查本地能否匹配
196 0 : HcclEnvelopeSummary envelopInfo;
197 0 : bool envelopeExist = GetSavedEnvelope(envelopInfo);
198 :
199 0 : auto probeSomething = [&]() -> HcclResult {
200 0 : CHK_RET(GenerateRecvMessage(envelopInfo, msg, status));
201 0 : matched = HCCL_IMPROBE_COMPLETED;
202 0 : return HCCL_SUCCESS;
203 0 : };
204 :
205 0 : if (envelopeExist) {
206 0 : return probeSomething();
207 : }
208 : // 再拉取roce cqe，检查是否能匹配
209 0 : CHK_RET(PullRecvRequestStatus());
210 :
211 0 : envelopeExist = GetSavedEnvelope(envelopInfo);
212 0 : if (envelopeExist) {
213 0 : return probeSomething();
214 : } else {
215 0 : return ProbeNothing(matched, msg, status);
216 : }
217 : }
218 :
219 : HcclResult
220 0 : TransportHeterogRoce::Iwrite(const TransData& sendData, const HcclEnvelope& envelope, HcclRequestInfo*& request)
221 : {
222 0 : if (isHdcMode_ && dataQpInfo_.qpMode != NORMAL_QP_MODE) {
223 0 : CHK_RET(TransportHeterog::WaitBuildLinkComplete());
224 : }
225 :
226 0 : TransportEndPointParam epParam{};
227 0 : CHK_RET(GenerateSendRequest(sendData, epParam, request));
228 0 : request->transportRequest.requestType = HcclRequestType::HCCL_REQUEST_RECV;
229 :
230 0 : u32 lkey = 0;
231 0 : CHK_RET(RegMr(
232 : reinterpret_cast<void*>(sendData.srcBuf), static_cast<u64>(sendData.count * SIZE_TABLE[sendData.dataType]),
233 : lkey, false));
234 :
235 0 : bool tmp = true;
236 0 : CHK_RET(GetQpStatus(tmp));
237 :
238 0 : if (!isHdcMode_ || dataQpInfo_.qpMode == NORMAL_QP_MODE) {
239 0 : dataWriteSge_.addr = static_cast<uint64_t>(sendData.srcBuf);
240 0 : dataWriteSge_.length = envelope.transData.count * SIZE_TABLE[envelope.transData.dataType];
241 0 : dataWriteSge_.lkey = lkey;
242 0 : dataWriteWr_.wr_id = reinterpret_cast<uint64_t>(request);
243 0 : dataWriteWr_.wr.rdma.remote_addr = static_cast<uint64_t>(envelope.transData.dstBuf);
244 0 : dataWriteWr_.wr.rdma.rkey = envelope.key;
245 :
246 0 : struct ibv_send_wr* badWr = nullptr;
247 0 : HCCL_INFO(
248 : "rdma write: remote addr[%llu] count[%d] datatype[%s] wrId[%llu]",
249 : reinterpret_cast<u64>(envelope.transData.dstBuf), envelope.transData.count,
250 : GetDataTypeEnumStr(envelope.transData.dataType).c_str(), dataWriteWr_.wr_id);
251 0 : CHK_RET(hrtIbvPostSend(dataQpInfo_.qp, &dataWriteWr_, &badWr));
252 : // 写notify
253 0 : if (deviceLogicId_ == HOST_DEVICE_ID) {
254 0 : Stream tmpStream(nullptr);
255 0 : CHK_RET(RecordNotify(tmpStream, RdmaNotifyOp::SEND_NOTIFY, dataWriteWr_.wr_id));
256 0 : }
257 0 : } else {
258 0 : struct SgList list = {};
259 0 : u64 srcBufDevAddr = 0;
260 0 : CHK_RET(dataQpMrManager_->GetDevVirAddr(
261 : reinterpret_cast<void*>(sendData.srcBuf),
262 : static_cast<u64>(envelope.transData.count * SIZE_TABLE[envelope.transData.dataType]), srcBufDevAddr));
263 :
264 0 : list.addr = srcBufDevAddr;
265 0 : list.len = envelope.transData.count * SIZE_TABLE[envelope.transData.dataType];
266 0 : list.lkey = lkey;
267 :
268 0 : struct SendWrV2 wr {};
269 0 : wr.wrId = reinterpret_cast<uint64_t>(request);
270 0 : HCCL_INFO("iwrite wr.wrId[%llu]", wr.wrId);
271 0 : wr.bufList = &list;
272 0 : wr.bufNum = 1;
273 0 : wr.dstAddr = static_cast<uint64_t>(envelope.transData.dstBuf);
274 0 : wr.rkey = envelope.key;
275 0 : wr.op = static_cast<u32>(RdmaOp::OP_WRITE);
276 0 : wr.sendFlag = RA_SEND_FENCE;
277 0 : struct SendWrRsp opRsp = {};
278 0 : CHK_RET(HrtRaSendWrV2(dataQpInfo_.qpHandle, &wr, &opRsp, GetWorkflowMode()));
279 0 : CHK_RET(DoorBellSend(dataQpInfo_.qpMode, opRsp));
280 :
281 : // 写notify
282 0 : if (deviceLogicId_ == HOST_DEVICE_ID) {
283 0 : Stream tmpStream(nullptr);
284 0 : CHK_RET(RecordNotify(tmpStream, RdmaNotifyOp::SEND_NOTIFY, wr.wrId));
285 0 : }
286 :
287 0 : s32 writeAndNotifyFlag = HCCL_TEST_INCOMPLETED;
288 0 : TIME_PRINT(CHK_RET(this->Wait(*request, writeAndNotifyFlag)));
289 : }
290 :
291 0 : return HCCL_SUCCESS;
292 : }
293 :
294 0 : HcclResult TransportHeterogRoce::Imrecv(
295 : const TransData& recvData, HcclMessageInfo& msg, HcclRequestInfo*& request, [[maybe_unused]] bool flag,
296 : [[maybe_unused]] bool needRecordFlag)
297 : {
298 0 : HcclResult ret = Imrecv(recvData, msg, request);
299 0 : return ret;
300 : }
301 :
302 0 : HcclResult TransportHeterogRoce::Imrecv(const TransData& recvData, HcclMessageInfo& msg, HcclRequestInfo*& request)
303 : {
304 0 : CHK_RET(GenerateRecvRequest(recvData, msg, request));
305 :
306 0 : u32 lkey = 0;
307 0 : CHK_RET(RegMr(
308 : reinterpret_cast<void*>(recvData.dstBuf), static_cast<u64>(recvData.count * SIZE_TABLE[recvData.dataType]),
309 : lkey, false));
310 :
311 0 : HcclEnvelope& envelope = msg.envelope.envelope;
312 :
313 0 : if (!isHdcMode_ || dataQpInfo_.qpMode == NORMAL_QP_MODE) {
314 0 : dataReadSge_.addr = static_cast<uint64_t>(recvData.dstBuf);
315 0 : dataReadSge_.length = envelope.transData.count * SIZE_TABLE[envelope.transData.dataType];
316 0 : dataReadSge_.lkey = lkey;
317 0 : dataReadWr_.wr_id = reinterpret_cast<uint64_t>(request);
318 0 : dataReadWr_.wr.rdma.remote_addr = static_cast<uint64_t>(envelope.transData.srcBuf);
319 0 : dataReadWr_.wr.rdma.rkey = envelope.key;
320 0 : dataReadWr_.next = nullptr;
321 :
322 0 : if (!(remoteIsHdc_ && (deviceLogicId_ == HOST_DEVICE_ID))) {
323 0 : HCCL_INFO("general server not load ack ");
324 0 : dataReadWr_.next = &dataAckWr_;
325 0 : dataAckSge_.addr = reinterpret_cast<uint64_t>(&envelope.msn);
326 0 : dataAckSge_.length = sizeof(uint64_t);
327 0 : dataAckSge_.lkey = 0;
328 0 : dataAckWr_.wr_id = 0;
329 : }
330 :
331 0 : struct ibv_send_wr* badWr = nullptr;
332 0 : HCCL_INFO(
333 : "rdma read: remote addr[%llx] count[%d] datatype[%s] wrId[%llu]",
334 : reinterpret_cast<u64>(envelope.transData.srcBuf), envelope.transData.count,
335 : GetDataTypeEnumStr(envelope.transData.dataType).c_str(), dataReadWr_.wr_id);
336 0 : CHK_RET(hrtIbvPostSend(dataQpInfo_.qp, &dataReadWr_, &badWr));
337 0 : } else {
338 0 : if (envelope.transData.count == 0) {
339 0 : request->transportRequest.transData.count = 0;
340 0 : CHK_RET(FreeRecvMessage(msg));
341 0 : return HCCL_SUCCESS;
342 : }
343 :
344 0 : struct SgList list = {};
345 0 : u64 devAddr = 0;
346 0 : CHK_RET(dataQpMrManager_->GetDevVirAddr(
347 : reinterpret_cast<void*>(recvData.dstBuf), static_cast<u64>(recvData.count * SIZE_TABLE[recvData.dataType]),
348 : devAddr));
349 0 : list.addr = static_cast<uint64_t>(devAddr);
350 0 : list.len = envelope.transData.count * SIZE_TABLE[envelope.transData.dataType];
351 0 : list.lkey = lkey;
352 :
353 0 : struct SendWrV2 wr = {};
354 0 : wr.wrId = reinterpret_cast<uint64_t>(request);
355 :
356 0 : HCCL_INFO("Imrecv wr.wrId[%llu]", wr.wrId);
357 0 : wr.bufList = &list;
358 0 : wr.bufNum = 1; /* 此处list只有一个，设置为1 */
359 0 : wr.dstAddr = static_cast<uint64_t>(envelope.transData.srcBuf);
360 0 : wr.rkey = envelope.key;
361 0 : wr.op = static_cast<u32>(RdmaOp::OP_READ);
362 0 : wr.sendFlag = RA_SEND_SIGNALED | RA_SEND_FENCE;
363 0 : struct SendWrRsp opRsp = {};
364 0 : CHK_RET(HrtRaSendWrV2(dataQpInfo_.qpHandle, &wr, &opRsp, GetWorkflowMode()));
365 0 : CHK_RET(DoorBellSend(dataQpInfo_.qpMode, opRsp));
366 :
367 0 : s32 imrecvFlag = HCCL_TEST_INCOMPLETED;
368 0 : TIME_PRINT(CHK_RET(this->Wait(*request, imrecvFlag)));
369 : }
370 :
371 0 : CHK_RET(FreeRecvMessage(msg));
372 :
373 0 : return HCCL_SUCCESS;
374 : }
375 :
376 0 : HcclResult TransportHeterogRoce::Test(HcclRequestInfo& request, s32& flag, HcclStatus& compState)
377 : {
378 0 : if (isHdcMode_ && dataQpInfo_.qpMode != NORMAL_QP_MODE) {
379 0 : flag = HCCL_TEST_COMPLETED;
380 0 : HCCL_INFO("TransportHeterogRoce QueryRequestStatus: flag [%d]", flag);
381 0 : compState.error = 0;
382 0 : CHK_RET(FreeRequest(request));
383 0 : return HCCL_SUCCESS;
384 : }
385 :
386 : // 建链未完成时，继续推进建链流程；
387 0 : if (GetState() != ConnState::CONN_STATE_COMPLETE) {
388 0 : CHK_RET(ConnectAsync());
389 : }
390 :
391 0 : CHK_RET(PullSendOrRecvStatus(request));
392 :
393 0 : return QueryRequestStatus(request, flag, compState);
394 : }
395 :
396 0 : HcclResult TransportHeterogRoce::PullSendOrRecvStatus(const HcclRequestInfo& request)
397 : {
398 0 : if ((GetState() != ConnState::CONN_STATE_COMPLETE) && (GetState() != ConnState::CONN_STATE_FLUSH_QUEUE)) {
399 0 : return HCCL_SUCCESS;
400 : }
401 :
402 0 : if (request.transportRequest.requestType == HcclRequestType::HCCL_REQUEST_SEND) {
403 0 : CHK_RET(PullSendStatus());
404 0 : } else if (request.transportRequest.requestType == HcclRequestType::HCCL_REQUEST_RECV) {
405 0 : CHK_RET(PullRecvStatus());
406 : } else {
407 0 : HCCL_ERROR("[HcclTest] requestType[%u] is invalid", request.transportRequest.requestType);
408 0 : return HCCL_E_PARA;
409 : }
410 :
411 0 : return HCCL_SUCCESS;
412 : }
413 :
414 0 : HcclResult TransportHeterogRoce::Wait(HcclRequestInfo& request, s32& flag)
415 : {
416 : // 建链未完成时，继续推进建链流程；
417 0 : if (GetState() != ConnState::CONN_STATE_COMPLETE) {
418 0 : CHK_RET(ConnectAsync());
419 : }
420 :
421 0 : auto startTime = chrono::steady_clock::now();
422 0 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
423 :
424 0 : while ((flag != HCCL_TEST_COMPLETED) && ((chrono::steady_clock::now() - startTime) < timeout)) {
425 0 : CHK_RET(PullSendOrRecvStatus(request));
426 :
427 0 : if (request.transportRequest.status >= 0) {
428 0 : flag = HCCL_TEST_COMPLETED;
429 0 : HCCL_INFO("QueryRequestStatus: flag[%d]", flag);
430 0 : return HCCL_SUCCESS;
431 : }
432 :
433 0 : SaluSleep(WAIT_SLEEP_TIME_US);
434 : }
435 :
436 0 : HCCL_ERROR("Wait Cqe timeOut[%d] s, State[%d]", GetExternalInputHcclLinkTimeOut(), GetState());
437 :
438 0 : return HCCL_E_TIMEOUT;
439 : }
440 :
441 0 : HcclResult TransportHeterogRoce::QueryRequestStatus(HcclRequestInfo& request, s32& flag, HcclStatus& compState)
442 : {
443 0 : if (request.transportRequest.status >= 0) {
444 : // 该request已完成
445 0 : flag = HCCL_TEST_COMPLETED;
446 0 : HCCL_INFO("QueryRequestStatus: flag [%d]", flag);
447 0 : compState.tag = request.transportRequest.epParam.src.tag;
448 0 : compState.srcRank = request.transportRequest.epParam.src.rank;
449 0 : compState.error = request.transportRequest.status;
450 0 : CHK_RET(FreeRequest(request));
451 : }
452 0 : return HCCL_SUCCESS;
453 : }
454 :
455 0 : HcclResult TransportHeterogRoce::PullSendStatus(bool allowNotify)
456 : {
457 0 : if (isHdcMode_ && (tagQpInfo_.qpMode == OFFLINE_QP_MODE || tagQpInfo_.qpMode == OFFLINE_QP_MODE_EXT)) {
458 0 : return HCCL_SUCCESS;
459 : }
460 :
461 : struct ibv_wc wcTagCq[HCCL_POLL_CQ_DEPTH];
462 0 : s32 tagCqNum = 0;
463 0 : CHK_RET(PollCq(tagQpInfo_, true, tagCqNum, wcTagCq));
464 0 : for (int i = 0; i < tagCqNum; i++) {
465 0 : if (wcTagCq[i].status != 0) {
466 0 : CHK_RET(ParseErrorTagSqe(wcTagCq, i));
467 0 : HCCL_ERROR("rdma poll tag sq failed, cqe status[%u]", wcTagCq[i].status);
468 0 : return HCCL_E_NETWORK;
469 : }
470 : }
471 : struct ibv_wc wcDataRq[HCCL_POLL_CQ_DEPTH];
472 0 : s32 dataRqNum = 0;
473 :
474 0 : CHK_RET(PollCq(dataQpInfo_, false, dataRqNum, wcDataRq));
475 0 : if ((dataRqNum == 0) && allowNotify) {
476 0 : CHK_RET(hrtIbvReqNotifyCq(dataQpInfo_.recvCq, 0));
477 0 : } else {
478 0 : HCCL_DEBUG("data rq: poll cq num:%d", dataRqNum);
479 0 : CHK_RET(ParseDataRqes(wcDataRq, dataRqNum));
480 : }
481 0 : return HCCL_SUCCESS;
482 : }
483 :
484 0 : HcclResult TransportHeterogRoce::PullRecvRequestStatus(bool allowNotify)
485 : {
486 : struct ibv_wc wc[HCCL_POLL_CQ_DEPTH];
487 0 : s32 num = 0;
488 0 : CHK_RET(PollCq(tagQpInfo_, false, num, wc));
489 0 : if ((num == 0) && allowNotify) {
490 0 : CHK_RET(hrtIbvReqNotifyCq(tagQpInfo_.recvCq, 0));
491 0 : } else {
492 0 : HCCL_DEBUG("tag rq: poll cq num:%d", num);
493 0 : CHK_RET(ParseTagRqes(wc, num));
494 : }
495 0 : return HCCL_SUCCESS;
496 : }
497 :
498 0 : HcclResult TransportHeterogRoce::PullRecvStatus(bool allowNotify)
499 : {
500 0 : HCCL_INFO("Pull dataQp RecvStatus");
501 : struct ibv_wc wc[HCCL_POLL_CQ_DEPTH];
502 0 : s32 num = 0;
503 0 : CHK_RET(PollCq(dataQpInfo_, true, num, wc));
504 0 : if ((num == 0) && allowNotify) {
505 0 : CHK_RET(hrtIbvReqNotifyCq(dataQpInfo_.sendCq, 0));
506 0 : } else {
507 0 : HCCL_DEBUG("data sq: poll cq num:%d", num);
508 0 : CHK_RET(ParseDataSqes(wc, num));
509 : }
510 0 : return HCCL_SUCCESS;
511 : }
512 :
513 0 : HcclResult TransportHeterogRoce::ParseTagRqes(const struct ibv_wc* wc, int num)
514 : {
515 0 : for (int i = 0; i < num; i++) {
516 0 : HCCL_INFO("rq cqe info: wrId[%llu] status[%u] opcode[%u]", wc[i].wr_id, wc[i].status, wc[i].opcode);
517 0 : CHK_PRT_RET(
518 : wc[i].status != 0,
519 : HCCL_ERROR(
520 : "rdma send failed, cqe status[%u] wrId[%llu] opcode[%u]", wc[i].status, wc[i].wr_id, wc[i].opcode),
521 : HCCL_E_INTERNAL);
522 0 : RecvWrInfo* info = reinterpret_cast<RecvWrInfo*>(wc[i].wr_id);
523 0 : CHK_PTR_NULL(info);
524 :
525 0 : TransportHeterogRoce* transportPtr = reinterpret_cast<TransportHeterogRoce*>(info->transportHandle);
526 0 : CHK_PTR_NULL(transportPtr);
527 0 : CHK_RET(transportPtr->SupplyTagRecvWqe());
528 0 : HcclEnvelope* envelope = nullptr;
529 0 : if (useDevMem_ && (deviceLogicId_ == HOST_DEVICE_ID)) {
530 : #ifndef CCL_KERNEL
531 : // 根据device内存求host内存
532 : CHK_RET(hrtSetDevice(index_));
533 : u64 uDevPtr = reinterpret_cast<u64>(info->buf);
534 : void* devPtr = reinterpret_cast<void*>(uDevPtr);
535 : u64 uHostPtr = uDevPtr - reinterpret_cast<uint64_t>(deviceEvePtr_) + hostAddrBegin_;
536 : void* hostPtr = reinterpret_cast<void*>(uHostPtr);
537 : HCCL_DEBUG("ParseTagRqes devPtr[%p][%llu] hostPtr[%p][%llu]", devPtr, uDevPtr, hostPtr, uHostPtr);
538 : CHK_RET(hrtMemcpy(
539 : hostPtr, MEM_BLOCK_SIZE, devPtr, MEM_BLOCK_SIZE, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_HOST));
540 : envelope = reinterpret_cast<HcclEnvelope*>(hostPtr);
541 : // ps侧DbSend对当前线程SetDevice后，改变了原GE通信域初始化时setdevice 0
542 : // 若后面使用本线程save会导致getctx失败获取不到通信域句柄，所以需要在此处重新set回默认
543 : CHK_RET(hrtSetDevice(0));
544 : #endif
545 : } else {
546 0 : envelope = reinterpret_cast<HcclEnvelope*>(info->buf);
547 : }
548 0 : CHK_PTR_NULL(envelope);
549 :
550 0 : HCCL_INFO(
551 : "recv request: tag:%d srcRank:%u dstRank:%u status:%u msn:0x%016llx count:%d", envelope->epParam.src.tag,
552 : envelope->epParam.src.rank, envelope->epParam.dst.rank, wc[i].status, envelope->msn,
553 : envelope->transData.count);
554 0 : HcclEnvelopeSummary envelopSummary(*envelope, wc[i].status);
555 0 : transportPtr->SaveEnvelope(envelopSummary);
556 0 : CHK_RET(transportPtr->FreeMemBlock(envelope));
557 0 : CHK_RET(transportPtr->FreeRecvWrId(wc[i].wr_id));
558 : }
559 0 : return HCCL_SUCCESS;
560 : }
561 :
562 0 : void TransportHeterogRoce::SaveEnvelope(HcclEnvelopeSummary& envelope)
563 : {
564 0 : unique_lock<mutex> lock(envelopeQueMutex_);
565 0 : envelopeQue_.push(envelope);
566 0 : }
567 :
568 0 : bool TransportHeterogRoce::GetSavedEnvelope(HcclEnvelopeSummary& envelope)
569 : {
570 0 : unique_lock<mutex> lock(envelopeQueMutex_);
571 0 : if (envelopeQue_.empty()) {
572 0 : return false;
573 : }
574 0 : envelope = envelopeQue_.front();
575 0 : envelopeQue_.pop();
576 0 : return true;
577 0 : }
578 :
579 0 : HcclResult TransportHeterogRoce::ParseErrorTagSqe(const struct ibv_wc* wc, int index)
580 : {
581 : // wr_id内容即信封中的msn
582 0 : HcclRequestInfo* wrPtr = reinterpret_cast<HcclRequestInfo*>(wc[index].wr_id);
583 0 : CHK_PTR_NULL(wrPtr);
584 0 : wrPtr->transportRequest.status = wc[index].status;
585 0 : TransportHeterogRoce* transportPtr = reinterpret_cast<TransportHeterogRoce*>(wrPtr->transportHandle);
586 0 : CHK_PTR_NULL(transportPtr);
587 :
588 0 : HCCL_INFO(
589 : "exception send msg: tag:%d srcRank:%u dstRank:%u status:%d msn:0x%016llx request:%p",
590 : wrPtr->transportRequest.epParam.src.tag, wrPtr->transportRequest.epParam.src.rank,
591 : wrPtr->transportRequest.epParam.dst.rank, wrPtr->transportRequest.status, wrPtr->transportRequest.msn, wrPtr);
592 :
593 0 : CHK_RET(transportPtr->DeregMr(
594 : reinterpret_cast<void*>(wrPtr->transportRequest.transData.srcBuf),
595 : static_cast<u64>(
596 : wrPtr->transportRequest.transData.count * SIZE_TABLE[wrPtr->transportRequest.transData.dataType])));
597 0 : return HCCL_SUCCESS;
598 : }
599 :
600 0 : HcclResult TransportHeterogRoce::ParseDataRqes(const struct ibv_wc* wc, int num)
601 : {
602 0 : for (int i = 0; i < num; i++) {
603 0 : HCCL_INFO("rq cqe info: wrId[%llu] status[%u] opcode[%u]", wc[i].wr_id, wc[i].status, wc[i].opcode);
604 0 : CHK_PRT_RET(
605 : wc[i].status != 0,
606 : HCCL_ERROR(
607 : "rdma poll data rq failed, cqe status[%u] wrId[%llu] opcode[%u]", wc[i].status, wc[i].wr_id,
608 : wc[i].opcode),
609 : HCCL_E_NETWORK);
610 0 : RecvWrInfo* info = reinterpret_cast<RecvWrInfo*>(wc[i].wr_id);
611 0 : CHK_PTR_NULL(info);
612 0 : HcclRequestInfo* wrPtr = reinterpret_cast<HcclRequestInfo*>(*reinterpret_cast<u64*>(info->buf));
613 0 : CHK_PRT_RET(
614 : wrPtr == nullptr, HCCL_ERROR("wrId[%llu] status[%u] opcode[%u]", wc[i].wr_id, wc[i].status, wc[i].opcode),
615 : HCCL_E_PTR);
616 0 : TransportHeterogRoce* transportPtr = reinterpret_cast<TransportHeterogRoce*>(wrPtr->transportHandle);
617 0 : CHK_PRT_RET(
618 : transportPtr == nullptr,
619 : HCCL_ERROR(
620 : "wrId[%llu] opcode[%u] tag:%d peerRank:%u status:%d msn:0x%016llx request:%p", wc[i].wr_id,
621 : wc[i].opcode, wrPtr->transportRequest.epParam.src.tag, wrPtr->transportRequest.epParam.src.rank,
622 : wrPtr->transportRequest.status, wrPtr->transportRequest.msn, wrPtr),
623 : HCCL_E_PTR);
624 0 : CHK_RET(transportPtr->SupplyDataRecvWqe());
625 0 : wrPtr->transportRequest.status = wc[i].status;
626 :
627 0 : CHK_RET(transportPtr->DeregMr(
628 : reinterpret_cast<void*>(wrPtr->transportRequest.transData.srcBuf),
629 : static_cast<u64>(
630 : wrPtr->transportRequest.transData.count * SIZE_TABLE[wrPtr->transportRequest.transData.dataType]),
631 : false));
632 0 : CHK_RET(transportPtr->FreeMemBlock(info->buf));
633 0 : CHK_RET(transportPtr->FreeRecvWrId(wc[i].wr_id));
634 0 : HCCL_INFO(
635 : "send completion: tag:%d peerRank:%u status:%d msn:0x%016llx request:%p",
636 : wrPtr->transportRequest.epParam.src.tag, wrPtr->transportRequest.epParam.src.rank,
637 : wrPtr->transportRequest.status, wrPtr->transportRequest.msn, wrPtr);
638 : }
639 0 : return HCCL_SUCCESS;
640 : }
641 :
642 0 : HcclResult TransportHeterogRoce::ParseDataSqes(const struct ibv_wc* wc, int num)
643 : {
644 0 : for (int i = 0; i < num; i++) {
645 0 : HCCL_INFO("sq cqe info: wrId[%llu] status[%u] opcode[%u]", wc[i].wr_id, wc[i].status, wc[i].opcode);
646 0 : CHK_PRT_RET(
647 : wc[i].status != 0,
648 : HCCL_ERROR(
649 : "rdma poll data sq failed, cqe status[%u] wrId[%llu] opcode[%u]", wc[i].status, wc[i].wr_id,
650 : wc[i].opcode),
651 : HCCL_E_NETWORK);
652 0 : HcclRequestInfo* wrPtr = reinterpret_cast<HcclRequestInfo*>(wc[i].wr_id);
653 :
654 0 : CHK_PRT_RET(
655 : wrPtr == nullptr, HCCL_ERROR("wrId[%llu] status[%u] opcode[%u]", wc[i].wr_id, wc[i].status, wc[i].opcode),
656 : HCCL_E_PTR);
657 0 : TransportHeterogRoce* transportPtr = reinterpret_cast<TransportHeterogRoce*>(wrPtr->transportHandle);
658 0 : CHK_PRT_RET(
659 : transportPtr == nullptr,
660 : HCCL_ERROR(
661 : "wrId[%llu] opcode[%u] tag:%d peerRank:%u status:%d msn:0x%016llx request:%p", wc[i].wr_id,
662 : wc[i].opcode, wrPtr->transportRequest.epParam.src.tag, wrPtr->transportRequest.epParam.src.rank,
663 : wrPtr->transportRequest.status, wrPtr->transportRequest.msn, wrPtr),
664 : HCCL_E_PTR);
665 0 : wrPtr->transportRequest.status = wc[i].status;
666 :
667 0 : if (!isHdcMode_ && !(remoteIsHdc_ && (deviceLogicId_ == HOST_DEVICE_ID))) {
668 0 : CHK_RET(transportPtr->DeregMr(
669 : reinterpret_cast<void*>(wrPtr->transportRequest.transData.dstBuf),
670 : static_cast<u64>(
671 : wrPtr->transportRequest.transData.count * SIZE_TABLE[wrPtr->transportRequest.transData.dataType]),
672 : false));
673 : }
674 0 : HCCL_INFO(
675 : "recv completion: tag:%d peerRank:%u status:%d msn:0x%016llx", wrPtr->transportRequest.epParam.src.tag,
676 : wrPtr->transportRequest.epParam.src.rank, wrPtr->transportRequest.status, wrPtr->transportRequest.msn);
677 : }
678 0 : return HCCL_SUCCESS;
679 : }
680 :
681 0 : HcclResult TransportHeterogRoce::SendEnvelope(HcclEnvelope& envelopInfo, void* stream)
682 : {
683 0 : if (isHdcMode_ && tagQpInfo_.qpMode != NORMAL_QP_MODE) {
684 0 : CHK_RET(TransportHeterog::WaitBuildLinkComplete());
685 : }
686 :
687 0 : if (!isHdcMode_ || tagQpInfo_.qpMode == NORMAL_QP_MODE) {
688 0 : CHK_RET(SendFlowControl());
689 : }
690 :
691 0 : if (!isHdcMode_ || tagQpInfo_.qpMode == NORMAL_QP_MODE) {
692 0 : envelopeSge_.addr = reinterpret_cast<uint64_t>(&envelopInfo);
693 0 : envelopeSge_.length = sizeof(envelopInfo);
694 0 : envelopeSge_.lkey = 0;
695 0 : envelopeWr_.wr_id = envelopInfo.msn;
696 :
697 0 : struct ibv_send_wr* badWr = nullptr;
698 0 : HCCL_INFO(
699 : "rdma send: srcRank[%u] dstRank[%u] tag[%d]: addr[%llu] count[%d] dtype[%s] msn[%llu]",
700 : envelopInfo.epParam.src.rank, envelopInfo.epParam.dst.rank, envelopInfo.epParam.src.tag,
701 : reinterpret_cast<u64>(envelopInfo.transData.srcBuf), envelopInfo.transData.count,
702 : GetDataTypeEnumStr(envelopInfo.transData.dataType).c_str(), envelopInfo.msn);
703 0 : CHK_RET(hrtIbvPostSend(tagQpInfo_.qp, &envelopeWr_, &badWr));
704 0 : } else {
705 0 : struct SgList list = {};
706 0 : list.addr = reinterpret_cast<uint64_t>(&envelopInfo);
707 0 : list.len = sizeof(envelopInfo);
708 0 : list.lkey = 0;
709 :
710 0 : struct SendWr wr = {};
711 0 : wr.bufList = &list;
712 0 : wr.bufNum = 1;
713 0 : wr.op = static_cast<u32>(RdmaOp::OP_SEND);
714 0 : wr.sendFlag = RA_SEND_SIGNALED;
715 0 : struct SendWrRsp opRsp = {};
716 0 : CHK_RET(HrtRaSendWr(tagQpInfo_.qpHandle, &wr, &opRsp));
717 0 : CHK_RET(DoorBellSend(tagQpInfo_.qpMode, opRsp, stream));
718 : }
719 :
720 0 : return HCCL_SUCCESS;
721 : }
722 :
723 0 : HcclResult TransportHeterogRoce::InitTagRecvWqe()
724 : {
725 0 : CHK_RET(IssueRecvWqe(tagQpInfo_.qp, recvWqeBatchNum_));
726 0 : tagRecvWqeNum_ = recvWqeBatchNum_;
727 0 : return HCCL_SUCCESS;
728 : }
729 :
730 0 : HcclResult TransportHeterogRoce::InitDataRecvWqe()
731 : {
732 0 : CHK_RET(IssueRecvWqe(dataQpInfo_.qp, recvWqeBatchNum_));
733 0 : dataRecvWqeNum_ = recvWqeBatchNum_;
734 0 : dataRecvWqeExpNum_ = recvWqeBatchNum_;
735 0 : return HCCL_SUCCESS;
736 : }
737 :
738 0 : HcclResult TransportHeterogRoce::SendFlowControl()
739 : {
740 0 : if (dataRecvWqeNum_ <= recvWqeBatchThreshold_) {
741 0 : CHK_RET(IssueRecvWqe(dataQpInfo_.qp, recvWqeBatchSupplement_));
742 0 : dataRecvWqeNum_ += recvWqeBatchSupplement_;
743 0 : dataRecvWqeExpNum_ += recvWqeBatchSupplement_;
744 : }
745 :
746 0 : u32 dataRecvWqeNum = dataRecvWqeNum_.load();
747 0 : u32 dataRecvWqeExpNum = dataRecvWqeExpNum_.load();
748 0 : if (dataRecvWqeNum - dataRecvWqeExpNum >= recvWqeBatchSupplement_) {
749 0 : CHK_RET(PullSendStatus());
750 0 : HCCL_RUN_INFO(
751 : "Flow control is activated, because dataRecvWqeNum[%u] - dataRecvWqeExpNum[%u] >="
752 : " recvWqeBatchSupplement[%u]",
753 : dataRecvWqeNum, dataRecvWqeExpNum, recvWqeBatchSupplement_);
754 :
755 0 : return HCCL_E_AGAIN;
756 : }
757 :
758 0 : dataRecvWqeExpNum_--;
759 0 : return HCCL_SUCCESS;
760 : }
761 :
762 0 : HcclResult TransportHeterogRoce::SupplyTagRecvWqe()
763 : {
764 0 : tagRecvWqeNum_--;
765 0 : if (tagRecvWqeNum_ <= recvWqeBatchThreshold_) {
766 0 : CHK_RET(IssueRecvWqe(tagQpInfo_.qp, recvWqeBatchSupplement_));
767 0 : tagRecvWqeNum_ += recvWqeBatchSupplement_;
768 : }
769 :
770 0 : return HCCL_SUCCESS;
771 : }
772 :
773 0 : HcclResult TransportHeterogRoce::SupplyDataRecvWqe()
774 : {
775 0 : dataRecvWqeNum_--;
776 0 : if (dataRecvWqeNum_ <= recvWqeBatchThreshold_) {
777 0 : CHK_RET(IssueRecvWqe(dataQpInfo_.qp, recvWqeBatchSupplement_));
778 0 : dataRecvWqeNum_ += recvWqeBatchSupplement_;
779 0 : dataRecvWqeExpNum_ += recvWqeBatchSupplement_;
780 : }
781 0 : return HCCL_SUCCESS;
782 : }
783 :
784 0 : HcclResult TransportHeterogRoce::IssueRecvWqe(struct ibv_qp* qp, u32 num)
785 : {
786 0 : if (isHdcMode_ && (tagQpInfo_.qpMode == OFFLINE_QP_MODE || tagQpInfo_.qpMode == OFFLINE_QP_MODE_EXT)) {
787 0 : return HCCL_SUCCESS;
788 : }
789 :
790 0 : list<void*> blockList(num, nullptr);
791 0 : CHK_RET(AllocMemBlocks(blockList));
792 :
793 0 : auto iter = blockList.begin();
794 0 : struct ibv_recv_wr* nextRqWr = nullptr;
795 0 : struct ibv_recv_wr rqWr[num];
796 0 : struct ibv_sge sgeList[num];
797 :
798 0 : std::vector<struct RecvWrlistData> recvWrVec(num);
799 0 : struct RecvWrlistData* recvWr = recvWrVec.data();
800 :
801 0 : if (!isHdcMode_ || tagQpInfo_.qpMode == NORMAL_QP_MODE) {
802 0 : for (int i = num - 1; i >= 0; i--) {
803 0 : CHK_PTR_NULL(*iter);
804 0 : u64 wrId = 0;
805 0 : CHK_RET(GenerateRecvWrId(*iter, wrId));
806 0 : rqWr[i].wr_id = wrId;
807 0 : rqWr[i].next = nextRqWr;
808 0 : rqWr[i].sg_list = &sgeList[i];
809 0 : rqWr[i].num_sge = 1;
810 0 : sgeList[i].addr = reinterpret_cast<uint64_t>(*iter);
811 0 : sgeList[i].length = MEM_BLOCK_SIZE;
812 0 : sgeList[i].lkey = blockMemLkey_;
813 0 : nextRqWr = &rqWr[i];
814 0 : iter++;
815 : }
816 :
817 0 : struct ibv_recv_wr* badRqWr = nullptr;
818 0 : CHK_RET(hrtIbvPostRecv(qp, &rqWr[0], &badRqWr));
819 0 : return HCCL_SUCCESS;
820 : } else {
821 0 : for (int i = num - 1; i >= 0; i--) {
822 0 : CHK_PTR_NULL(*iter);
823 0 : u64 wrId = 0;
824 0 : if (useDevMem_) {
825 : // 根据host内存地址计算出device内存地址
826 : u64 uDevPtr
827 0 : = reinterpret_cast<uint64_t>(*iter) - hostAddrBegin_ + reinterpret_cast<uint64_t>(deviceEvePtr_);
828 0 : CHK_RET(GenerateRecvWrId(reinterpret_cast<void*>(uDevPtr), wrId));
829 0 : recvWr[i].memList.addr = uDevPtr;
830 0 : recvWr[i].memList.lkey = deviceEveLkey_;
831 : } else {
832 0 : CHK_RET(GenerateRecvWrId(*iter, wrId));
833 0 : recvWr[i].memList.addr
834 0 : = HostAddrToDev(reinterpret_cast<uint64_t>(*iter), hostAddrBegin_, devAddrBegin_);
835 0 : recvWr[i].memList.lkey = blockMemLkey_;
836 : }
837 0 : recvWr[i].wrId = wrId;
838 0 : recvWr[i].memList.len = MEM_BLOCK_SIZE;
839 0 : iter++;
840 : }
841 : }
842 :
843 0 : u32 completeNum = 0;
844 0 : s32 ret = hrtRaRecvWrlist(tagQpInfo_.qpHandle, recvWr, num, &completeNum);
845 0 : if (ret == HCCL_SUCCESS && completeNum == num) {
846 0 : HCCL_INFO("hrtRaRecvWrlist success ");
847 0 : return HCCL_SUCCESS;
848 : } else {
849 0 : HCCL_ERROR("[Transport][RdmaData]In RdmaDataTransport, hrtRaRecvWrlist failed. ret[%d]", ret);
850 0 : return HCCL_E_NETWORK;
851 : }
852 :
853 : return HCCL_SUCCESS;
854 0 : }
855 :
856 0 : HcclResult TransportHeterogRoce::GetQpStatus(bool& completed)
857 : {
858 0 : int qpStatus = 0;
859 0 : s32 ret = 0;
860 :
861 0 : ret = hrtGetRaQpStatus(tagQpInfo_.qpHandle, &qpStatus);
862 0 : if (ret != 0) {
863 0 : HCCL_ERROR("get tag qp status fail. qpStatus[%d] ret[%d]", qpStatus, ret);
864 0 : return HCCL_E_INTERNAL;
865 0 : } else if (ret == 0 && qpStatus != 1) { // 为1时，qp 建链成功
866 0 : return HCCL_E_AGAIN;
867 : }
868 :
869 0 : ret = hrtGetRaQpStatus(dataQpInfo_.qpHandle, &qpStatus);
870 0 : if (ret != 0) {
871 0 : HCCL_ERROR("get data qp status fail. qpStatus[%d] ret[%d]", qpStatus, ret);
872 0 : return HCCL_E_INTERNAL;
873 0 : } else if (ret == 0 && qpStatus != 1) { // 为1时，qp 建链成功
874 0 : return HCCL_E_AGAIN;
875 : }
876 :
877 0 : completed = true;
878 0 : return HCCL_SUCCESS;
879 : }
880 :
881 0 : HcclResult TransportHeterogRoce::AllocMemBlocks(list<void*>& blockList)
882 : {
883 : const std::unique_ptr<HeterogMemBlocksManager>& memBlocksManagerPtr
884 0 : = (IsRamdHandleLevelMr()) ? memBlocksManager_ : tagMemBlocksManager_;
885 0 : CHK_PTR_NULL(memBlocksManagerPtr);
886 0 : CHK_RET(memBlocksManagerPtr->Alloc(blockList));
887 0 : if (isHdcMode_) {
888 0 : for (auto iter : blockList) {
889 0 : wqeBlockLists_.push_back(iter);
890 : }
891 : }
892 0 : return HCCL_SUCCESS;
893 : }
894 :
895 0 : HcclResult TransportHeterogRoce::FreeMemBlock(void* block)
896 : {
897 : const std::unique_ptr<HeterogMemBlocksManager>& memBlocksManagerPtr
898 0 : = ((IsRamdHandleLevelMr())) ? memBlocksManager_ : tagMemBlocksManager_;
899 0 : CHK_PTR_NULL(memBlocksManagerPtr);
900 0 : CHK_RET(memBlocksManagerPtr->Free(block));
901 0 : if (isHdcMode_) {
902 0 : auto iter = std::find(wqeBlockLists_.begin(), wqeBlockLists_.end(), block);
903 0 : if (iter != wqeBlockLists_.end()) {
904 0 : wqeBlockLists_.erase(iter);
905 : }
906 : }
907 0 : return HCCL_SUCCESS;
908 : }
909 :
910 0 : HcclResult TransportHeterogRoce::FreeRecvWrId(u64 wrId)
911 : {
912 0 : pRecvWrInfosMem_->Free(reinterpret_cast<RecvWrInfo*>(wrId));
913 0 : return HCCL_SUCCESS;
914 : }
915 :
916 0 : HcclResult TransportHeterogRoce::GenerateRecvWrId(void* recvBuf, u64& wrId)
917 : {
918 0 : RecvWrInfo* data = pRecvWrInfosMem_->Alloc();
919 0 : CHK_PTR_NULL(data);
920 0 : data->buf = recvBuf;
921 0 : data->transportHandle = reinterpret_cast<void*>(this);
922 0 : CHK_PTR_NULL(data->transportHandle);
923 0 : wrId = reinterpret_cast<uint64_t>(data);
924 0 : return HCCL_SUCCESS;
925 : }
926 :
927 0 : HcclResult TransportHeterogRoce::GetNetworkResource()
928 : {
929 0 : RaResourceInfo raResourceInfo;
930 0 : CHK_RET(NetworkManager::GetInstance(index_).GetRaResourceInfo(raResourceInfo));
931 0 : auto it = raResourceInfo.nicSocketMap.find(selfIp_);
932 0 : if (it == raResourceInfo.nicSocketMap.end()) {
933 0 : HCCL_ERROR("[TransportHeterogRoce][Init]nic socket handle did not found");
934 0 : return HCCL_E_PARA;
935 : }
936 0 : nicSocketHandle_ = it->second.nicSocketHandle;
937 0 : CHK_PTR_NULL(nicSocketHandle_);
938 0 : nicRdmaHandle_ = it->second.nicRdmaHandle;
939 0 : CHK_PTR_NULL(nicRdmaHandle_);
940 0 : HCCL_INFO(
941 : "TransportHeterogRoce GetNetworkResource index_[%d] nicSocketHandle_[%p] nicRdmaHandle_[%p]", index_,
942 : nicSocketHandle_, nicRdmaHandle_);
943 0 : return HCCL_SUCCESS;
944 0 : }
945 :
946 0 : HcclResult TransportHeterogRoce::PreQpConnect()
947 : {
948 : // 创建QP及CQ，多个QP可共享CQ
949 0 : CHK_RET(CreateCqAndQp());
950 :
951 0 : if (isHdcMode_) { // 不是HDC模式，走的peer，但是训练时，wqe下发情况和hdc相同
952 0 : CHK_RET(PreHdcResource());
953 : } else {
954 : // 下发post recv, 注：HCCP完成QP建链后需要两端握手确认QP状态OK后才能发起通信
955 0 : CHK_RET(InitTagRecvWqe());
956 0 : if (!(remoteIsHdc_ && (deviceLogicId_ == HOST_DEVICE_ID))) {
957 0 : CHK_RET(InitDataRecvWqe());
958 : }
959 : }
960 :
961 : // 为提高收发处理速度，提前准备post send需要的wr模板
962 0 : CHK_SAFETY_FUNC_RET(memset_s(&envelopeWr_, sizeof(struct ibv_send_wr), 0, sizeof(struct ibv_send_wr)));
963 0 : envelopeWr_.sg_list = &envelopeSge_;
964 0 : envelopeWr_.next = nullptr;
965 0 : envelopeWr_.num_sge = 1;
966 0 : envelopeWr_.opcode = IBV_WR_SEND;
967 0 : envelopeWr_.send_flags = IBV_SEND_SIGNALED | IBV_SEND_INLINE;
968 :
969 0 : CHK_SAFETY_FUNC_RET(memset_s(&dataReadWr_, sizeof(struct ibv_send_wr), 0, sizeof(struct ibv_send_wr)));
970 0 : dataReadWr_.sg_list = &dataReadSge_;
971 0 : dataReadWr_.next = nullptr;
972 0 : dataReadWr_.num_sge = 1;
973 0 : dataReadWr_.opcode = IBV_WR_RDMA_READ;
974 0 : dataReadWr_.send_flags = IBV_SEND_SIGNALED | IBV_SEND_FENCE;
975 :
976 0 : CHK_SAFETY_FUNC_RET(memset_s(&dataWriteWr_, sizeof(struct ibv_send_wr), 0, sizeof(struct ibv_send_wr)));
977 0 : dataWriteWr_.sg_list = &dataWriteSge_;
978 0 : dataWriteWr_.next = nullptr;
979 0 : dataWriteWr_.num_sge = 1;
980 0 : dataWriteWr_.opcode = IBV_WR_RDMA_WRITE;
981 0 : dataWriteWr_.send_flags = IBV_SEND_FENCE;
982 :
983 0 : CHK_SAFETY_FUNC_RET(memset_s(¬ifyWriteWr_, sizeof(struct ibv_send_wr), 0, sizeof(struct ibv_send_wr)));
984 0 : notifyWriteWr_.sg_list = ¬ifyWriteSge_;
985 0 : notifyWriteWr_.next = nullptr;
986 0 : notifyWriteWr_.num_sge = 1;
987 0 : notifyWriteWr_.opcode = IBV_WR_RDMA_WRITE;
988 0 : notifyWriteWr_.send_flags = IBV_SEND_SIGNALED | IBV_SEND_FENCE;
989 :
990 0 : CHK_SAFETY_FUNC_RET(memset_s(&dataAckWr_, sizeof(struct ibv_send_wr), 0, sizeof(struct ibv_send_wr)));
991 0 : dataAckWr_.sg_list = &dataAckSge_;
992 0 : dataAckWr_.next = nullptr;
993 0 : dataAckWr_.num_sge = 1;
994 0 : dataAckWr_.opcode = IBV_WR_SEND_WITH_IMM;
995 0 : dataAckWr_.send_flags = IBV_SEND_FENCE | IBV_SEND_INLINE;
996 :
997 0 : return HCCL_SUCCESS;
998 : }
999 :
1000 0 : HcclResult TransportHeterogRoce::CreateCqAndQp()
1001 : {
1002 0 : HCCL_INFO("TransportHeterogRoce CreateCqAndQp");
1003 0 : CHK_RET(CreateQpWithCq(nicRdmaHandle_, -1, -1, nullptr, nullptr, tagQpInfo_, isHdcMode_, isESMode_));
1004 0 : CHK_RET(CreateQpWithCq(nicRdmaHandle_, -1, -1, nullptr, nullptr, dataQpInfo_, isHdcMode_, isESMode_));
1005 0 : return HCCL_SUCCESS;
1006 : }
1007 :
1008 0 : HcclResult TransportHeterogRoce::DestroyCqAndQp()
1009 : {
1010 0 : HCCL_INFO("TransportHeterogRoce DestroyCqAndQp");
1011 0 : CHK_RET(DestroyQpWithCq(tagQpInfo_, isHdcMode_));
1012 0 : tagQpInfo_ = QpInfo();
1013 0 : CHK_RET(DestroyQpWithCq(dataQpInfo_, isHdcMode_));
1014 0 : dataQpInfo_ = QpInfo();
1015 0 : return HCCL_SUCCESS;
1016 : }
1017 :
1018 0 : HcclResult TransportHeterogRoce::QpConnect(bool& completed)
1019 : {
1020 0 : CHK_RET(HrtRaQpNonBlockConnectAsync(tagQpInfo_.qpHandle, initSM_.locInitInfo.socketInfo[0].fdHandle));
1021 0 : CHK_RET(HrtRaQpNonBlockConnectAsync(dataQpInfo_.qpHandle, initSM_.locInitInfo.socketInfo[1].fdHandle));
1022 :
1023 0 : completed = true;
1024 0 : return HCCL_SUCCESS;
1025 : }
1026 :
1027 0 : HcclResult TransportHeterogRoce::RegMr(void* mem, u64 size, u32& lkey, bool isTagQpHandle)
1028 : {
1029 0 : HCCL_DEBUG("reg mr mem[%p] size[%llu Byte]", mem, size);
1030 0 : if (size == 0) {
1031 0 : lkey = 0;
1032 0 : return HCCL_SUCCESS;
1033 : }
1034 0 : CHK_PTR_NULL(mem);
1035 :
1036 0 : if (isTagQpHandle || IsRamdHandleLevelMr()) {
1037 0 : CHK_RET(mrManager_->GetKey(mem, size, lkey));
1038 : } else {
1039 0 : CHK_RET(dataQpMrManager_->GetKey(mem, size, lkey));
1040 : }
1041 0 : return HCCL_SUCCESS;
1042 : }
1043 :
1044 0 : HcclResult TransportHeterogRoce::DeregMr(void* mem, u64 size, bool isTagQpHandle)
1045 : {
1046 0 : HCCL_DEBUG("dereg mr mem[%p] size[%llu Byte]", mem, size);
1047 0 : if (size == 0) {
1048 0 : return HCCL_SUCCESS;
1049 : }
1050 :
1051 0 : if (isTagQpHandle || IsRamdHandleLevelMr()) {
1052 0 : CHK_RET(mrManager_->ReleaseKey(mem, size));
1053 : } else {
1054 0 : CHK_RET(dataQpMrManager_->ReleaseKey(mem, size));
1055 : }
1056 0 : return HCCL_SUCCESS;
1057 : }
1058 :
1059 0 : HcclResult TransportHeterogRoce::RoceConnectSocket(SocketConnectInfoT conn[], u32 num, bool& completed)
1060 : {
1061 0 : if (initSM_.locInitInfo.role == CLIENT_ROLE_SOCKET) {
1062 0 : return ConnectSocket(conn, num, completed);
1063 : } else {
1064 0 : completed = true;
1065 0 : return HCCL_SUCCESS;
1066 : }
1067 : }
1068 :
1069 0 : HcclResult TransportHeterogRoce::FlushSendQueue(bool& completed)
1070 : {
1071 0 : if (envelopeBacklogQueue_.size() > 0) {
1072 0 : HcclEnvelope tmpEnvelopeInfo;
1073 0 : while (!envelopeBacklogQueue_.empty()) {
1074 0 : tmpEnvelopeInfo = envelopeBacklogQueue_.front();
1075 0 : CHK_RET(SendEnvelope(tmpEnvelopeInfo));
1076 0 : envelopeBacklogQueue_.pop();
1077 : }
1078 : }
1079 0 : completed = true;
1080 0 : return HCCL_SUCCESS;
1081 : }
1082 :
1083 0 : HcclResult TransportHeterogRoce::EnterStateProcess(ConnState nextState)
1084 : {
1085 0 : switch (nextState) {
1086 0 : case ConnState::CONN_STATE_CONNECT_CHECK_SOCKET:
1087 0 : initSM_.socketNum = 1;
1088 0 : break;
1089 0 : case ConnState::CONN_STATE_GET_CHECK_SOCKET:
1090 0 : initSM_.socketNum = 1;
1091 0 : initSM_.completeNum = 0;
1092 0 : break;
1093 0 : case ConnState::CONN_STATE_SEND_CF:
1094 : case ConnState::CONN_STATE_RECV_CF:
1095 0 : initSM_.size = HETEROG_MAX_FRAME_LEN;
1096 0 : initSM_.completeSize = 0;
1097 0 : break;
1098 0 : case ConnState::CONN_STATE_CHECK_CF:
1099 0 : CHK_RET(CheckConsistentFrame());
1100 0 : CHK_RET(TryTransition(HCCL_SUCCESS, true, ConnState::CONN_STATE_CONNECT_ALL_SOCKET));
1101 0 : break;
1102 0 : case ConnState::CONN_STATE_CONNECT_ALL_SOCKET:
1103 0 : initSM_.socketNum = initSM_.locInitInfo.socketConnInfo.size() - 1;
1104 0 : break;
1105 0 : case ConnState::CONN_STATE_GET_ALL_SOCKET:
1106 0 : initSM_.socketNum = initSM_.locInitInfo.socketInfo.size() - 1;
1107 0 : initSM_.completeNum = 0;
1108 0 : break;
1109 0 : case ConnState::CONN_STATE_SEND_STATUS:
1110 0 : initSM_.size = sizeof(initSM_.locInitInfo.signal);
1111 0 : initSM_.completeSize = 0;
1112 0 : break;
1113 0 : case ConnState::CONN_STATE_RECV_STATUS:
1114 0 : initSM_.size = sizeof(initSM_.remInitInfo.signal);
1115 0 : initSM_.completeSize = 0;
1116 0 : break;
1117 0 : case ConnState::CONN_STATE_COMPLETE:
1118 0 : HCCL_INFO("link[%s]: connect complete", initSM_.locInitInfo.socketInfo[0].tag);
1119 0 : break;
1120 0 : default:
1121 0 : HCCL_INFO("link[%s]: state[%u] no need to do anything", initSM_.locInitInfo.socketInfo[0].tag, nextState);
1122 : }
1123 :
1124 0 : return HCCL_SUCCESS;
1125 : }
1126 : // 需要循环检查的状态
1127 0 : HcclResult TransportHeterogRoce::LoopStateProcess()
1128 : {
1129 0 : HcclResult testRet = HCCL_SUCCESS;
1130 0 : bool completed = false;
1131 0 : switch (GetState()) {
1132 0 : case ConnState::CONN_STATE_CONNECT_CHECK_SOCKET:
1133 0 : testRet = RoceConnectSocket(initSM_.locInitInfo.socketConnInfo.data(), initSM_.socketNum, completed);
1134 0 : CHK_RET(TryTransition(testRet, completed, ConnState::CONN_STATE_GET_CHECK_SOCKET));
1135 0 : break;
1136 0 : case ConnState::CONN_STATE_GET_CHECK_SOCKET:
1137 0 : testRet = GetSocket(
1138 0 : initSM_.locInitInfo.role, initSM_.locInitInfo.socketInfo.data(), initSM_.socketNum, initSM_.completeNum,
1139 : completed);
1140 0 : CHK_RET(TryTransition(testRet, completed, ConnState::CONN_STATE_SEND_CF));
1141 0 : break;
1142 0 : case ConnState::CONN_STATE_SEND_CF:
1143 0 : testRet = SocketSend(
1144 0 : initSM_.locInitInfo.socketInfo[0].fdHandle, initSM_.locInitInfo.checkFrame, initSM_.size,
1145 0 : initSM_.completeSize, completed);
1146 0 : CHK_RET(TryTransition(testRet, completed, ConnState::CONN_STATE_RECV_CF));
1147 0 : break;
1148 0 : case ConnState::CONN_STATE_RECV_CF:
1149 0 : testRet = SocketRecv(
1150 0 : initSM_.locInitInfo.socketInfo[0].fdHandle, initSM_.remInitInfo.checkFrame, initSM_.size,
1151 0 : initSM_.completeSize, completed);
1152 0 : CHK_RET(TryTransition(testRet, completed, ConnState::CONN_STATE_CHECK_CF));
1153 0 : break;
1154 0 : case ConnState::CONN_STATE_CONNECT_ALL_SOCKET:
1155 0 : testRet = RoceConnectSocket(
1156 0 : reinterpret_cast<SocketConnectInfoT*>(initSM_.locInitInfo.socketConnInfo.data()) + 1, initSM_.socketNum,
1157 : completed);
1158 0 : CHK_RET(TryTransition(testRet, completed, ConnState::CONN_STATE_GET_ALL_SOCKET));
1159 0 : break;
1160 0 : case ConnState::CONN_STATE_GET_ALL_SOCKET:
1161 0 : testRet = GetSocket(
1162 : initSM_.locInitInfo.role,
1163 0 : reinterpret_cast<struct SocketInfoT*>(initSM_.locInitInfo.socketInfo.data()) + 1, initSM_.socketNum,
1164 0 : initSM_.completeNum, completed);
1165 0 : testRet = ((testRet == HCCL_SUCCESS) && completed) ? CreatSignalMesg() : testRet;
1166 0 : CHK_RET(TryTransition(testRet, completed, ConnState::CONN_STATE_CONNECT_QP));
1167 0 : break;
1168 0 : case ConnState::CONN_STATE_CONNECT_QP:
1169 0 : testRet = QpConnect(completed);
1170 0 : CHK_RET(TryTransition(testRet, completed, ConnState::CONN_STATE_GET_QP));
1171 0 : break;
1172 0 : case ConnState::CONN_STATE_GET_QP:
1173 0 : testRet = GetQpStatus(completed);
1174 0 : CHK_RET(TryTransition(testRet, completed, ConnState::CONN_STATE_SEND_STATUS));
1175 0 : break;
1176 0 : case ConnState::CONN_STATE_SEND_STATUS:
1177 0 : testRet = SocketSend(
1178 0 : initSM_.locInitInfo.socketInfo[SOCKET_FOR_SENDRECV_QP].fdHandle, &(initSM_.locInitInfo.signal),
1179 0 : initSM_.size, initSM_.completeSize, completed);
1180 0 : CHK_RET(TryTransition(testRet, completed, ConnState::CONN_STATE_RECV_STATUS));
1181 0 : break;
1182 0 : case ConnState::CONN_STATE_RECV_STATUS:
1183 0 : testRet = SocketRecv(
1184 0 : initSM_.locInitInfo.socketInfo[SOCKET_FOR_SENDRECV_QP].fdHandle, &(initSM_.remInitInfo.signal),
1185 0 : initSM_.size, initSM_.completeSize, completed);
1186 0 : fdHandle_ = initSM_.locInitInfo.socketInfo[SOCKET_FOR_SENDRECV_QP].fdHandle;
1187 0 : testRet = ((testRet == HCCL_SUCCESS) && completed && !isRawConn_) ? ExchangeSignalMesg() : testRet;
1188 0 : CHK_RET(TryTransition(testRet, completed, ConnState::CONN_STATE_FLUSH_QUEUE));
1189 0 : break;
1190 0 : case ConnState::CONN_STATE_FLUSH_QUEUE: {
1191 : // 为防止Isend中积压信封入队和TestSome中flush积压信封队列并发问题，
1192 : // 该处flush积压信封队列并状态迁移完成后，再解锁。
1193 0 : std::unique_lock<std::mutex> lock(envelopeBacklogQueueLock_);
1194 0 : testRet = FlushSendQueue(completed);
1195 0 : CHK_RET(TryTransition(testRet, completed, ConnState::CONN_STATE_COMPLETE));
1196 0 : break;
1197 0 : }
1198 0 : default:
1199 0 : HCCL_ERROR(
1200 : "Establish communication connection failed[%s]: state[%u]", initSM_.locInitInfo.socketInfo[0].tag,
1201 : GetState());
1202 0 : return HCCL_E_INTERNAL;
1203 : }
1204 0 : return HCCL_SUCCESS;
1205 : }
1206 :
1207 0 : HcclResult TransportHeterogRoce::GetSocketInfos(std::vector<std::vector<HcclSocketInfo>>& socketInfos)
1208 : {
1209 0 : std::vector<HcclSocketInfo> hcclSocketInfo;
1210 0 : for (SocketInfoT raSocketInfo : initSM_.locInitInfo.socketInfo) {
1211 0 : hcclSocketInfo.push_back({raSocketInfo.socketHandle, raSocketInfo.fdHandle});
1212 : }
1213 0 : socketInfos.push_back(hcclSocketInfo);
1214 0 : return HCCL_SUCCESS;
1215 0 : }
1216 :
1217 0 : void TransportHeterogRoce::GetTransportResourceInfo(const TransportResourceInfo& transportResourceInfo)
1218 : {
1219 0 : tagQpInfo_.flag = transportResourceInfo.flag;
1220 0 : tagQpInfo_.qpMode = transportResourceInfo.qpMode;
1221 0 : dataQpInfo_.flag = transportResourceInfo.flag;
1222 0 : dataQpInfo_.qpMode = transportResourceInfo.qpMode;
1223 0 : isHdcMode_ = transportResourceInfo.isHdcMode;
1224 0 : deviceLogicId_ = transportResourceInfo.deviceLogicId;
1225 0 : memBlockNum_ = transportResourceInfo.memBlockNum;
1226 0 : remoteIsHdc_ = transportResourceInfo.remoteIsHdc;
1227 0 : isESMode_ = transportResourceInfo.isESMode;
1228 0 : isGlobalMrmanagerInit_ = transportResourceInfo.isGlobalMrmanagerInit;
1229 0 : hdcHostWqeBatchNum_ = transportResourceInfo.hdcHostWqeBatchNum;
1230 0 : HCCL_INFO(
1231 : "tagQpInfo_.flag[%d] tagQpInfo_.qpMode[%d] dataQpInfo_.flag[%d] dataQpInfo_.qpMode[%d] isHdcMode_[%d] "
1232 : "deviceLogicId_[%d] memBlockNum_[%u] remoteIsHdc_[%d] isESMode_[%d] isGlobalMrmanagerInit_[%d] "
1233 : "hdcHostWqeBatchNum_[%u]",
1234 : tagQpInfo_.flag, tagQpInfo_.qpMode, dataQpInfo_.flag, dataQpInfo_.qpMode, isHdcMode_, deviceLogicId_,
1235 : memBlockNum_, remoteIsHdc_, isESMode_, isGlobalMrmanagerInit_, hdcHostWqeBatchNum_);
1236 0 : }
1237 :
1238 0 : HcclResult TransportHeterogRoce::PollCq(QpInfo& qpInfo, bool isSend, s32& num, struct ibv_wc* wc)
1239 : {
1240 0 : if (!isHdcMode_ || tagQpInfo_.qpMode == NORMAL_QP_MODE) {
1241 0 : if (isSend) {
1242 0 : CHK_RET(hrtIbvPollCq(qpInfo.sendCq, HCCL_POLL_CQ_DEPTH, wc, num));
1243 : } else {
1244 0 : CHK_RET(hrtIbvPollCq(qpInfo.recvCq, HCCL_POLL_CQ_DEPTH, wc, num));
1245 : }
1246 0 : } else {
1247 0 : s32 ret = hrtRaPollCq(qpInfo.qpHandle, isSend, HCCL_POLL_CQ_ONETIME, wc);
1248 0 : if (ret >= 0 && static_cast<u32>(ret) <= HCCL_POLL_CQ_ONETIME) {
1249 0 : num = ret;
1250 : } else {
1251 0 : HCCL_ERROR("call trace: hcclRet -> %d", ret);
1252 0 : return HCCL_E_REMOTE;
1253 : }
1254 : }
1255 0 : return HCCL_SUCCESS;
1256 : }
1257 :
1258 0 : HcclResult TransportHeterogRoce::GetRemoteIsendDoneSignal(std::shared_ptr<LocalIpcNotify>& signal)
1259 : {
1260 0 : signal = remoteIsendDoneSignal_;
1261 0 : CHK_SMART_PTR_NULL(signal);
1262 0 : return HCCL_SUCCESS;
1263 : }
1264 :
1265 0 : HcclResult TransportHeterogRoce::GetRemoteImrecvDoneSignal(std::shared_ptr<LocalIpcNotify>& signal)
1266 : {
1267 0 : signal = remoteImrecvDoneSignal_;
1268 0 : CHK_SMART_PTR_NULL(signal);
1269 0 : return HCCL_SUCCESS;
1270 : }
1271 :
1272 0 : HcclResult TransportHeterogRoce::GetNotifySize()
1273 : {
1274 : DevType devType;
1275 0 : CHK_RET(hrtHalGetDeviceType(index_, devType));
1276 :
1277 0 : if (devType == DevType::DEV_TYPE_910) {
1278 0 : notifySize_ = 8; // 910A 每个notify占8个字节
1279 0 : } else if ((devType == DevType::DEV_TYPE_910B) || (devType == DevType::DEV_TYPE_910_93)) {
1280 0 : notifySize_ = 4; // 910B/910_93 每个notify占4个字节
1281 : } else {
1282 0 : notifySize_ = 8; // 其余芯片类型每个notify占8个字节
1283 : }
1284 0 : HCCL_INFO("devType[%d] notifySize[%d]", devType, notifySize_);
1285 0 : return HCCL_SUCCESS;
1286 : }
1287 :
1288 0 : HcclResult TransportHeterogRoce::CreateRdmaSignal(
1289 : std::shared_ptr<LocalIpcNotify>& localNotify, HcclRdmaSignalInfo& rdmaSignalInfo, MemType notifyType)
1290 : {
1291 0 : EXCEPTION_CATCH((localNotify = std::make_shared<LocalIpcNotify>()), return HCCL_E_PTR);
1292 0 : CHK_SMART_PTR_NULL(localNotify);
1293 0 : s32 pid = 0;
1294 0 : CHK_RET(SalGetBareTgid(&pid)); // 当前进程id
1295 0 : CHK_RET(localNotify->Init(deviceLogicId_, deviceLogicId_));
1296 0 : s64 recvId = 0xFFFFFFFF00000000 | (static_cast<s64>(pid) & 0xFFFFFFFF);
1297 0 : CHK_RET(localNotify->Grant(recvId));
1298 :
1299 0 : u64 notifyOffset = 0;
1300 0 : u64 notifyBaseVa = 0; // notify寄存器虚拟地址
1301 0 : u64 notifyTotalSize = 0;
1302 0 : CHK_RET(HrtRaGetNotifyBaseAddr(nicRdmaHandle_, ¬ifyBaseVa, ¬ifyTotalSize));
1303 0 : CHK_RET(localNotify->GetNotifyOffset(notifyOffset));
1304 0 : u64 notifyVa = notifyBaseVa + notifyOffset;
1305 :
1306 0 : rdmaSignalInfo.mrRegFlag = 0;
1307 0 : rdmaSignalInfo.notifyAddr = reinterpret_cast<void*>(notifyVa);
1308 0 : rdmaSignalInfo.len = notifySize_;
1309 0 : rdmaSignalInfo.type = notifyType;
1310 :
1311 0 : struct MrInfoT mrInfo = {};
1312 0 : mrInfo.addr = rdmaSignalInfo.notifyAddr;
1313 0 : mrInfo.size = rdmaSignalInfo.len;
1314 0 : mrInfo.access = access_;
1315 0 : CHK_RET(HrtRaMrReg(dataQpInfo_.qpHandle, &mrInfo));
1316 0 : rdmaSignalInfo.lkey = mrInfo.lkey;
1317 0 : return HCCL_SUCCESS;
1318 : }
1319 :
1320 0 : HcclResult TransportHeterogRoce::PsRdmaDbSend(uint32_t dbindex, uint64_t dbinfo, rtStream_t stream)
1321 : {
1322 0 : CHK_RET(hrtSetDevice(index_));
1323 0 : s32 ret = hrtRDMADBSend(dbindex, dbinfo, stream);
1324 0 : CHK_PRT_RET(
1325 : ret != RT_ERROR_NONE,
1326 : HCCL_ERROR(
1327 : "[rtRDMADBSend]errNo[0x%016llx] rt rdma send fail, "
1328 : "return[%d]. para: dbindex[%u]dbinfo[%llu].",
1329 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret, dbindex, dbinfo),
1330 : HCCL_E_RUNTIME);
1331 0 : if (deviceLogicId_ == HOST_DEVICE_ID) {
1332 : // ps侧DbSend对当前线程SetDevice后，改变了原GE通信域初始化时setdevice 0
1333 : // 若后面使用本线程save会导致getctx失败获取不到通信域句柄，所以需要在此处重新set回默认
1334 0 : CHK_RET(hrtSetDevice(0));
1335 : }
1336 0 : return HCCL_SUCCESS;
1337 : }
1338 :
1339 0 : HcclResult TransportHeterogRoce::CreateDevMemForNotify(DeviceMem& devMem, u64 size, u32 value)
1340 : {
1341 0 : HCCL_INFO("Use dev mem for notify value");
1342 0 : void* devMemAddr{nullptr};
1343 0 : CHK_RET(hrtSetDevice(index_));
1344 0 : CHK_RET(HrtDevMalloc(&devMemAddr, size));
1345 :
1346 0 : devMemPtrs_.emplace_back(devMemAddr);
1347 :
1348 0 : devMem = DeviceMem::create(devMemAddr, size);
1349 0 : CHK_RET(hrtMemcpy(devMemAddr, size, &value, sizeof(u32), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1350 :
1351 0 : return HCCL_SUCCESS;
1352 : }
1353 :
1354 0 : HcclResult TransportHeterogRoce::CreateHostMemForNotify(DeviceMem& devMem, u64 size, u32 value, bool needMap)
1355 : {
1356 0 : HCCL_INFO("PS use host mem for notify value");
1357 0 : u64 memLen = size + SMALL_PAGE_SIZE;
1358 0 : s8* ptr = new (std::nothrow) s8[memLen];
1359 0 : CHK_PTR_NULL(ptr);
1360 0 : hostMemPtr_.emplace_back(ptr);
1361 0 : u64 pageSizeNum = reinterpret_cast<u64>(ptr) / SMALL_PAGE_SIZE;
1362 0 : void* ptrVoid = reinterpret_cast<void*>((pageSizeNum + 1) * SMALL_PAGE_SIZE);
1363 0 : void* devVirAddr = ptrVoid;
1364 0 : if (needMap) {
1365 0 : s32 ret = dataQpMrManager_->MapMem(ptrVoid, size, devVirAddr);
1366 0 : if (ret != 0 || devVirAddr == nullptr) {
1367 0 : HCCL_ERROR("PS malloc device mem fail[%d]", HCCL_E_MEMORY);
1368 0 : return HCCL_E_MEMORY;
1369 : }
1370 :
1371 0 : devMem = DeviceMem::create(devVirAddr, size);
1372 0 : CHK_RET(hrtMemcpy(
1373 : ptrVoid, notifyMem_.size(), &value, sizeof(u32), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_HOST));
1374 0 : return HCCL_SUCCESS;
1375 : }
1376 :
1377 0 : devMem = DeviceMem::create(devVirAddr, size);
1378 0 : s32 ret = memcpy_s(ptrVoid, notifyMem_.size(), &value, sizeof(u32));
1379 0 : if (ret < 0) {
1380 0 : HCCL_ERROR("memcpy_s fail[%d]", ret);
1381 0 : return HCCL_E_MEMORY;
1382 : }
1383 :
1384 0 : return HCCL_SUCCESS;
1385 : }
1386 :
1387 0 : HcclResult TransportHeterogRoce::CreateNotifyValueBuffer()
1388 : {
1389 0 : if (notifyMem_.ptr() == nullptr) {
1390 0 : u32 notifyVaule = 1;
1391 :
1392 0 : if (deviceLogicId_ == HOST_DEVICE_ID && isHdcMode_ && (dataQpInfo_.qpMode != NORMAL_QP_MODE)) {
1393 : // ES多机AI server的PS，申请device内存
1394 0 : CHK_RET(CreateDevMemForNotify(notifyMem_, notifyValueSize_, notifyVaule));
1395 0 : } else {
1396 0 : CHK_RET(CreateHostMemForNotify(notifyMem_, notifyValueSize_, notifyVaule, isHdcMode_));
1397 : }
1398 :
1399 0 : CHK_PRT_RET(!notifyMem_.ptr(), HCCL_ERROR("CreateNotifyValueBuffer malloc failed."), HCCL_E_MEMORY);
1400 : }
1401 :
1402 0 : struct MrInfoT mrInfo = {};
1403 0 : mrInfo.addr = notifyMem_.ptr();
1404 0 : mrInfo.size = notifyValueSize_;
1405 0 : mrInfo.access = access_;
1406 0 : CHK_RET(HrtRaMrReg(dataQpInfo_.qpHandle, &mrInfo));
1407 :
1408 0 : notifyMemMsg_[static_cast<u32>(MemType::NOTIFY_VALUE_MEM)].mrRegFlag = REG_VALID;
1409 0 : notifyMemMsg_[static_cast<u32>(MemType::NOTIFY_VALUE_MEM)].addr = notifyMem_.ptr();
1410 0 : notifyMemMsg_[static_cast<u32>(MemType::NOTIFY_VALUE_MEM)].len = notifyValueSize_;
1411 0 : notifyMemMsg_[static_cast<u32>(MemType::NOTIFY_VALUE_MEM)].memType = MemType::NOTIFY_VALUE_MEM;
1412 0 : notifyMemMsg_[static_cast<u32>(MemType::NOTIFY_VALUE_MEM)].lkey = mrInfo.lkey;
1413 0 : return HCCL_SUCCESS;
1414 : }
1415 :
1416 0 : HcclResult TransportHeterogRoce::DeleteNotifyValueBuffer()
1417 : {
1418 0 : for (u64 i = 0; i < devMemPtrs_.size(); i++) {
1419 0 : if (devMemPtrs_[i] != nullptr) {
1420 0 : CHK_RET(HrtDevFree(devMemPtrs_[i]));
1421 0 : devMemPtrs_[i] = nullptr;
1422 : }
1423 : }
1424 :
1425 0 : devMemPtrs_.clear();
1426 :
1427 0 : for (u64 i = 0; i < hostMemPtr_.size(); i++) {
1428 0 : if (hostMemPtr_[i] != nullptr) {
1429 0 : delete[] hostMemPtr_[i];
1430 0 : hostMemPtr_[i] = nullptr;
1431 : }
1432 : }
1433 :
1434 0 : hostMemPtr_.clear();
1435 :
1436 0 : return HCCL_SUCCESS;
1437 : }
1438 :
1439 0 : HcclResult TransportHeterogRoce::RecoverNotifyMsg(HcclRdmaSignalInfo* remoteRdmaSignal, u64 signalNum)
1440 : {
1441 0 : if (signalNum <= 0) {
1442 0 : return HCCL_E_NOT_FOUND;
1443 : }
1444 :
1445 0 : for (u64 i = 0; i < signalNum; i++) {
1446 0 : u32 tmpMemType = (remoteRdmaSignal + i)->type;
1447 0 : notifyMemMsg_[tmpMemType].mrRegFlag = (remoteRdmaSignal + i)->mrRegFlag;
1448 0 : notifyMemMsg_[tmpMemType].addr = (remoteRdmaSignal + i)->notifyAddr;
1449 0 : notifyMemMsg_[tmpMemType].len = (remoteRdmaSignal + i)->len;
1450 0 : notifyMemMsg_[tmpMemType].memType = (remoteRdmaSignal + i)->memType;
1451 0 : notifyMemMsg_[tmpMemType].rkey = (remoteRdmaSignal + i)->lkey;
1452 : }
1453 :
1454 0 : return HCCL_SUCCESS;
1455 : }
1456 :
1457 0 : HcclResult TransportHeterogRoce::CreatSignalMesg()
1458 : {
1459 0 : if (deviceLogicId_ == HOST_DEVICE_ID) {
1460 : // ps
1461 : // 310soc的ps不需要申请notify value，直接返回
1462 0 : if (!isHdcMode_ && !remoteIsHdc_) {
1463 0 : return HCCL_SUCCESS;
1464 : }
1465 : // Notify start
1466 0 : if (isHdcMode_) {
1467 0 : CHK_RET(hrtSetDevice(index_));
1468 : }
1469 0 : CHK_RET(CreateNotifyValueBuffer());
1470 : } else {
1471 : // worker
1472 : // 310soc的worker不需要在这里申请notify，直接返回
1473 0 : if (!isHdcMode_) {
1474 0 : return HCCL_SUCCESS;
1475 : }
1476 : // Notify start
1477 0 : CHK_RET(GetNotifySize());
1478 0 : CHK_RET(CreateRdmaSignal(remoteIsendDoneSignal_, rdmaSignal_[0], MemType::SEND_NOTIFY_MEM));
1479 0 : CHK_RET(CreateRdmaSignal(remoteImrecvDoneSignal_, rdmaSignal_[1], MemType::RECV_NOTIFY_MEM));
1480 : }
1481 :
1482 0 : return HCCL_SUCCESS;
1483 : }
1484 :
1485 0 : HcclResult TransportHeterogRoce::ExchangeSignalMesg()
1486 : {
1487 0 : if (deviceLogicId_ == HOST_DEVICE_ID) {
1488 : // ps
1489 : // Notify start
1490 0 : HcclRdmaSignalInfo remoteRdmaSignal[REMOTE_RDMA_SIGNAL_SIZE];
1491 0 : CHK_RET(
1492 : hrtRaSocketBlockRecv(fdHandle_, remoteRdmaSignal, sizeof(HcclRdmaSignalInfo) * REMOTE_RDMA_SIGNAL_SIZE));
1493 0 : CHK_RET(RecoverNotifyMsg(remoteRdmaSignal, REMOTE_RDMA_SIGNAL_SIZE));
1494 : } else {
1495 : // worker
1496 : // Notify start
1497 0 : CHK_RET(hrtRaSocketBlockSend(fdHandle_, rdmaSignal_, sizeof(HcclRdmaSignalInfo) * REMOTE_RDMA_SIGNAL_SIZE));
1498 : }
1499 0 : return HCCL_SUCCESS;
1500 : }
1501 :
1502 0 : HcclResult TransportHeterogRoce::RecordNotifyWithReq(Stream& stream, RdmaNotifyOp type, HcclRequestInfo*& request)
1503 : {
1504 0 : TransData sendData{};
1505 0 : TransportEndPointParam epParam{};
1506 :
1507 0 : CHK_RET(GenerateSendRequest(sendData, epParam, request));
1508 0 : request->transportRequest.requestType = HcclRequestType::HCCL_REQUEST_RECV;
1509 0 : u64 wrId = reinterpret_cast<uint64_t>(request);
1510 0 : CHK_RET(RecordNotify(stream, type, wrId));
1511 :
1512 0 : s32 notifyFlag = HCCL_TEST_INCOMPLETED;
1513 0 : TIME_PRINT(CHK_RET(this->Wait(*request, notifyFlag)));
1514 :
1515 0 : return HCCL_SUCCESS;
1516 : }
1517 :
1518 0 : HcclResult TransportHeterogRoce::RecordNotify([[maybe_unused]] Stream& stream, RdmaNotifyOp type, u64 wrId)
1519 : {
1520 0 : HCCL_INFO(
1521 : "RecordNotify notifyType[%u], wrId[%llu] isHdcMode_[%d] qpMode[%d]", type, wrId, isHdcMode_,
1522 : dataQpInfo_.qpMode);
1523 0 : MemType opType = MemType::MEM_TYPE_RESERVED;
1524 0 : if (type == RdmaNotifyOp::SEND_NOTIFY) {
1525 0 : opType = MemType::SEND_NOTIFY_MEM;
1526 0 : } else if (type == RdmaNotifyOp::RECV_NOTIFY) {
1527 0 : opType = MemType::RECV_NOTIFY_MEM;
1528 : } else {
1529 0 : HCCL_ERROR("TransportHeterogRoce::TYPE is not supported.");
1530 0 : return HCCL_E_PARA;
1531 : }
1532 :
1533 0 : if (!isHdcMode_ || dataQpInfo_.qpMode == NORMAL_QP_MODE) {
1534 0 : notifyWriteSge_.addr = static_cast<u64>(reinterpret_cast<uintptr_t>(notifyMem_.ptr()));
1535 0 : notifyWriteSge_.length = notifyMemMsg_[static_cast<u32>(opType)].len;
1536 0 : notifyWriteSge_.lkey = notifyMemMsg_[static_cast<u32>(MemType::NOTIFY_VALUE_MEM)].lkey;
1537 :
1538 0 : notifyWriteWr_.wr_id = wrId;
1539 : notifyWriteWr_.wr.rdma.remote_addr
1540 0 : = static_cast<u64>(reinterpret_cast<uintptr_t>(notifyMemMsg_[static_cast<u32>(opType)].addr));
1541 0 : notifyWriteWr_.wr.rdma.rkey = notifyMemMsg_[static_cast<u32>(opType)].rkey;
1542 :
1543 0 : struct ibv_send_wr* badWr = nullptr;
1544 0 : HCCL_INFO(
1545 : "notify write: remote addr[%llu] length[%d] wrId[%llu]", notifyWriteWr_.wr.rdma.remote_addr,
1546 : notifyWriteSge_.length, notifyWriteWr_.wr_id);
1547 0 : CHK_RET(hrtIbvPostSend(dataQpInfo_.qp, ¬ifyWriteWr_, &badWr));
1548 0 : } else {
1549 0 : struct SgList list = {};
1550 0 : list.addr = static_cast<u64>(reinterpret_cast<uintptr_t>(notifyMem_.ptr()));
1551 0 : list.len = notifyMemMsg_[static_cast<u32>(opType)].len;
1552 0 : list.lkey = notifyMemMsg_[static_cast<u32>(MemType::NOTIFY_VALUE_MEM)].lkey;
1553 :
1554 0 : struct SendWrV2 wr {};
1555 0 : wr.wrId = wrId;
1556 0 : wr.bufList = &list;
1557 0 : wr.bufNum = 1; /* 此处list只有一个，设置为1 */
1558 0 : wr.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(notifyMemMsg_[static_cast<u32>(opType)].addr));
1559 0 : wr.rkey = notifyMemMsg_[static_cast<u32>(opType)].rkey;
1560 0 : wr.op = static_cast<u32>(RdmaOp::OP_WRITE); /* RDMA_WRITE: 0 */
1561 0 : wr.sendFlag = RA_SEND_SIGNALED | RA_SEND_FENCE;
1562 0 : struct SendWrRsp opRsp = {};
1563 0 : CHK_RET(HrtRaSendWrV2(dataQpInfo_.qpHandle, &wr, &opRsp, GetWorkflowMode()));
1564 0 : CHK_RET(DoorBellSend(dataQpInfo_.qpMode, opRsp));
1565 : }
1566 :
1567 0 : return HCCL_SUCCESS;
1568 : }
1569 :
1570 0 : HcclResult TransportHeterogRoce::DoorBellSend(const s32 qpMode, const SendWrRsp& opRsp, void* stream)
1571 : {
1572 0 : if (qpMode == OPBASE_QP_MODE || qpMode == OPBASE_QP_MODE_EXT || qpMode == OFFLINE_QP_MODE_EXT) {
1573 0 : HCCL_DEBUG("entry PsRdmaDbSend");
1574 0 : u32 dbIndex = static_cast<u32>(opRsp.db.dbIndex);
1575 0 : u64 dbInfo = static_cast<u64>(opRsp.db.dbInfo);
1576 0 : CHK_RET(PsRdmaDbSend(dbIndex, dbInfo, stream));
1577 0 : } else {
1578 0 : HCCL_DEBUG("entry hrtRDMASend");
1579 0 : u32 qpn = opRsp.wqeTmp.sqIndex;
1580 0 : u32 wqe_index = opRsp.wqeTmp.wqeIndex;
1581 0 : CHK_RET(hrtRDMASend(qpn, wqe_index, stream));
1582 : }
1583 :
1584 0 : return HCCL_SUCCESS;
1585 : }
1586 :
1587 0 : HcclResult TransportHeterogRoce::MrManagerInit()
1588 : {
1589 : // mrManager_管理信封内存
1590 0 : if (IsRamdHandleLevelMr()) {
1591 : // 通信域初始化时外部还未传入全局内存，需要在这里面手动去初始化需要的全局内存
1592 0 : CHK_PTR_NULL(mrManager_);
1593 0 : std::map<MrMapKey, MrInfo> unRegMrMap = MrManager::GetInstance().GetUnregMap();
1594 0 : mrManager_->InitUnRegMrMap(unRegMrMap);
1595 : // 使用全局的MrManager时dataQp也使用全局的MrManager
1596 0 : dataQpMrManager_ = mrManager_;
1597 0 : return HCCL_SUCCESS;
1598 0 : }
1599 0 : mrManager_ = new (nothrow) MrManager();
1600 0 : CHK_PTR_NULL(mrManager_);
1601 0 : std::map<MrMapKey, MrInfo> unRegMrMap = MrManager::GetInstance().GetUnregMap();
1602 0 : CHK_PRT(mrManager_->Init(tagQpInfo_.qpHandle, index_, deviceLogicId_ == HOST_DEVICE_ID, unRegMrMap));
1603 :
1604 : // dataQpManager_管理数据收发内存
1605 0 : dataQpMrManager_ = new (nothrow) MrManager();
1606 0 : CHK_PTR_NULL(dataQpMrManager_);
1607 0 : unRegMrMap = MrManager::GetInstance().GetUnregMap();
1608 0 : CHK_PRT(dataQpMrManager_->Init(dataQpInfo_.qpHandle, index_, deviceLogicId_ == HOST_DEVICE_ID, unRegMrMap));
1609 :
1610 0 : return HCCL_SUCCESS;
1611 0 : }
1612 :
1613 0 : HcclResult TransportHeterogRoce::MrManagerDeInit()
1614 : {
1615 0 : if (IsRamdHandleLevelMr()) {
1616 : // 若mrManager是全局的，那么就在通信类外部释放
1617 0 : dataQpMrManager_ = nullptr;
1618 0 : return HCCL_SUCCESS;
1619 : }
1620 0 : HCCL_INFO("entry MrManagerDeInit");
1621 0 : CHK_PTR_NULL(mrManager_);
1622 0 : CHK_PRT(mrManager_->DeInit(tagQpInfo_.qpHandle));
1623 0 : delete mrManager_;
1624 :
1625 0 : CHK_PTR_NULL(dataQpMrManager_);
1626 0 : CHK_PRT(dataQpMrManager_->DeInit(dataQpInfo_.qpHandle));
1627 0 : delete dataQpMrManager_;
1628 :
1629 0 : return HCCL_SUCCESS;
1630 : }
1631 :
1632 0 : HcclResult TransportHeterogRoce::PreHdcResource()
1633 : {
1634 : // hdc模式下在通信类内部注册内存
1635 0 : CHK_PRT(MrManagerInit());
1636 : // worker侧不做信封内存注册
1637 0 : if (deviceLogicId_ == HOST_DEVICE_ID) {
1638 0 : if (!IsRamdHandleLevelMr()) {
1639 0 : CHK_PRT(MemBlocksManagerInit());
1640 0 : CHK_RET(mrManager_->GetKey(
1641 : tagMemBlocksManager_->GetMemAddr(), tagMemBlocksManager_->GetMemSize(), blockMemLkey_));
1642 : }
1643 :
1644 : const std::unique_ptr<HeterogMemBlocksManager>& memBlocksManagerPtr
1645 0 : = (IsRamdHandleLevelMr()) ? memBlocksManager_ : tagMemBlocksManager_;
1646 : // mrmanager是全局的时使用的信封内存管理类也是外部传入的全局的管理类
1647 0 : hostAddrBegin_ = (u64)memBlocksManagerPtr->GetMemAddr();
1648 :
1649 0 : devAddrBegin_ = MrManager::g_devAddr;
1650 0 : recvWqeBatchNum_ = hdcHostWqeBatchNum_;
1651 0 : recvWqeBatchThreshold_ = hdcHostWqeBatchNum_;
1652 0 : recvWqeBatchSupplement_ = RECV_WQE_HDC_BATCH_SUPPLEMENT;
1653 0 : HCCL_INFO(
1654 : "PreHdcResource IsRamdHandleLevelMr[%d] recvWqeBatchNum_[%u] recvWqeBatchThreshold_[%u]",
1655 : IsRamdHandleLevelMr(), recvWqeBatchNum_, recvWqeBatchThreshold_);
1656 0 : if (useDevMem_) {
1657 : #ifndef CCL_KERNEL
1658 : CHK_RET(hrtSetDevice(index_));
1659 : // device内存申请跟host内存一样大的内存
1660 : u64 memSize = memBlocksManagerPtr->GetMemSize();
1661 : s32 ret = HrtDevMalloc(&deviceEvePtr_, memSize);
1662 : if (ret != 0 || deviceEvePtr_ == nullptr) {
1663 : HCCL_ERROR("PS HrtDevMalloc device mem fail ret=[%d]", ret);
1664 : return HCCL_E_MEMORY;
1665 : }
1666 :
1667 : struct MrInfoT mrInfo = {};
1668 : mrInfo.addr = deviceEvePtr_;
1669 : mrInfo.size = memSize;
1670 : mrInfo.access = RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_WRITE | RA_ACCESS_REMOTE_READ;
1671 : CHK_RET(HrtRaMrReg(dataQpInfo_.qpHandle, &mrInfo));
1672 : deviceEveLkey_ = mrInfo.lkey;
1673 : HCCL_INFO("index_[%u] deviceEvePtr_[%p] memSize[%llu]", index_, deviceEvePtr_, memSize);
1674 : #endif
1675 : }
1676 0 : CHK_RET(InitTagRecvWqe());
1677 0 : return HCCL_SUCCESS;
1678 : }
1679 :
1680 0 : return HCCL_SUCCESS;
1681 : }
1682 :
1683 0 : HcclResult TransportHeterogRoce::MemBlocksManagerInit()
1684 : {
1685 : // 初始化信封内存
1686 0 : tagMemBlocksManager_.reset(new (std::nothrow) HeterogMemBlocksManager());
1687 0 : CHK_SMART_PTR_NULL(tagMemBlocksManager_);
1688 0 : CHK_RET(tagMemBlocksManager_->Init(memBlockNum_));
1689 :
1690 0 : return HCCL_SUCCESS;
1691 : }
1692 :
1693 0 : HcclResult TransportHeterogRoce::MemBlocksManagerDeInit()
1694 : {
1695 0 : while (wqeBlockLists_.size() > 0) {
1696 0 : CHK_RET(FreeMemBlock(wqeBlockLists_.front()));
1697 : }
1698 :
1699 0 : if (IsRamdHandleLevelMr()) {
1700 : // MrManager是全局的时候在通信类外部统一释放
1701 0 : return HCCL_SUCCESS;
1702 : }
1703 0 : CHK_SMART_PTR_NULL(tagMemBlocksManager_);
1704 0 : CHK_PTR_NULL(mrManager_);
1705 0 : CHK_RET(mrManager_->ReleaseKey(tagMemBlocksManager_->GetMemAddr(), tagMemBlocksManager_->GetMemSize()));
1706 0 : tagMemBlocksManager_ = nullptr;
1707 :
1708 0 : return HCCL_SUCCESS;
1709 : }
1710 :
1711 0 : void TransportHeterogRoce::GetLinkTag(std::string& tag)
1712 : {
1713 0 : tag = initSM_.locInitInfo.socketInfo[0].tag;
1714 0 : return;
1715 : }
1716 :
1717 0 : bool TransportHeterogRoce::IsRamdHandleLevelMr()
1718 : {
1719 : // 非hdc模式或者AI-Server910B场景ps下不分平面时以RdmaHandle粒度注册MR
1720 0 : return (!isHdcMode_ || (isHdcMode_ && isGlobalMrmanagerInit_));
1721 : }
1722 :
1723 : } // namespace hccl
|
