Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "local_ipc_rma_buffer_impl.h"
12 : #include "adapter_rts.h"
13 : #include "hccl_network.h"
14 : #include "mem_mapping_manager.h"
15 :
16 : namespace hccl {
17 : namespace {
18 7 : void* GetAliasDevAddr(void* addr, const LocalIpcRmaBufferImpl& parent)
19 : {
20 7 : const uintptr_t childAddr = reinterpret_cast<uintptr_t>(addr);
21 7 : const uintptr_t parentAddr = reinterpret_cast<uintptr_t>(parent.GetAddr());
22 7 : const uintptr_t parentDevAddr = reinterpret_cast<uintptr_t>(parent.GetDevAddr());
23 7 : return reinterpret_cast<void*>(parentDevAddr + (childAddr - parentAddr));
24 : }
25 : } // namespace
26 :
27 16 : LocalIpcRmaBufferImpl::LocalIpcRmaBufferImpl(
28 16 : const HcclNetDevCtx netDevCtx, void* addr, u64 size, const RmaMemType memType)
29 16 : : RmaBuffer(netDevCtx, addr, size, memType, RmaType::IPC_RMA)
30 16 : {}
31 :
32 7 : LocalIpcRmaBufferImpl::LocalIpcRmaBufferImpl(
33 7 : const HcclNetDevCtx netDevCtx, void* addr, u64 size, const RmaMemType memType, const LocalIpcRmaBufferImpl& parent)
34 : : RmaBuffer(netDevCtx, addr, size, memType, RmaType::IPC_RMA, true),
35 7 : deviceLogicId(parent.deviceLogicId),
36 7 : memName(parent.memName),
37 7 : memOffset(parent.memOffset),
38 7 : initialized_(true)
39 : {
40 7 : devAddr = GetAliasDevAddr(addr, parent);
41 7 : HCCL_INFO("[LocalIpcRmaBufferImpl] alias constructor, ipcName[%s]", memName.ipcName);
42 7 : }
43 :
44 46 : LocalIpcRmaBufferImpl::~LocalIpcRmaBufferImpl()
45 : {
46 23 : if (!isAlias_) {
47 16 : HcclResult res = Destroy();
48 16 : if (res != HCCL_SUCCESS) {
49 0 : HCCL_ERROR("[LocalIpcRmaBufferImpl][~LocalIpcRmaBufferImpl]failed, ret[%d]", res);
50 : }
51 : }
52 46 : }
53 :
54 9 : HcclResult LocalIpcRmaBufferImpl::Init()
55 : {
56 9 : CHK_PTR_NULL(netDevCtx);
57 9 : deviceLogicId = (static_cast<NetDevContext*>(netDevCtx))->GetLogicId();
58 :
59 : // host内存地址映射
60 9 : devAddr = addr;
61 9 : if (memType == RmaMemType::HOST) {
62 0 : CHK_RET(MemMappingManager::GetInstance(deviceLogicId).GetDevVA(deviceLogicId, addr, size, devAddr));
63 : } else {
64 : // 设置ipc mem name
65 9 : HCCL_INFO("[LocalIpcRmaBufferImpl][Init]ipc set mem name");
66 : HcclResult ret = MemNameRepository::GetInstance(deviceLogicId)
67 9 : ->SetIpcMem(devAddr, size, memName.ipcName, HCCL_IPC_MEM_NAME_LEN);
68 9 : CHK_PRT_RET(
69 : ret != HCCL_SUCCESS,
70 : HCCL_ERROR(
71 : "[LocalIpcRmaBufferImpl][Init]errNo[0x%016llx], get para mem name failed. "
72 : "mem addr[%p] deviceLogicId[%d]",
73 : HCCL_ERROR_CODE(ret), devAddr, deviceLogicId),
74 : ret);
75 : }
76 9 : HCCL_DEBUG(
77 : "[LocalIpcRmaBufferImpl][Init]addr[%p], size[%llu], devAddr[%p], memType[%d]", addr, size, devAddr, memType);
78 9 : initialized_ = true;
79 9 : return HCCL_SUCCESS;
80 : }
81 :
82 2 : std::string& LocalIpcRmaBufferImpl::Serialize()
83 : {
84 2 : if (!serializeStr_.empty()) {
85 0 : return serializeStr_;
86 : }
87 : // 序列化信息
88 2 : std::ostringstream oss;
89 2 : u8 type{static_cast<u8>(rmaType)};
90 2 : oss.write(reinterpret_cast<const char_t*>(&type), sizeof(type));
91 2 : oss.write(reinterpret_cast<const char_t*>(&addr), sizeof(addr));
92 2 : oss.write(reinterpret_cast<const char_t*>(&size), sizeof(size));
93 2 : oss.write(reinterpret_cast<const char_t*>(&devAddr), sizeof(devAddr));
94 2 : oss.write(reinterpret_cast<const char_t*>(&memType), sizeof(memType));
95 2 : oss.write(reinterpret_cast<const char_t*>(&memName.ipcName), sizeof(memName.ipcName));
96 2 : oss.write(reinterpret_cast<const char_t*>(&memOffset), sizeof(memOffset));
97 2 : HCCL_DEBUG(
98 : "[LocalIpcRmaBufferImpl][Serialize] addr[%p], size[%llu], devAddr[%p], memType[%d], ipcName[%s], "
99 : "memOffset[%llu]",
100 : reinterpret_cast<void*>(addr), size, reinterpret_cast<void*>(devAddr), memType, memName.ipcName, memOffset);
101 :
102 2 : serializeStr_ = oss.str();
103 2 : return serializeStr_;
104 2 : }
105 :
106 : constexpr s32 IPC_NOTIFY_PID_ARRAY_SIZE = 1;
107 2 : HcclResult LocalIpcRmaBufferImpl::Grant(u32 remotePid, u32 remoteSdid)
108 : {
109 2 : if (memType == RmaMemType::HOST) {
110 0 : HCCL_DEBUG("[LocalIpcRmaBufferImpl][Grant]memType is [%d].", memType);
111 0 : return HCCL_SUCCESS;
112 : }
113 :
114 2 : HCCL_DEBUG("[LocalIpcRmaBufferImpl][Grant]ipcName[%s], pid[%u], sdid[%u]", memName.ipcName, remotePid, remoteSdid);
115 2 : s32 peerPid = static_cast<s32>(remotePid);
116 2 : s32 peerSdid = static_cast<s32>(remoteSdid);
117 2 : if (peerSdid != INVALID_INT) {
118 1 : CHK_RET(hrtSetIpcMemorySuperPodPid(memName.ipcName, peerSdid, &peerPid, IPC_NOTIFY_PID_ARRAY_SIZE));
119 : } else {
120 1 : CHK_RET(hrtIpcSetMemoryPid(memName.ipcName, &peerPid, IPC_NOTIFY_PID_ARRAY_SIZE));
121 : }
122 2 : return HCCL_SUCCESS;
123 : }
124 :
125 32 : HcclResult LocalIpcRmaBufferImpl::Destroy()
126 : {
127 32 : if (addr != nullptr && initialized_) {
128 : // host内存解映射
129 9 : HcclResult ret = HCCL_SUCCESS;
130 9 : if (memType == RmaMemType::HOST) {
131 0 : ret = MemMappingManager::GetInstance(deviceLogicId).ReleaseDevVA(deviceLogicId, addr, size);
132 0 : if (ret != HCCL_SUCCESS) {
133 0 : HCCL_ERROR(
134 : "[LocalIpcRmaBufferImpl][Destroy]release dev va failed, "
135 : "ret[%d], dev[%d], ptr[%p], size[%llu]",
136 : ret, deviceLogicId, addr, size);
137 : }
138 : } else {
139 : // 销毁ipc mem name
140 9 : MemNameRepository::GetInstance(deviceLogicId)->DestroyIpcMem(devAddr, size);
141 9 : HCCL_INFO(
142 : "[LocalIpcRmaBufferImpl][Destroy]ipc destroy mem name. "
143 : "mem addr[%p] deviceLogicId[%d]",
144 : devAddr, deviceLogicId);
145 : }
146 :
147 9 : addr = nullptr;
148 9 : size = 0;
149 9 : initialized_ = false;
150 9 : return ret;
151 : }
152 :
153 23 : return HCCL_SUCCESS;
154 : }
155 : } // namespace hccl
|
