Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "reduce_scatter_slim_ring.h"
12 : #include "alg_template_register.h"
13 :
14 : namespace hccl {
15 0 : ReduceScatterSlimRing::ReduceScatterSlimRing(const HcclDispatcher dispatcher)
16 0 : : AlgTemplateBase(dispatcher)
17 : {
18 0 : }
19 :
20 0 : ReduceScatterSlimRing::~ReduceScatterSlimRing()
21 : {
22 0 : }
23 :
24 0 : HcclResult ReduceScatterSlimRing::Prepare(u64 reduceAttrBitMap, HcomCollOpInfo *opInfo)
25 : {
26 : (void)opInfo;
27 0 : reduceAttr_ = reduceAttrBitMap;
28 0 : return HCCL_SUCCESS;
29 : }
30 :
31 0 : HcclResult ReduceScatterSlimRing::RunVectorSourceReducer(const LINK &link, const std::vector<Slice> &txSlices,
32 : const std::vector<Slice> &txSlicetemp)
33 : {
34 : /* 1、对外reduce_scatter，output的大小为每块数据*rank_size。只能发送到对端地址偏移为0开始。
35 : 2、allreduce中使用reduce_scatter，output与Input大小相等，接收和发送偏移相等都为slice.offset */
36 0 : std::vector<SenderMemoryInfo> txMems;
37 0 : for (u32 i = 0; i < txSlices.size(); i++) {
38 0 : DeviceMem srcMem = inputMem_.range(txSlices[i].offset, txSlices[i].size);
39 0 : HCCL_DEBUG("send inputmem range[%llu], size[%llu] tx dstmem offset[%llu]", txSlices[i].offset,
40 : txSlices[i].size, txSlicetemp[i].offset);
41 0 : txMems.emplace_back(SenderMemoryInfo{baseOffset_ + txSlicetemp[i].offset, srcMem});
42 0 : }
43 0 : CHK_RET(senderInfo_->run(link, txMems, notifyIdx_, stream_));
44 :
45 0 : return HCCL_SUCCESS;
46 0 : }
47 :
48 0 : HcclResult ReduceScatterSlimRing::RunVectorDestRducer(const LINK &link, const std::vector<Slice> &rxSlices,
49 : const std::vector<Slice> &rxSlicetemp)
50 : {
51 0 : std::vector<ReducerMemoryInfo> rxReduceMems;
52 0 : for (u32 i = 0; i < rxSlices.size(); i++) {
53 0 : DeviceMem dstMem = inputMem_.range(rxSlices[i].offset, rxSlices[i].size);
54 0 : DeviceMem srcMemTemp = scratchMem_.range(rxSlicetemp[i].offset, rxSlicetemp[i].size);
55 0 : HCCL_DEBUG("rcv offset[%llu], size[%llu] ,then reduce with "
56 : "offset[%llu] size[%llu] ",
57 : rxSlicetemp[i].offset, rxSlicetemp[i].size, rxSlices[i].offset, rxSlices[i].size);
58 0 : rxReduceMems.emplace_back(ReducerMemoryInfo{baseOffset_ + rxSlices[i].offset, dstMem, dstMem, srcMemTemp});
59 0 : }
60 0 : CHK_RET(reducerInfo_->run(dispatcher_, link, rxReduceMems, notifyIdx_, stream_));
61 :
62 0 : return HCCL_SUCCESS;
63 0 : }
64 :
65 0 : HcclResult ReduceScatterSlimRing::RunVectorFinRducer(const u32 rank,
66 : const LINK &link,
67 : const u32 sliceSize,
68 : const std::vector<Slice> &inputSlices,
69 : const std::vector<Slice> &outputSlices)
70 : {
71 0 : std::vector<ReducerMemoryInfo> rxReduceMems;
72 0 : for (u32 i = 0; i < sliceSize; i++) {
73 : DeviceMem dstMem =
74 0 : outputMem_.range(outputSlices[rank * sliceSize + i].offset, outputSlices[rank * sliceSize + i].size);
75 : // reduce目的操作
76 : DeviceMem srcMem =
77 0 : inputMem_.range(inputSlices[rank * sliceSize + i].offset, inputSlices[rank * sliceSize + i].size);
78 : DeviceMem scratchMem =
79 0 : scratchMem_.range(outputSlices[rank * sliceSize + i].offset, outputSlices[rank * sliceSize + i].size);
80 0 : rxReduceMems.emplace_back(ReducerMemoryInfo{baseOffset_ + inputSlices[rank * sliceSize + i].offset,
81 : srcMem, dstMem, scratchMem});
82 0 : }
83 0 : CHK_RET(reducerInfo_->run(dispatcher_, link, rxReduceMems, notifyIdx_, stream_));
84 :
85 0 : notifyIdx_++;
86 :
87 0 : return HCCL_SUCCESS;
88 0 : }
89 :
90 0 : HcclResult ReduceScatterSlimRing::RunSourceReducer(const LINK &link, const Slice &txSlice, const Slice &txSlicetemp)
91 : {
92 : /* 1、对外reduce_scatter，output的大小为每块数据*rank_size。只能发送到对端地址偏移为0开始。
93 : 2、allreduce中使用reduce_scatter，output与Input大小相等，接收和发送偏移相等都为slice.offset */
94 0 : DeviceMem srcMem = inputMem_.range(txSlice.offset, txSlice.size);
95 0 : HCCL_DEBUG(" send inputmem range[%llu], size[%llu] tx dstmem offset[%llu]", txSlice.offset, txSlice.size,
96 : txSlicetemp.offset);
97 0 : CHK_RET(senderInfo_->run(link, baseOffset_ + txSlicetemp.offset, srcMem, stream_));
98 :
99 0 : return HCCL_SUCCESS;
100 0 : }
101 :
102 0 : HcclResult ReduceScatterSlimRing::RunDestRducer(const LINK &link, const Slice &rxSlice, const Slice &rxSlicetemp)
103 : {
104 0 : DeviceMem dstMem = inputMem_.range(rxSlice.offset, rxSlice.size);
105 0 : DeviceMem srcMemTemp = scratchMem_.range(rxSlicetemp.offset, rxSlicetemp.size);
106 0 : HCCL_DEBUG("rcv offset[%llu], size[%llu] ,then reduce with "
107 : "offset[%llu] size[%llu] ",
108 : rxSlicetemp.offset, rxSlicetemp.size, rxSlice.offset, rxSlice.size);
109 0 : CHK_RET(reducerInfo_->run(dispatcher_, link, baseOffset_ + rxSlice.offset, dstMem, dstMem, srcMemTemp, stream_));
110 :
111 0 : return HCCL_SUCCESS;
112 0 : }
113 :
114 0 : HcclResult ReduceScatterSlimRing::InitSlice(std::vector<Slice>& outputSlices, u32 rank, u32 rankSize, u32 unitSize){
115 0 : if (slices_.size() == 0) {
116 0 : slices_.resize(rankSize);
117 0 : outputSlices.resize(rankSize);
118 0 : u64 sliceSize = count_ * unitSize;
119 0 : for (u32 i = 0; i < rankSize; i++) {
120 0 : slices_[i].size = sliceSize;
121 0 : slices_[i].offset = (i * sliceSize);
122 0 : outputSlices[i].size = sliceSize;
123 0 : outputSlices[i].offset = (inputMem_.size() > outputMem_.size()) ? 0 : (i * sliceSize);
124 0 : HCCL_DEBUG("rank[%u], slices[%u].offset=[%llu], slices[%u].size=[%llu] outputSlices[%u].offset=[%llu], \
125 : outputSlices[%u].size=[%llu] ", rank, i, slices_[i].offset, i, slices_[i].size, i, \
126 : outputSlices[i].offset, i, outputSlices[i].size);
127 : }
128 : }
129 0 : return HCCL_SUCCESS;
130 : }
131 :
132 :
133 0 : HcclResult ReduceScatterSlimRing::RunAsync(const u32 rank, const u32 rankSize, const std::vector<LINK> &links)
134 : {
135 : // 判断stream, dispatcher是否为空
136 0 : CHK_SMART_PTR_NULL(dispatcher_);
137 0 : CHK_PTR_NULL(stream_.ptr());
138 0 : if (!outputMem_ || !inputMem_) {
139 0 : HCCL_ERROR("[ReduceScatterSlimRing][RunAsync]rank[%u] run_async inputmem or outputmem is null", rank);
140 0 : return HCCL_E_PTR;
141 : }
142 0 : HCCL_INFO("ReduceScatterRing run: rank[%u] ranksize[%u] inputMem[%p] outputMem[%p] count[%llu]",
143 : rank, rankSize, inputMem_.ptr(), outputMem_.ptr(), count_);
144 :
145 : // 判断rank_size == 1
146 0 : if (rankSize == 1) {
147 0 : if (inputMem_ != outputMem_) {
148 0 : return HcclD2DMemcpyAsync(dispatcher_, outputMem_, inputMem_, stream_);
149 : }
150 0 : return HCCL_SUCCESS;
151 : }
152 :
153 : // 创建reducer & sender
154 0 : senderInfo_.reset(new (std::nothrow) Sender(dataType_, reductionOp_, reduceAttr_));
155 0 : CHK_SMART_PTR_NULL(senderInfo_);
156 :
157 0 : reducerInfo_.reset(new (std::nothrow) Reducer(dataType_, reductionOp_, reduceAttr_));
158 0 : CHK_SMART_PTR_NULL(reducerInfo_);
159 :
160 0 : if (links.size() < rankSize) {
161 0 : HCCL_ERROR("[ReduceScatterSlimRing][RunAsync]rank[%u] link size[%llu] is less than rank size[%u]",
162 : rank, links.size(), rankSize);
163 0 : return HCCL_E_INTERNAL;
164 : }
165 :
166 0 : u32 ringPrevRank = (rank + rankSize - 1) % rankSize;
167 0 : linkLeft_ = links[ringPrevRank];
168 0 : CHK_SMART_PTR_NULL(linkLeft_);
169 :
170 0 : u32 ringNextRank = (rank + 1) % rankSize;
171 0 : linkRight_ = links[ringNextRank];
172 0 : CHK_SMART_PTR_NULL(linkRight_);
173 :
174 0 : u32 unitSize = DataUnitSize(dataType_);
175 0 : if (unitSize == 0) {
176 0 : HCCL_ERROR("[ReduceScatterSlimRing][RunAsync]rank[%u] unit data size is zero", rank);
177 0 : return HCCL_E_INTERNAL;
178 : }
179 :
180 0 : std::vector<Slice> outputSlices(slices_);
181 0 : InitSlice(outputSlices, rank, rankSize, unitSize);
182 : // 运行reduce-scatter, ring算法
183 : // 单环场景下 nicRankList_ 长度默认为 8。
184 : // 多环场景下 nicRankList_ 长度为网口数量。此时若 rankSize != nicRankList_ 则为网口裁剪场景
185 0 : if (rankSize != HCCL_NIC_MAX_NUM || nicRankList_.size() == HCCL_NIC_MAX_NUM) {
186 : // 非网口裁剪场景:
187 0 : CHK_RET(RunReduceScatter(rank, rankSize, slices_, outputSlices));
188 : }
189 :
190 0 : if (barrierSwitchOn_) {
191 : // 执行barrier，保证数据收发完成
192 0 : CHK_RET(ExecuteBarrier(linkRight_, linkLeft_, notifyIdx_));
193 0 : notifyIdx_++;
194 : }
195 :
196 0 : HCCL_INFO("ReduceScatterRing finished: rank[%u]", rank);
197 0 : return HCCL_SUCCESS;
198 0 : }
199 :
200 0 : HcclResult ReduceScatterSlimRing::RunReduceScatter(const u32 rank, const u32 rankSize,
201 : const std::vector<Slice> &inputSlices,
202 : const std::vector<Slice> &outputSlices)
203 : {
204 0 : bool bRetSize = (inputSlices.size() < rankSize);
205 0 : CHK_PRT_RET(bRetSize,
206 : HCCL_ERROR("[Run][ReduceScatter]rank[%u] inputslice size[%llu] is less than rank size[%u]",
207 : rank, inputSlices.size(), rankSize), HCCL_E_INTERNAL);
208 :
209 0 : bRetSize = (outputSlices.size() < rankSize);
210 0 : CHK_PRT_RET(bRetSize,
211 : HCCL_ERROR("[Run][ReduceScatter]rank[%u] outputslice size[%llu] is less than rank size[%u]",
212 : rank, outputSlices.size(), rankSize), HCCL_E_INTERNAL);
213 :
214 0 : HcclResult ret = HCCL_SUCCESS;
215 :
216 0 : u32 sliceSize = inputSlices.size() / rankSize;
217 :
218 : // 获取rx_slice, 首先向本rank前2个rank处发ack消息
219 0 : u32 rxSliceIndex = (rank + rankSize - 2) % rankSize;
220 :
221 : // reduce源操作, 获取tx_slice，从本rank前一rank开始接收ack
222 0 : u32 txSliceIndex = (rank + rankSize - 1) % rankSize;
223 :
224 0 : std::vector<Slice> txInputSegsSlice;
225 0 : std::vector<Slice> txOutputSegsSlice;
226 0 : for (u32 j = 0; j < sliceSize; j++) {
227 0 : txInputSegsSlice.push_back(inputSlices[txSliceIndex * sliceSize + j]);
228 0 : txOutputSegsSlice.push_back(outputSlices[txSliceIndex * sliceSize + j]);
229 : }
230 0 : ret = RunVectorSourceReducer(linkRight_, txInputSegsSlice, txOutputSegsSlice); // NotifyRecord
231 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
232 : HCCL_ERROR("[Run][ReduceScatter]rank[%u] txSliceIndex[%u] Reducer src run failed", rank, txSliceIndex), ret);
233 :
234 : // 本rank既当reduce源, 也当reduce操作的目的
235 0 : for (u32 i = 0; i < (rankSize - 2); i++) { // 中间rank_size - 2次传输
236 : // reduce目的操作
237 0 : std::vector<Slice> rxInputSegsSlice;
238 0 : std::vector<Slice> rxOutputSegsSlice;
239 0 : for (u32 j = 0; j < sliceSize; j++) {
240 0 : rxInputSegsSlice.push_back(inputSlices[rxSliceIndex * sliceSize + j]);
241 0 : rxOutputSegsSlice.push_back(outputSlices[rxSliceIndex * sliceSize + j]);
242 : }
243 0 : ret = RunVectorDestRducer(linkLeft_, rxInputSegsSlice, rxOutputSegsSlice);
244 :
245 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
246 : HCCL_ERROR("[Run][ReduceScatter]rank[%u] round[%u] rxSlice[%u] Reducer dst run failed", rank, i,
247 : rxSliceIndex),
248 : ret);
249 :
250 0 : notifyIdx_++;
251 :
252 : // 获取rx_slice
253 0 : rxSliceIndex = (rxSliceIndex + rankSize - 1) % rankSize;
254 :
255 : // reduce源操作, 获取tx_slice
256 0 : txSliceIndex = (txSliceIndex + rankSize - 1) % rankSize;
257 :
258 0 : std::vector<Slice> txInputSlice;
259 0 : std::vector<Slice> txOutputSlice;
260 0 : for (u32 j = 0; j < sliceSize; j++) {
261 0 : txInputSlice.push_back(inputSlices[txSliceIndex * sliceSize + j]);
262 0 : txOutputSlice.push_back(outputSlices[txSliceIndex * sliceSize + j]);
263 : }
264 0 : ret = RunVectorSourceReducer(linkRight_, txInputSlice, txOutputSlice);
265 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
266 : HCCL_ERROR("[Run][ReduceScatter]rank[%u] round[%u] Reducer src run failed", rank, i), ret);
267 0 : }
268 :
269 : /* * 末尾传输, 本rank只当reduce目的, 根据单buffer还是双buffer来决定如何搬移
270 : 当前简化处理, 只考虑单buffer的场景, 双buffer则在run_async中多拷贝一次 */
271 0 : RunVectorFinRducer(rank, linkLeft_, sliceSize, inputSlices, outputSlices);
272 :
273 0 : return HCCL_SUCCESS;
274 0 : }
275 :
276 0 : HcclResult ReduceScatterSlimRing::SetNotifyIdx(u32 notifyIdx)
277 : {
278 0 : notifyIdx_ = notifyIdx;
279 0 : return HCCL_SUCCESS;
280 : }
281 :
282 0 : HcclResult ReduceScatterSlimRing::GetNotifyIdx(u32 ¬ifyIdx)
283 : {
284 0 : notifyIdx = notifyIdx_;
285 0 : return HCCL_SUCCESS;
286 : }
287 :
288 : REGISTER_TEMPLATE(TemplateType::TEMPLATE_REDUCESCATTER_SLIM_RING, ReduceScatterSlimRing);
289 : } // namespace hccl
|
