Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "alg_template_register.h"
12 : #include "all_reduce_nhr_v1.h"
13 :
14 : namespace hccl {
15 0 : AllReduceNHRV1::AllReduceNHRV1(const HcclDispatcher dispatcher) : NHRV1Base(dispatcher) {}
16 :
17 0 : AllReduceNHRV1::~AllReduceNHRV1() {}
18 :
19 0 : HcclResult AllReduceNHRV1::Prepare(u64 reduceAttrBitMap, [[maybe_unused]] HcomCollOpInfo* opInfo)
20 : {
21 0 : reduceAttr_ = reduceAttrBitMap;
22 0 : return HCCL_SUCCESS;
23 : }
24 :
25 : HcclResult
26 0 : AllReduceNHRV1::RunAsync(const u32 rank, const u32 rankSize, const std::vector<std::shared_ptr<Transport>>& links)
27 : {
28 0 : CHK_RET(PrepareRunAsync(rank, rankSize, links));
29 0 : CHK_PRT_RET(
30 : rankSize == 1, HCCL_INFO("[AllReduceNHRV1][RunAsync] rankSize[%u], do nothing.", rankSize), HCCL_SUCCESS);
31 :
32 0 : HcclResult ret = HCCL_SUCCESS;
33 : // 获取通信关系
34 0 : RingInfo info = GetRingInfo(rankSize);
35 : // 水平方向做broken reducescatter ring
36 :
37 0 : ret = RunReduceScatterOnHorizontal(rank, links, info);
38 0 : CHK_PRT_RET(
39 : ret != HCCL_SUCCESS,
40 : HCCL_ERROR(
41 : "[AllReduceNHRV1][RunAsync]rank[%u] count[%llu] failed in "
42 : "RunReduceScatterOnHorizontal step",
43 : rank, count_),
44 : ret);
45 :
46 : // 垂直方向做allreduce ring
47 0 : ret = RunAllReduceOnVertical(rank, links, info);
48 0 : CHK_PRT_RET(
49 : ret != HCCL_SUCCESS,
50 : HCCL_ERROR(
51 : "[AllReduceNHRV1][RunAsync]rank[%u] count[%llu] failed in "
52 : "RunAllReduceOnVertical step",
53 : rank, count_),
54 : ret);
55 :
56 : // 水平方向做broken allgather ring
57 0 : ret = RunAllGatherOnHorizontal(rank, links, info);
58 0 : CHK_PRT_RET(
59 : ret != HCCL_SUCCESS,
60 : HCCL_ERROR(
61 : "[AllReduceNHRV1][RunAsync]rank[%u] count[%llu] failed in "
62 : "RunAllGatherOnHorizontal step",
63 : rank, count_),
64 : ret);
65 :
66 0 : HCCL_INFO("AllReduceNHRV1 finished: rank[%u]", rank);
67 0 : return HCCL_SUCCESS;
68 0 : }
69 :
70 : HcclResult
71 0 : AllReduceNHRV1::RunAsyncStaged(const u32 rank, const u32 rankSize, const std::vector<LINK>& links, RunStage stage)
72 : {
73 0 : CHK_PRT_RET(
74 : rankSize == 1 && stage != RunStage::RUN_PREPARE,
75 : HCCL_INFO("[AllReduceNHRV1][RunAsyncStaged] rankSize[%u], stage[%d], do nothing.", rankSize, stage),
76 : HCCL_SUCCESS);
77 : // 获取通信关系
78 0 : RingInfo info = GetRingInfo(rankSize);
79 :
80 0 : HcclResult ret = HCCL_SUCCESS;
81 0 : switch (stage) {
82 0 : case RunStage::RUN_PREPARE:
83 0 : ret = PrepareRunAsync(rank, rankSize, links);
84 0 : CHK_PRT_RET(
85 : ret != HCCL_SUCCESS,
86 : HCCL_ERROR(
87 : "[AllReduceNHRV1][RunAsyncStaged]rank[%u] count[%llu] failed in PrepareRunAsync step", rank,
88 : count_),
89 : ret);
90 0 : break;
91 0 : case RunStage::RUN_REDUCE_SCATTER:
92 : // 水平方向做broken reducescatter ring
93 0 : ret = RunReduceScatterOnHorizontal(rank, links, info);
94 0 : CHK_PRT_RET(
95 : ret != HCCL_SUCCESS,
96 : HCCL_ERROR(
97 : "[AllReduceNHRV1][RunAsync]rank[%u] count[%llu] failed in "
98 : "RunReduceScatterOnHorizontal step",
99 : rank, count_),
100 : ret);
101 0 : break;
102 0 : case RunStage::RUN_ALLREDUCE:
103 : // 垂直方向做allreduce ring
104 0 : ret = RunAllReduceOnVertical(rank, links, info);
105 0 : CHK_PRT_RET(
106 : ret != HCCL_SUCCESS,
107 : HCCL_ERROR(
108 : "[AllReduceNHRV1][RunAsync]rank[%u] count[%llu] failed in "
109 : "RunAllReduceOnVertical step",
110 : rank, count_),
111 : ret);
112 0 : break;
113 0 : case RunStage::RUN_ALLGATHER:
114 : // 水平方向做broken allgather ring
115 0 : ret = RunAllGatherOnHorizontal(rank, links, info);
116 0 : CHK_PRT_RET(
117 : ret != HCCL_SUCCESS,
118 : HCCL_ERROR(
119 : "[AllReduceNHRV1][RunAsync]rank[%u] count[%llu] failed in "
120 : "RunAllGatherOnHorizontal step",
121 : rank, count_),
122 : ret);
123 0 : break;
124 0 : default:
125 0 : HCCL_ERROR("[AllReduceNHRV1][RunAsyncStaged]stage[%d]is not support", stage);
126 0 : return HCCL_E_NOT_SUPPORT;
127 : }
128 0 : HCCL_INFO("AllReduceNHRV1 RunAsyncStaged stage[%d] finished: rank[%u] ranksize[%u]", stage, rank, rankSize);
129 0 : return HCCL_SUCCESS;
130 0 : }
131 :
132 0 : HcclResult AllReduceNHRV1::PrepareRunAsync(const u32 rank, const u32 rankSize, const std::vector<LINK>& links)
133 : {
134 0 : HcclResult ret = HCCL_SUCCESS;
135 0 : CHK_SMART_PTR_NULL(dispatcher_);
136 0 : CHK_PTR_NULL(stream_.ptr());
137 0 : CHK_PRT_RET(
138 : !outputMem_ || !inputMem_,
139 : HCCL_ERROR("[AllReduceNHRV1][RunAsync]rank[%u] run_async inputmem or outputmem is null", rank), HCCL_E_PTR);
140 :
141 0 : HCCL_INFO(
142 : "AllReduceNHRV1 run: rank[%u] ranksize[%u] inputMem[%p] outputMem[%p] count[%llu]", rank, rankSize,
143 : inputMem_.ptr(), outputMem_.ptr(), count_);
144 :
145 0 : CHK_PRT_RET(
146 : links.size() < rankSize,
147 : HCCL_ERROR(
148 : "[AllReduceNHRV1][RunAsync]rank[%u] linksize[%llu] is less "
149 : "than rankSize[%u]",
150 : rank, links.size(), rankSize),
151 : HCCL_E_INTERNAL);
152 :
153 : // 如果ranksize为1, inline reduce和普通跨片reduce操作一致，从input->output
154 0 : if (rankSize == 1) {
155 0 : if (inputMem_ != outputMem_) {
156 0 : ret = HcclD2DMemcpyAsync(dispatcher_, outputMem_, inputMem_, stream_);
157 0 : CHK_PRT_RET(
158 : ret != HCCL_SUCCESS, HCCL_ERROR("[AllReduceNHRV1][RunAsync]rank[%u] memcpy async failed", rank), ret);
159 : }
160 0 : return ret;
161 : }
162 :
163 : // 检查、并清空slices_
164 0 : if (slices_.size() != 0) {
165 0 : HCCL_WARNING("[AllReduceNHRV1][RunAsync]AllReduceNHRV1 not supported passing in parameter slice_, "
166 : "otherwise will be cleared");
167 0 : slices_.clear();
168 : }
169 0 : return HCCL_SUCCESS;
170 : }
171 :
172 0 : HcclResult AllReduceNHRV1::CalcHSlicesAndLinks(
173 : const u32 rank, const std::vector<LINK>& links, const RingInfo& info, std::vector<LINK>& hLinks,
174 : std::vector<Slice>& hSlices)
175 : {
176 0 : u32 ringSize = info.GetHSizeByRank(rank); // 查找自己所处的行长度，也即Ring的大小
177 0 : u32 vIndex = info.GetVIndex(rank); // 查找自己位于第几行
178 :
179 : // 计算水平方向每个rank结果上的offset和size
180 0 : u64 sliceSizeCalculated = (count_ + (info.GetRowSize() - 1)) / info.GetRowSize() * DataUnitSize(dataType_);
181 0 : u64 totalSize = count_ * DataUnitSize(dataType_);
182 0 : u64 residueSize = totalSize;
183 0 : u64 sliceSizeAligned = AlgTemplateBase::RoundUpWithDivisor(sliceSizeCalculated, HCCL_MIN_SLICE_ALIGN);
184 :
185 : // 水平方向都为broken ring，故最后一列有可能不需要参与计算，此时size为0
186 0 : for (u32 hIdx = 0; hIdx < ringSize; hIdx++) {
187 0 : u32 oldRank = info.GetRank(vIndex, hIdx);
188 :
189 0 : CHK_PRT_RET(
190 : oldRank >= links.size(),
191 : HCCL_ERROR(
192 : "[AllReduceNHRV1] rank[%u] out of range, "
193 : "oldRank=%u, links.size=%u",
194 : rank, oldRank, links.size()),
195 : HCCL_E_INTERNAL);
196 0 : hLinks.push_back(links[oldRank]);
197 0 : Slice slice;
198 0 : if (info.GetVSizeByHIndex(hIdx) == info.GetVSizeByHIndex(0)) {
199 0 : slice.size = (residueSize > sliceSizeAligned) ? sliceSizeAligned : residueSize;
200 0 : slice.offset = totalSize - residueSize;
201 0 : residueSize -= slice.size;
202 : } else {
203 0 : slice.size = 0;
204 0 : slice.offset = 0;
205 : }
206 0 : HCCL_DEBUG(
207 : "[AllReduceNHRV1][CalcHSlicesAndLinks] rank[%u], slices[%u].offset=%llu, slices[%u].size=%llu", rank, hIdx,
208 : slice.offset, hIdx, slice.size);
209 0 : hSlices.push_back(slice);
210 : }
211 0 : return HCCL_SUCCESS;
212 : }
213 :
214 0 : HcclResult AllReduceNHRV1::CalcVSlicesAndLinks(
215 : const u32 rank, const std::vector<LINK>& links, const RingInfo& info, std::vector<LINK>& vLinks,
216 : std::vector<Slice>& vSlices)
217 : {
218 0 : u32 ringSize = info.GetVSizeByRank(rank); // 查找自己所处的列长度，也即Ring的大小
219 0 : u32 hIndex = info.GetHIndex(rank); // 查找自己位于第几列
220 :
221 0 : std::vector<Slice> hSlices;
222 0 : std::vector<LINK> hLinks;
223 0 : CHK_RET(CalcHSlicesAndLinks(rank, links, info, hLinks, hSlices));
224 :
225 : // 计算垂直方向每个rank结果上的offset和size
226 : u64 sliceSizeCalculated
227 0 : = (hSlices[hIndex].size / DataUnitSize(dataType_) + (ringSize - 1)) / ringSize * DataUnitSize(dataType_);
228 0 : u64 totalSize = hSlices[hIndex].size;
229 0 : u64 residueSize = totalSize;
230 0 : u64 sliceSizeAligned = AlgTemplateBase::RoundUpWithDivisor(sliceSizeCalculated, HCCL_MIN_SLICE_ALIGN);
231 :
232 0 : for (u32 vIdx = 0; vIdx < ringSize; vIdx++) {
233 0 : u32 oldRank = info.GetRank(vIdx, hIndex);
234 0 : CHK_PRT_RET(
235 : oldRank >= links.size(),
236 : HCCL_ERROR(
237 : "[AllReduceNHRV1] rank[%u] out of range, "
238 : "oldRank=%u, links.size=%u",
239 : rank, oldRank, links.size()),
240 : HCCL_E_INTERNAL);
241 0 : vLinks.push_back(links[oldRank]);
242 0 : Slice slice;
243 0 : slice.size = (residueSize > sliceSizeAligned) ? sliceSizeAligned : residueSize;
244 0 : slice.offset = hSlices[hIndex].offset + totalSize - residueSize;
245 0 : residueSize -= slice.size;
246 0 : HCCL_DEBUG(
247 : "[AllReduceNHRV1][CalcVSlicesAndLinks] rank[%u], slices[%u].offset=%llu, slices[%u].size=%llu", rank, vIdx,
248 : slice.offset, vIdx, slice.size);
249 0 : vSlices.push_back(slice);
250 : }
251 0 : return HCCL_SUCCESS;
252 0 : }
253 :
254 : HcclResult
255 0 : AllReduceNHRV1::RunReduceScatterOnHorizontal(const u32 rank, const std::vector<LINK>& links, const RingInfo& info)
256 : {
257 0 : u32 ringRank = info.GetHIndex(rank); // 查找自己位于第几列，也即处于Ring中的第几个rank
258 :
259 : // 计算reducescatter每个rank结果上的offset和size
260 0 : std::vector<Slice> hSlices;
261 0 : std::vector<LINK> hLinks;
262 0 : CHK_RET(CalcHSlicesAndLinks(rank, links, info, hLinks, hSlices));
263 :
264 : // 长度不足2，直接跳过
265 0 : if (hLinks.size() < 2) {
266 0 : return HCCL_SUCCESS;
267 : }
268 :
269 0 : HCCL_DEBUG("[AllReduceNHRV1][ReduceScatter-H] rank[%u] ringRank=%u, ringSize=%u", rank, ringRank, hLinks.size());
270 0 : return RunReduceScatterBrokenRing(ringRank, hLinks, hSlices);
271 0 : }
272 :
273 0 : HcclResult AllReduceNHRV1::RunAllReduceOnVertical(const u32 rank, const std::vector<LINK>& links, const RingInfo& info)
274 : {
275 0 : u32 ringRank = info.GetVIndex(rank); // 查找自己位于第几行，也即处于Ring中的第几个rank
276 0 : u32 ringSize = info.GetVSizeByRank(rank); // 查找自己所处的列长度，也即Ring的大小
277 : // 若最后一列不完整，则不做allreduce操作直接返回success
278 0 : if (ringSize < info.GetVSizeByHIndex(0)) {
279 0 : return HCCL_SUCCESS;
280 : }
281 : // 计算allreduce 阶段每个rank结果上的offset和size
282 0 : std::vector<Slice> vSlices;
283 0 : std::vector<LINK> vLinks;
284 0 : CHK_RET(CalcVSlicesAndLinks(rank, links, info, vLinks, vSlices));
285 :
286 0 : std::unique_ptr<AlgTemplateBase> tempAlg;
287 0 : tempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_REDUCE_RING, dispatcher_);
288 0 : CHK_SMART_PTR_NULL(tempAlg);
289 0 : CHK_RET(tempAlg->Prepare(reduceAttr_));
290 :
291 : // 判断是否关闭allreduce的barrier
292 0 : if (!barrierSwitchOn_) {
293 0 : tempAlg->CloseBarrier();
294 : }
295 :
296 0 : CHK_RET(tempAlg->Prepare(
297 : inputMem_, outputMem_, outputMem_, count_, dataType_, stream_, reductionOp_, root_, vSlices, baseOffset_));
298 :
299 0 : CHK_RET(tempAlg->RegisterProfiler(profilerInput_.planeID, profilerInput_.stage, profilerInput_.step, stream_));
300 :
301 0 : HCCL_DEBUG("[AllReduceNHRV1][AllReduce-V] rank[%u] ringRank=%u, ringSize=%u", rank, ringRank, ringSize);
302 0 : return tempAlg->RunAsync(ringRank, ringSize, vLinks);
303 0 : }
304 :
305 : HcclResult
306 0 : AllReduceNHRV1::RunAllGatherOnHorizontal(const u32 rank, const std::vector<LINK>& links, const RingInfo& info)
307 : {
308 0 : u32 ringRank = info.GetHIndex(rank); // 查找自己位于第几列，也即处于Ring中的第几个rank
309 :
310 : // 计算allgather阶段每个rank结果上的offset和size
311 0 : std::vector<Slice> hSlices;
312 0 : std::vector<LINK> hLinks;
313 0 : CHK_RET(CalcHSlicesAndLinks(rank, links, info, hLinks, hSlices));
314 :
315 : // 长度不足2，直接跳过
316 0 : if (hLinks.size() < 2)
317 0 : return HCCL_SUCCESS;
318 :
319 0 : HCCL_DEBUG("[AllReduceNHRV1][AllGather-H] rank[%u] ringRank=%u, ringSize=%u", rank, ringRank, hLinks.size());
320 0 : return RunAllGatherBrokenRing(ringRank, hLinks, hSlices);
321 0 : }
322 :
323 0 : HcclResult AllReduceNHRV1::RunReduceScatterBrokenRing(
324 : const u32 rank, const std::vector<LINK>& links, const std::vector<Slice>& slices)
325 : {
326 : std::unique_ptr<AlgTemplateBase> tempAlg
327 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_REDUCESCATTER_RING, dispatcher_);
328 0 : CHK_SMART_PTR_NULL(tempAlg);
329 0 : CHK_RET(tempAlg->Prepare(reduceAttr_));
330 :
331 : // 判断是否关闭reducescatter的barrier
332 0 : if (!barrierSwitchOn_) {
333 0 : tempAlg->CloseBarrier();
334 : }
335 :
336 : // 调用reducescatter ring的算法执行
337 0 : CHK_RET(tempAlg->Prepare(
338 : inputMem_, inputMem_, outputMem_, count_, dataType_, stream_, reductionOp_, root_, slices, baseOffset_));
339 :
340 0 : CHK_RET(tempAlg->RegisterProfiler(profilerInput_.planeID, profilerInput_.stage, profilerInput_.step, stream_));
341 :
342 0 : return tempAlg->RunAsync(rank, links.size(), links);
343 0 : }
344 :
345 : HcclResult
346 0 : AllReduceNHRV1::RunAllGatherBrokenRing(const u32 rank, const std::vector<LINK>& links, const std::vector<Slice>& slices)
347 : {
348 : std::unique_ptr<AlgTemplateBase> tempAlg
349 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_RING, dispatcher_);
350 0 : CHK_SMART_PTR_NULL(tempAlg);
351 : // 判断是否关闭allgather的barrier
352 0 : if (!barrierSwitchOn_) {
353 0 : tempAlg->CloseBarrier();
354 : }
355 :
356 : // 调用allgather ring的算法执行
357 0 : CHK_RET(tempAlg->Prepare(
358 : outputMem_, outputMem_, outputMem_, count_, dataType_, stream_, reductionOp_, root_, slices, baseOffset_));
359 :
360 0 : CHK_RET(tempAlg->RegisterProfiler(profilerInput_.planeID, profilerInput_.stage, profilerInput_.step, stream_));
361 :
362 0 : return tempAlg->RunAsync(rank, links.size(), links);
363 0 : }
364 : REGISTER_TEMPLATE(TemplateType::TEMPLATE_ALL_REDUCE_NHR_V1, AllReduceNHRV1);
365 : } // namespace hccl
|
