Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "coll_scatter_mesh_executor.h"
12 :
13 : namespace hccl {
14 4 : CollScatterMeshExecutor::CollScatterMeshExecutor(
15 4 : const HcclDispatcher dispatcher, std::unique_ptr<TopoMatcher>& topoMatcher)
16 4 : : CollScatterExecutor(dispatcher, topoMatcher)
17 4 : {}
18 :
19 4 : HcclResult CollScatterMeshExecutor::CalcLevel0CommInfo(
20 : TransportMemType inputType, TransportMemType outputType, std::vector<LevelNSubCommTransport>& opTransport)
21 : {
22 4 : HCCL_INFO("[CollScatterMeshExecutor][CalcLevel0CommInfo]tag[%s] start", tag_.c_str());
23 4 : CommParaInfo commParaLevel0(COMM_LEVEL0, CommType::COMM_TAG_MESH);
24 :
25 4 : CHK_RET(CalcCommPlaneInfo(tag_, commParaLevel0, opTransport[COMM_LEVEL0], inputType, outputType));
26 4 : HCCL_INFO("[CollScatterMeshExecutor][CalcLevel0CommInfo]tag[%s] Calc meshComm finish", tag_.c_str());
27 4 : return HCCL_SUCCESS;
28 4 : }
29 :
30 4 : HcclResult CollScatterMeshExecutor::CalcStreamNum(u32& streamNum)
31 : {
32 4 : u32 totalStreamNum = topoAttr_.deviceNumPerAggregation > 1U ? topoAttr_.deviceNumPerAggregation - 1U : 1U;
33 4 : streamNum = totalStreamNum - 1U;
34 4 : return HCCL_SUCCESS;
35 : }
36 :
37 0 : HcclResult CollScatterMeshExecutor::KernelRun(const OpParam& param, ExecMem& execMem)
38 : {
39 0 : HCCL_CONFIG_INFO(HCCL_ALG, "[CollScatterMeshExecutor] scatter starts.");
40 0 : Stream& stream = const_cast<Stream&>(param.stream);
41 :
42 0 : u32 perDataSize = SIZE_TABLE[param.DataDes.dataType];
43 :
44 0 : SubCommInfo level0CommInfo = GetSubCommInfo(COMM_LEVEL0, COMM_INDEX_0);
45 0 : u32 level0LocalRank = level0CommInfo.localRank;
46 0 : u32 level0LocalRankSize = level0CommInfo.localRankSize;
47 :
48 0 : u32 commIndex = level0LocalRank;
49 :
50 0 : CHK_RET(CheckCommSize(COMM_LEVEL1, commIndex + 1));
51 0 : SubCommInfo level1CommInfo = GetSubCommInfo(COMM_LEVEL1, commIndex);
52 0 : u32 level1LocalRank = level1CommInfo.localRank;
53 0 : u32 level1LocalRankSize = level1CommInfo.localRankSize;
54 :
55 0 : bool bRet = level0LocalRankSize == 0;
56 0 : CHK_PRT_RET(
57 : bRet, HCCL_ERROR("[CollScatterMeshExecutor][KernelRun]tag[%s],comm level0 is empty", tag_.c_str()),
58 : HCCL_E_INTERNAL);
59 :
60 : /* ***********第一步: 节点间scatter ****************************/
61 0 : u32 subRoot = INVALID_VALUE_RANKID;
62 0 : CHK_RET(topoMatcher_->GetSubRootForScatter(param.root, subRoot));
63 0 : CHK_PRT_RET(
64 : subRoot == INVALID_VALUE_RANKID,
65 : HCCL_ERROR(
66 : "[CollScatterMeshExecutor][KernelRun]GetSubRootForScatter failed, "
67 : "userRank[%u], root[%u], subRoot[%u]",
68 : topoAttr_.userRank, param.root, subRoot),
69 : HCCL_E_INTERNAL);
70 0 : HCCL_DEBUG(
71 : "[CollScatterMeshExecutor][KernelRun]GetSubRootForScatter, userRank[%u], root[%u], subRoot[%u]",
72 : topoAttr_.userRank, param.root, subRoot);
73 0 : CHK_RET(KernelRunLevel1(
74 : execMem.inputMem, execMem.count, param.DataDes.dataType, commIndex, param.root, subRoot, COMM_LEVEL1, stream));
75 :
76 : /* ***********第二步: 节点内scatter*****************************/
77 : // 根据数据量算每个环上数据的偏移和大小
78 0 : u32 sliceNum = level0LocalRankSize;
79 0 : std::vector<Slice> dataSegsSlice;
80 0 : CHK_RET(PrepareDataSlice(execMem.count, perDataSize, sliceNum, dataSegsSlice));
81 :
82 : // 每个server分配的slice大小
83 0 : u64 serverSliceSize = execMem.inputMem.size() / level1LocalRankSize;
84 : // 每个服务器对应的偏移
85 0 : u64 serverSliceOffset = serverSliceSize * level1LocalRank;
86 0 : DeviceMem scatterMeshInput = execMem.inputMem.range(serverSliceOffset, serverSliceSize);
87 0 : CHK_SMART_PTR_NULL(scatterMeshInput);
88 0 : DeviceMem scatterMeshOutput = execMem.inputMem.range(serverSliceOffset, serverSliceSize);
89 0 : CHK_SMART_PTR_NULL(scatterMeshOutput);
90 :
91 : std::unique_ptr<AlgTemplateBase> level0TempAlg
92 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_SCATTER_MESH, dispatcher_);
93 0 : CHK_SMART_PTR_NULL(level0TempAlg);
94 0 : CHK_RET(level0TempAlg->Prepare(level0LocalRank, level0LocalRankSize));
95 : // 偏移需要带入prepare
96 0 : u32 rootRankLevel0 = 0;
97 0 : CHK_RET(GetRankByUserRank(COMM_LEVEL0, COMM_INDEX_0, subRoot, rootRankLevel0));
98 0 : CHK_PRT_RET(
99 : rootRankLevel0 == INVALID_VALUE_RANKID,
100 : HCCL_ERROR(
101 : "[CollScatterMeshExecutor][KernelRun]rootRankLevel0[%u] is invalid, userRank[%u], subRoot[%u]",
102 : rootRankLevel0, topoAttr_.userRank, subRoot),
103 : HCCL_E_INTERNAL);
104 :
105 0 : CHK_RET(level0TempAlg->Prepare(
106 : scatterMeshInput, scatterMeshOutput, execMem.inputMem, execMem.count, param.DataDes.dataType, stream,
107 : HCCL_REDUCE_RESERVED, rootRankLevel0, dataSegsSlice, serverSliceOffset));
108 :
109 0 : HcclResult ret = RunTemplate(level0TempAlg, level0CommInfo);
110 0 : CHK_PRT_RET(
111 : ret != HCCL_SUCCESS,
112 : HCCL_ERROR("[CollScatterMeshExecutor][KernelRun]scatter(mesh) RunTemplate failed,return[%d]", ret), ret);
113 :
114 : // 将scratchMem赋值给outputMem
115 0 : u8* scatterMeshOutputPtr = static_cast<u8*>(scatterMeshOutput.ptr());
116 0 : DeviceMem resultMem(scatterMeshOutputPtr + execMem.outputMem.size() * level0LocalRank, execMem.outputMem.size());
117 0 : CHK_RET(HcclD2DMemcpyAsync(dispatcher_, execMem.outputMem, resultMem, stream));
118 0 : return HCCL_SUCCESS;
119 0 : }
120 :
121 : REGISTER_EXEC("ScatterMeshExecutor", ScatterMesh, CollScatterMeshExecutor);
122 :
123 : } // namespace hccl
|
