Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "task_overflow.h"
12 : #include "externalinput_pub.h"
13 :
14 : namespace hccl {
15 :
16 : constexpr u32 DUMPTASK_COUNT_UPPER_LIMIT = 1000; // 算子溢出的task最大值为1000，防止内存占用量过大
17 : constexpr u32 SUBTASK_DEFAULT_VALUE = 10; // subTaskType默认参数
18 : constexpr u32 REDUCE_INLINE = 0;
19 : constexpr u32 REDUCE_TBE = 1;
20 :
21 312 : TaskOverflow::TaskOverflow(u32 deviceLogicId) : ProfilerBase(deviceLogicId) {}
22 624 : TaskOverflow::~TaskOverflow() {}
23 :
24 0 : uint32_t TaskOverflow::GetTaskName(TaskType taskType) const
25 : {
26 0 : uint32_t taskNameId = SUBTASK_DEFAULT_VALUE;
27 0 : switch (taskType) {
28 0 : case TaskType::TASK_SDMA:
29 : case TaskType::TASK_RDMA:
30 : case TaskType::TASK_NOTIFY_RECORD:
31 : case TaskType::TASK_NOTIFY_WAIT:
32 0 : break;
33 0 : case TaskType::TASK_REDUCE_INLINE:
34 0 : taskNameId = REDUCE_INLINE;
35 0 : break;
36 0 : case TaskType::TASK_REDUCE_TBE:
37 0 : taskNameId = REDUCE_TBE;
38 0 : break;
39 0 : default:
40 0 : break;
41 : }
42 0 : return taskNameId;
43 : }
44 :
45 9 : HcclResult TaskOverflow::Save(
46 : [[maybe_unused]] u32 captureStreamID, u32 streamID, u32 taskID, TaskType& taskType,
47 : const TaskParaReduce& paraReduce)
48 : {
49 9 : if (GetExternalInputHcclDumpDebug()) {
50 0 : HCCL_DEBUG("HcclDumpInfo save start");
51 0 : HcclDumpInfo hcclDumpInfo{};
52 0 : hcclDumpInfo.task_id = taskID;
53 0 : hcclDumpInfo.stream_id = streamID;
54 0 : hcclDumpInfo.output_addr = const_cast<void*>(paraReduce.dst);
55 0 : hcclDumpInfo.output_size = static_cast<u64>(paraReduce.size);
56 0 : hcclDumpInfo.input_addr = const_cast<void*>(paraReduce.src);
57 0 : hcclDumpInfo.input_size = static_cast<u64>(paraReduce.size);
58 0 : hcclDumpInfo.sub_task_type = GetTaskName(taskType);
59 :
60 0 : std::unique_lock<std::mutex> lock(dumpInfoVetcorMutex_);
61 : // 防止上层未及时清理dumpInfoVetcor_信息，导致内存占满。
62 0 : if (dumpInfoVetcor_.size() < DUMPTASK_COUNT_UPPER_LIMIT) {
63 0 : dumpInfoVetcor_.push_back(hcclDumpInfo);
64 0 : return HCCL_SUCCESS;
65 : }
66 0 : }
67 9 : return HCCL_SUCCESS;
68 : }
69 :
70 9 : HcclResult TaskOverflow::Save(u32& streamID, u32& taskID, TaskType& taskType, const TaskParaReduce& paraReduce)
71 : {
72 9 : return Save(streamID, streamID, taskID, taskType, paraReduce);
73 : }
74 :
75 0 : HcclResult TaskOverflow::Save(
76 : [[maybe_unused]] u32 captureStreamID, [[maybe_unused]] u32 streamID, [[maybe_unused]] u32 taskID,
77 : [[maybe_unused]] TaskType& taskType, [[maybe_unused]] const TaskParaDMA& paraDMA)
78 : {
79 0 : return HCCL_SUCCESS;
80 : }
81 :
82 32 : HcclResult TaskOverflow::Save(
83 : [[maybe_unused]] u32& streamID, [[maybe_unused]] u32& taskID, [[maybe_unused]] TaskType& taskType,
84 : [[maybe_unused]] const TaskParaDMA& paraDMA)
85 : {
86 32 : return HCCL_SUCCESS;
87 : }
88 :
89 0 : HcclResult TaskOverflow::Save(
90 : [[maybe_unused]] u32 captureStreamID, [[maybe_unused]] u32 streamID, [[maybe_unused]] u32 taskID,
91 : [[maybe_unused]] TaskType& taskType, [[maybe_unused]] const TaskParaNotify& paraNotify)
92 : {
93 0 : return HCCL_SUCCESS;
94 : }
95 :
96 0 : HcclResult TaskOverflow::Save(
97 : [[maybe_unused]] u32& streamID, [[maybe_unused]] u32& taskID, [[maybe_unused]] TaskType& taskType,
98 : [[maybe_unused]] const TaskParaNotify& paraNotify)
99 : {
100 0 : return HCCL_SUCCESS;
101 : }
102 :
103 0 : HcclResult TaskOverflow::Save(
104 : [[maybe_unused]] u32 captureStreamID, [[maybe_unused]] u32 streamID, [[maybe_unused]] u32 taskID,
105 : [[maybe_unused]] const void* descBuf, [[maybe_unused]] size_t descBufLen)
106 : {
107 0 : return HCCL_SUCCESS;
108 : }
109 :
110 3 : HcclResult TaskOverflow::Save(
111 : [[maybe_unused]] u32 captureStreamID, [[maybe_unused]] u32 streamID, [[maybe_unused]] u32 taskID,
112 : [[maybe_unused]] const TaskParaAiv& paraAiv)
113 : {
114 3 : return HCCL_SUCCESS;
115 : }
116 :
117 2 : HcclResult TaskOverflow::Save(
118 : [[maybe_unused]] u32 streamID, [[maybe_unused]] u32 taskID, [[maybe_unused]] const TaskParaAiv& paraAiv)
119 : {
120 2 : return HCCL_SUCCESS;
121 : }
122 :
123 0 : HcclResult TaskOverflow::Save(
124 : [[maybe_unused]] u32& streamID, [[maybe_unused]] u32& taskID, [[maybe_unused]] const void* descBuf,
125 : [[maybe_unused]] size_t descBufLen)
126 : {
127 0 : return HCCL_SUCCESS;
128 : }
129 :
130 0 : HcclResult TaskOverflow::GetandClearOverFlowTasks(std::vector<HcclDumpInfo>& hcclDumpInfoVector)
131 : {
132 0 : hcclDumpInfoVector.assign(dumpInfoVetcor_.begin(), dumpInfoVetcor_.end());
133 0 : dumpInfoVetcor_.clear();
134 :
135 0 : return HCCL_SUCCESS;
136 : }
137 :
138 0 : HcclResult TaskOverflow::Flush() { return HCCL_SUCCESS; }
139 :
140 0 : HcclResult TaskOverflow::Run([[maybe_unused]] const StepData& stepData) { return HCCL_SUCCESS; }
141 :
142 0 : HcclResult TaskOverflow::SaveToLog([[maybe_unused]] const TaskParaHost& paraHost) { return HCCL_SUCCESS; }
143 : } // namespace hccl
|
