Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "dfx/endpoint_monitor.h"
12 : #include "hcom_common.h"
13 :
14 : #include <array>
15 : #include <memory>
16 :
17 : namespace hcomm {
18 :
19 : constexpr u32 EndpointMonitor::MONITOR_INTERVAL;
20 :
21 : namespace {
22 38 : uint32_t MonitorSlot(s32 deviceLogicId)
23 : {
24 38 : if ((deviceLogicId < 0) || (static_cast<u32>(deviceLogicId) > MAX_MODULE_DEVICE_NUM)) {
25 0 : HCCL_ERROR("[EndpointMonitor] deviceLogicId[%d] not in range [0,%u]", deviceLogicId, MAX_MODULE_DEVICE_NUM);
26 0 : return MAX_MODULE_DEVICE_NUM;
27 : }
28 38 : return static_cast<u32>(deviceLogicId);
29 : }
30 :
31 2 : std::array<std::shared_ptr<EndpointMonitor>, MAX_MODULE_DEVICE_NUM + 1> MakeMonitorHolders()
32 : {
33 2 : std::array<std::shared_ptr<EndpointMonitor>, MAX_MODULE_DEVICE_NUM + 1> holders;
34 134 : for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; ++i) {
35 132 : holders[i] = std::make_shared<EndpointMonitor>();
36 : }
37 2 : return holders;
38 0 : }
39 : } // namespace
40 :
41 132 : EndpointMonitor::~EndpointMonitor() { DeInit(deviceLogicId_); }
42 :
43 38 : std::shared_ptr<EndpointMonitor> EndpointMonitor::GetHolder(s32 deviceId)
44 : {
45 38 : static auto holders = MakeMonitorHolders();
46 38 : return holders[MonitorSlot(deviceId)];
47 : }
48 :
49 8 : HcclResult EndpointMonitor::RegisterToEndpointMonitor(s32 deviceLogicId, EndpointHandle epHandle)
50 : {
51 8 : if ((deviceLogicId < 0) || (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM)) {
52 2 : HCCL_ERROR(
53 : "[EndpointMonitor][%s] deviceLogicId[%d] not in range [0,%u)", __func__, deviceLogicId,
54 : MAX_MODULE_DEVICE_NUM);
55 2 : return HCCL_E_PARA;
56 : }
57 6 : CHK_PRT_RET(epHandle == nullptr, HCCL_ERROR("[EndpointMonitor][%s] epHandle is null", __func__), HCCL_E_PTR);
58 :
59 5 : HCCL_INFO(
60 : "[EndpointMonitor] deviceLogicId[%d] epHandle[%p] RegisterToEndpointMonitor begin.", deviceLogicId, epHandle);
61 5 : u32 devPhyId{0};
62 5 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId), devPhyId));
63 :
64 : {
65 5 : std::lock_guard<std::mutex> lock(threadLock_);
66 5 : epHandleSet_.emplace(reinterpret_cast<u64>(epHandle));
67 5 : if (!initialized_) {
68 1 : deviceLogicId_ = deviceLogicId;
69 1 : devPhyId_ = devPhyId;
70 1 : CHK_RET(RunMonitorThread());
71 : }
72 5 : }
73 :
74 5 : HCCL_INFO(
75 : "[EndpointMonitor] deviceLogicId[%d] epHandle[%p] RegisterToEndpointMonitor Completed.", deviceLogicId,
76 : epHandle);
77 5 : return HCCL_SUCCESS;
78 : }
79 :
80 1 : HcclResult EndpointMonitor::RunMonitorThread()
81 : {
82 1 : HCCL_INFO("[EndpointMonitor][%s] deviceLogicId[%d] Start Thread.", __func__, deviceLogicId_);
83 1 : endpointMonitorThreadFlag_ = true;
84 1 : EXCEPTION_CATCH(
85 : endpointMonitorThread_ = std::make_unique<std::thread>(&EndpointMonitor::MonitorThread, this),
86 : return HCCL_E_INTERNAL);
87 1 : CHK_SMART_PTR_NULL(endpointMonitorThread_);
88 1 : initialized_ = true;
89 1 : return HCCL_SUCCESS;
90 : }
91 :
92 1 : void EndpointMonitor::MonitorThread()
93 : {
94 1 : SetThreadName("Hccl_Ub_Event_Monitor");
95 :
96 1 : HcclResult ret = hrtSetDevice(deviceLogicId_);
97 1 : if (ret != HCCL_SUCCESS) {
98 0 : HCCL_ERROR(
99 : "[EndpointMonitor][%s] hrtSetDevice failed, deviceLogicId[%d], ret[%d]", __func__, deviceLogicId_, ret);
100 0 : return;
101 : }
102 :
103 5 : while (endpointMonitorThreadFlag_) {
104 4 : ProcessUbAsyncEvents();
105 4 : std::this_thread::sleep_for(std::chrono::milliseconds(MONITOR_INTERVAL));
106 : }
107 :
108 1 : ret = hrtResetDevice(deviceLogicId_);
109 1 : if (ret != HCCL_SUCCESS) {
110 0 : HCCL_ERROR(
111 : "[EndpointMonitor][%s] hrtResetDevice failed, deviceLogicId[%d], ret[%d]", __func__, deviceLogicId_, ret);
112 : }
113 : }
114 :
115 1 : HcclResult EndpointMonitor::UnRegisterToEndpointMonitor()
116 : {
117 1 : s32 deviceLogicId = deviceLogicId_;
118 1 : HCCL_INFO("[EndpointMonitor] deviceId[%d] UnRegisterToEndpointMonitor begin.", deviceLogicId);
119 : {
120 1 : std::lock_guard<std::mutex> lock(threadLock_);
121 1 : CHK_PRT_RET(
122 : !initialized_,
123 : HCCL_WARNING(
124 : "[EndpointMonitor] deviceId[%d] hcclUbEventMonitor has been destroyed, or not initialized",
125 : deviceLogicId),
126 : HCCL_SUCCESS);
127 1 : epHandleSet_.clear();
128 1 : }
129 :
130 1 : CHK_RET(DeInit(deviceLogicId_));
131 1 : deviceLogicId_ = 0;
132 1 : devPhyId_ = 0;
133 1 : initialized_ = false;
134 :
135 1 : HCCL_INFO("[EndpointMonitor] deviceId[%d] UnRegisterToEndpointMonitor completed.", deviceLogicId);
136 1 : return HCCL_SUCCESS;
137 : }
138 :
139 25 : void EndpointMonitor::RemoveEpHandleFromEndpointMonitor(EndpointHandle epHandle)
140 : {
141 25 : if (epHandle == nullptr) {
142 1 : HCCL_ERROR("[EndpointMonitor][%s] epHandle is null", __func__);
143 1 : return;
144 : }
145 :
146 : {
147 24 : std::lock_guard<std::mutex> lock(threadLock_);
148 24 : auto it = epHandleSet_.find(reinterpret_cast<u64>(epHandle));
149 24 : if (it != epHandleSet_.end()) {
150 5 : epHandleSet_.erase(it);
151 5 : HCCL_INFO(
152 : "[EndpointMonitor][%s] epHandle[%p] is remove from deviceId[%d]", __func__, epHandle, deviceLogicId_);
153 : }
154 24 : }
155 : }
156 :
157 133 : HcclResult EndpointMonitor::DeInit(s32 deviceLogicId)
158 : {
159 133 : endpointMonitorThreadFlag_ = false;
160 133 : if (endpointMonitorThread_) {
161 1 : if (endpointMonitorThread_->joinable()) {
162 : try {
163 1 : HCCL_INFO("[EndpointMonitor][%s] deviceId[%d] thread join", __func__, deviceLogicId);
164 1 : endpointMonitorThread_->join();
165 1 : endpointMonitorThread_.reset();
166 0 : } catch (const std::exception& e) {
167 0 : HCCL_ERROR("[EndpointMonitor][%s] deviceId[%d] join failed: %s", __func__, deviceLogicId, e.what());
168 0 : return HCCL_E_INTERNAL;
169 0 : }
170 : }
171 : }
172 133 : return HCCL_SUCCESS;
173 : }
174 :
175 8 : void EndpointMonitor::ProcessUbAsyncEvents()
176 : {
177 8 : std::lock_guard<std::mutex> lock(threadLock_);
178 11 : for (auto it = epHandleSet_.begin(); it != epHandleSet_.end();) {
179 3 : u32 num = ASYNC_EVENT_MAX_NUM;
180 3 : Endpoint* localEpPtr = reinterpret_cast<Endpoint*>(*it);
181 3 : HcclResult ret = localEpPtr->GetAsyncEvents(devPhyId_, events_, num);
182 3 : if (ret != HCCL_SUCCESS) {
183 2 : it = epHandleSet_.erase(it);
184 2 : HCCL_ERROR(
185 : "[EndpointMonitor][%s] deviceId[%d] HcommGetAsyncEvents failed ret[%d], "
186 : "epHandle[%p] removed from monitor",
187 : __func__, deviceLogicId_, ret, localEpPtr);
188 2 : continue;
189 : }
190 :
191 5 : for (u32 i = 0; i < num; ++i) {
192 4 : PrintUbAsyncEventsContext(static_cast<void*>(localEpPtr), i, events_[i]);
193 : }
194 :
195 1 : ++it;
196 : }
197 8 : }
198 :
199 : constexpr u32 SECOND_LAST_OFFSET = 2; // 倒数第二个字节偏移
200 : constexpr u32 LAST_OFFSET = 3; // 倒数第一个字节偏移
201 2 : void EndpointMonitor::PrintUbAsyncEventsContext(void* epHandle, u32 seq, const struct AsyncEvent& event)
202 : {
203 2 : u32 contextLen = event.len;
204 2 : if (contextLen > CONTEXT_MAX_LEN) {
205 1 : HCCL_ERROR(
206 : "[EndpointMonitor][%s] deviceId[%d] epHandle[%p] seq[%u] context len[%u] exceed max[%u]", __func__,
207 : deviceLogicId_, epHandle, seq, contextLen, CONTEXT_MAX_LEN);
208 1 : return;
209 : }
210 :
211 1 : HCCL_ERROR("************************************** ub async event **************************************");
212 1 : HCCL_ERROR(
213 : "deviceId[%d] epHandle[%p] seq[%u] resId[%u] eventType[%u] contextLen[%u]", deviceLogicId_, epHandle, seq,
214 : event.resId, event.eventType, event.len);
215 1 : if (contextLen != 0) {
216 1 : HCCL_ERROR("bytes order: high -> low");
217 : }
218 1 : constexpr u32 bytesPerLine = 4;
219 4 : for (u32 i = 0; i < contextLen; i += bytesPerLine) {
220 3 : u32 endIndex = std::min(i + bytesPerLine, contextLen);
221 3 : HCCL_ERROR(
222 : "context[byte %3u]: %02x%02x%02x%02x", endIndex,
223 : (i + LAST_OFFSET < contextLen) ? event.context[i + LAST_OFFSET] : 0,
224 : (i + SECOND_LAST_OFFSET < contextLen) ? event.context[i + SECOND_LAST_OFFSET] : 0,
225 : (i + 1 < contextLen) ? event.context[i + 1] : 0, event.context[i]);
226 : }
227 1 : HCCL_ERROR("********************************************************************************************");
228 : }
229 :
230 : } // namespace hcomm
|
