Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "dfx/endpoint_monitor.h"
12 : #include "hcom_common.h"
13 :
14 : namespace hcomm {
15 :
16 : constexpr u32 EndpointMonitor::MONITOR_INTERVAL;
17 :
18 264 : EndpointMonitor::~EndpointMonitor() { DeInit(deviceLogicId_); }
19 :
20 76 : EndpointMonitor& EndpointMonitor::GetInstance(s32 deviceLogicId)
21 : {
22 76 : static std::array<EndpointMonitor, MAX_MODULE_DEVICE_NUM + 1> instances;
23 : uint32_t deviceId;
24 76 : if ((deviceLogicId < 0) || (static_cast<u32>(deviceLogicId) > MAX_MODULE_DEVICE_NUM)) {
25 0 : HCCL_ERROR(
26 : "[EndpointMonitor][%s] deviceLogicId[%d] not in range [0,%u]", __func__, deviceLogicId,
27 : MAX_MODULE_DEVICE_NUM);
28 0 : deviceId = MAX_MODULE_DEVICE_NUM;
29 : } else {
30 76 : deviceId = static_cast<u32>(deviceLogicId);
31 : }
32 76 : return instances[deviceId];
33 : }
34 :
35 8 : HcclResult EndpointMonitor::RegisterToEndpointMonitor(s32 deviceLogicId, EndpointHandle epHandle)
36 : {
37 8 : if ((deviceLogicId < 0) || (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM)) {
38 2 : HCCL_ERROR(
39 : "[EndpointMonitor][%s] deviceLogicId[%d] not in range [0,%u)", __func__, deviceLogicId,
40 : MAX_MODULE_DEVICE_NUM);
41 2 : return HCCL_E_PARA;
42 : }
43 6 : CHK_PRT_RET(epHandle == nullptr, HCCL_ERROR("[EndpointMonitor][%s] epHandle is null", __func__), HCCL_E_PTR);
44 :
45 5 : HCCL_INFO(
46 : "[EndpointMonitor] deviceLogicId[%d] epHandle[%p] RegisterToEndpointMonitor begin.", deviceLogicId, epHandle);
47 5 : u32 devPhyId{0};
48 5 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId), devPhyId));
49 :
50 : {
51 5 : std::lock_guard<std::mutex> lock(threadLock_);
52 5 : epHandleSet_.emplace(reinterpret_cast<u64>(epHandle));
53 5 : if (!initialized_) {
54 1 : deviceLogicId_ = deviceLogicId;
55 1 : devPhyId_ = devPhyId;
56 1 : CHK_RET(RunMonitorThread());
57 : }
58 5 : }
59 :
60 5 : HCCL_INFO(
61 : "[EndpointMonitor] deviceLogicId[%d] epHandle[%p] RegisterToEndpointMonitor Completed.", deviceLogicId,
62 : epHandle);
63 5 : return HCCL_SUCCESS;
64 : }
65 :
66 1 : HcclResult EndpointMonitor::RunMonitorThread()
67 : {
68 1 : HCCL_INFO("[EndpointMonitor][%s] deviceLogicId[%d] Start Thread.", __func__, deviceLogicId_);
69 1 : endpointMonitorThreadFlag_ = true;
70 1 : EXCEPTION_CATCH(
71 : endpointMonitorThread_ = std::make_unique<std::thread>(&EndpointMonitor::MonitorThread, this),
72 : return HCCL_E_INTERNAL);
73 1 : CHK_SMART_PTR_NULL(endpointMonitorThread_);
74 1 : initialized_ = true;
75 1 : return HCCL_SUCCESS;
76 : }
77 :
78 1 : void EndpointMonitor::MonitorThread()
79 : {
80 1 : SetThreadName("Hccl_Ub_Event_Monitor");
81 :
82 1 : HcclResult ret = hrtSetDevice(deviceLogicId_);
83 1 : if (ret != HCCL_SUCCESS) {
84 0 : HCCL_ERROR(
85 : "[EndpointMonitor][%s] hrtSetDevice failed, deviceLogicId[%d], ret[%d]", __func__, deviceLogicId_, ret);
86 0 : return;
87 : }
88 :
89 5 : while (endpointMonitorThreadFlag_) {
90 4 : ProcessUbAsyncEvents();
91 4 : std::this_thread::sleep_for(std::chrono::milliseconds(MONITOR_INTERVAL));
92 : }
93 :
94 1 : ret = hrtResetDevice(deviceLogicId_);
95 1 : if (ret != HCCL_SUCCESS) {
96 0 : HCCL_ERROR(
97 : "[EndpointMonitor][%s] hrtResetDevice failed, deviceLogicId[%d], ret[%d]", __func__, deviceLogicId_, ret);
98 : }
99 : }
100 :
101 1 : HcclResult EndpointMonitor::UnRegisterToEndpointMonitor()
102 : {
103 1 : s32 deviceLogicId = deviceLogicId_;
104 1 : HCCL_INFO("[EndpointMonitor] deviceId[%d] UnRegisterToEndpointMonitor begin.", deviceLogicId);
105 : {
106 1 : std::lock_guard<std::mutex> lock(threadLock_);
107 1 : CHK_PRT_RET(
108 : !initialized_,
109 : HCCL_WARNING(
110 : "[EndpointMonitor] deviceId[%d] hcclUbEventMonitor has been destroyed, or not initialized",
111 : deviceLogicId),
112 : HCCL_SUCCESS);
113 1 : epHandleSet_.clear();
114 1 : }
115 :
116 1 : CHK_RET(DeInit(deviceLogicId_));
117 1 : deviceLogicId_ = 0;
118 1 : devPhyId_ = 0;
119 1 : initialized_ = false;
120 :
121 1 : HCCL_INFO("[EndpointMonitor] deviceId[%d] UnRegisterToEndpointMonitor completed.", deviceLogicId);
122 1 : return HCCL_SUCCESS;
123 : }
124 :
125 53 : void EndpointMonitor::RemoveEpHandleFromEndpointMonitor(EndpointHandle epHandle)
126 : {
127 53 : if (epHandle == nullptr) {
128 2 : HCCL_ERROR("[EndpointMonitor][%s] epHandle is null", __func__);
129 2 : return;
130 : }
131 :
132 : {
133 51 : std::lock_guard<std::mutex> lock(threadLock_);
134 51 : auto it = epHandleSet_.find(reinterpret_cast<u64>(epHandle));
135 51 : if (it != epHandleSet_.end()) {
136 4 : epHandleSet_.erase(it);
137 4 : HCCL_INFO(
138 : "[EndpointMonitor][%s] epHandle[%p] is remove from deviceId[%d]", __func__, epHandle, deviceLogicId_);
139 : }
140 51 : }
141 : }
142 :
143 265 : HcclResult EndpointMonitor::DeInit(s32 deviceLogicId)
144 : {
145 265 : endpointMonitorThreadFlag_ = false;
146 265 : if (endpointMonitorThread_) {
147 1 : if (endpointMonitorThread_->joinable()) {
148 : try {
149 1 : HCCL_INFO("[EndpointMonitor][%s] deviceId[%d] thread join", __func__, deviceLogicId);
150 1 : endpointMonitorThread_->join();
151 1 : endpointMonitorThread_.reset();
152 0 : } catch (const std::exception& e) {
153 0 : HCCL_ERROR("[EndpointMonitor][%s] deviceId[%d] join failed: %s", __func__, deviceLogicId, e.what());
154 0 : return HCCL_E_INTERNAL;
155 0 : }
156 : }
157 : }
158 265 : return HCCL_SUCCESS;
159 : }
160 :
161 8 : void EndpointMonitor::ProcessUbAsyncEvents()
162 : {
163 8 : std::lock_guard<std::mutex> lock(threadLock_);
164 11 : for (auto it = epHandleSet_.begin(); it != epHandleSet_.end();) {
165 3 : u32 num = ASYNC_EVENT_MAX_NUM;
166 3 : Endpoint* localEpPtr = reinterpret_cast<Endpoint*>(*it);
167 3 : HcclResult ret = localEpPtr->GetAsyncEvents(devPhyId_, events_, num);
168 3 : if (ret != HCCL_SUCCESS) {
169 2 : it = epHandleSet_.erase(it);
170 2 : HCCL_ERROR(
171 : "[EndpointMonitor][%s] deviceId[%d] HcommGetAsyncEvents failed ret[%d], "
172 : "epHandle[%p] removed from monitor",
173 : __func__, deviceLogicId_, ret, localEpPtr);
174 2 : continue;
175 : }
176 :
177 5 : for (u32 i = 0; i < num; ++i) {
178 4 : PrintUbAsyncEventsContext(static_cast<void*>(localEpPtr), i, events_[i]);
179 : }
180 :
181 1 : ++it;
182 : }
183 8 : }
184 :
185 : constexpr u32 SECOND_LAST_OFFSET = 2; // 倒数第二个字节偏移
186 : constexpr u32 LAST_OFFSET = 3; // 倒数第一个字节偏移
187 2 : void EndpointMonitor::PrintUbAsyncEventsContext(void* epHandle, u32 seq, const struct AsyncEvent& event)
188 : {
189 2 : u32 contextLen = event.len;
190 2 : if (contextLen > CONTEXT_MAX_LEN) {
191 1 : HCCL_ERROR(
192 : "[EndpointMonitor][%s] deviceId[%d] epHandle[%p] seq[%u] context len[%u] exceed max[%u]", __func__,
193 : deviceLogicId_, epHandle, seq, contextLen, CONTEXT_MAX_LEN);
194 1 : return;
195 : }
196 :
197 1 : HCCL_ERROR("************************************** ub async event **************************************");
198 1 : HCCL_ERROR(
199 : "deviceId[%d] epHandle[%p] seq[%u] resId[%u] eventType[%u] contextLen[%u]", deviceLogicId_, epHandle, seq,
200 : event.resId, event.eventType, event.len);
201 1 : if (contextLen != 0) {
202 1 : HCCL_ERROR("bytes order: high -> low");
203 : }
204 1 : constexpr u32 bytesPerLine = 4;
205 4 : for (u32 i = 0; i < contextLen; i += bytesPerLine) {
206 3 : u32 endIndex = std::min(i + bytesPerLine, contextLen);
207 3 : HCCL_ERROR(
208 : "context[byte %3u]: %02x%02x%02x%02x", endIndex,
209 : (i + LAST_OFFSET < contextLen) ? event.context[i + LAST_OFFSET] : 0,
210 : (i + SECOND_LAST_OFFSET < contextLen) ? event.context[i + SECOND_LAST_OFFSET] : 0,
211 : (i + 1 < contextLen) ? event.context[i + 1] : 0, event.context[i]);
212 : }
213 1 : HCCL_ERROR("********************************************************************************************");
214 : }
215 :
216 : } // namespace hcomm
|
