Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "dfx/endpoint_monitor.h"
11 : #include "hcom_common.h"
12 :
13 : namespace hcomm {
14 :
15 : constexpr u32 EndpointMonitor::MONITOR_INTERVAL;
16 :
17 264 : EndpointMonitor::~EndpointMonitor() { DeInit(deviceLogicId_); }
18 :
19 72 : EndpointMonitor& EndpointMonitor::GetInstance(s32 deviceLogicId)
20 : {
21 72 : static std::array<EndpointMonitor, MAX_MODULE_DEVICE_NUM + 1> instances;
22 : uint32_t deviceId;
23 72 : if ((deviceLogicId < 0) || (static_cast<u32>(deviceLogicId) > MAX_MODULE_DEVICE_NUM)) {
24 0 : HCCL_ERROR(
25 : "[EndpointMonitor][%s] deviceLogicId[%d] not in range [0,%u]", __func__, deviceLogicId,
26 : MAX_MODULE_DEVICE_NUM);
27 0 : deviceId = MAX_MODULE_DEVICE_NUM;
28 : } else {
29 72 : deviceId = static_cast<u32>(deviceLogicId);
30 : }
31 72 : return instances[deviceId];
32 : }
33 :
34 6 : HcclResult EndpointMonitor::RegisterToEndpointMonitor(s32 deviceLogicId, EndpointHandle epHandle)
35 : {
36 6 : if ((deviceLogicId < 0) || (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM)) {
37 2 : HCCL_ERROR(
38 : "[EndpointMonitor][%s] deviceLogicId[%d] not in range [0,%u)", __func__, deviceLogicId,
39 : MAX_MODULE_DEVICE_NUM);
40 2 : return HCCL_E_PARA;
41 : }
42 4 : CHK_PRT_RET(epHandle == nullptr, HCCL_ERROR("[EndpointMonitor][%s] epHandle is null", __func__), HCCL_E_PTR);
43 :
44 3 : HCCL_INFO(
45 : "[EndpointMonitor] deviceLogicId[%d] epHandle[%p] RegisterToEndpointMonitor begin.", deviceLogicId, epHandle);
46 3 : u32 devPhyId{0};
47 3 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId), devPhyId));
48 :
49 : {
50 3 : std::lock_guard<std::mutex> lock(threadLock_);
51 3 : epHandleSet_.emplace(reinterpret_cast<u64>(epHandle));
52 3 : if (!initialized_) {
53 1 : deviceLogicId_ = deviceLogicId;
54 1 : devPhyId_ = devPhyId;
55 1 : CHK_RET(RunMonitorThread());
56 : }
57 3 : }
58 :
59 3 : HCCL_INFO(
60 : "[EndpointMonitor] deviceLogicId[%d] epHandle[%p] RegisterToEndpointMonitor Completed.", deviceLogicId,
61 : epHandle);
62 3 : return HCCL_SUCCESS;
63 : }
64 :
65 1 : HcclResult EndpointMonitor::RunMonitorThread()
66 : {
67 1 : HCCL_INFO("[EndpointMonitor][%s] deviceLogicId[%d] Start Thread.", __func__, deviceLogicId_);
68 1 : endpointMonitorThreadFlag_ = true;
69 1 : EXCEPTION_CATCH(
70 : endpointMonitorThread_ = std::make_unique<std::thread>(&EndpointMonitor::MonitorThread, this),
71 : return HCCL_E_INTERNAL);
72 1 : CHK_SMART_PTR_NULL(endpointMonitorThread_);
73 1 : initialized_ = true;
74 1 : return HCCL_SUCCESS;
75 : }
76 :
77 1 : void EndpointMonitor::MonitorThread()
78 : {
79 1 : SetThreadName("Hccl_Ub_Event_Monitor");
80 :
81 1 : HcclResult ret = hrtSetDevice(deviceLogicId_);
82 1 : if (ret != HCCL_SUCCESS) {
83 0 : HCCL_ERROR(
84 : "[EndpointMonitor][%s] hrtSetDevice failed, deviceLogicId[%d], ret[%d]", __func__, deviceLogicId_, ret);
85 0 : return;
86 : }
87 :
88 5 : while (endpointMonitorThreadFlag_) {
89 4 : ProcessUbAsyncEvents();
90 4 : std::this_thread::sleep_for(std::chrono::milliseconds(MONITOR_INTERVAL));
91 : }
92 :
93 1 : ret = hrtResetDevice(deviceLogicId_);
94 1 : if (ret != HCCL_SUCCESS) {
95 0 : HCCL_ERROR(
96 : "[EndpointMonitor][%s] hrtResetDevice failed, deviceLogicId[%d], ret[%d]", __func__, deviceLogicId_, ret);
97 : }
98 : }
99 :
100 1 : HcclResult EndpointMonitor::UnRegisterToEndpointMonitor()
101 : {
102 1 : s32 deviceLogicId = deviceLogicId_;
103 1 : HCCL_INFO("[EndpointMonitor] deviceId[%d] UnRegisterToEndpointMonitor begin.", deviceLogicId);
104 : {
105 1 : std::lock_guard<std::mutex> lock(threadLock_);
106 1 : CHK_PRT_RET(
107 : !initialized_,
108 : HCCL_WARNING(
109 : "[EndpointMonitor] deviceId[%d] hcclUbEventMonitor has been destroyed, or not initialized",
110 : deviceLogicId),
111 : HCCL_SUCCESS);
112 1 : epHandleSet_.clear();
113 1 : }
114 :
115 1 : CHK_RET(DeInit(deviceLogicId_));
116 1 : deviceLogicId_ = 0;
117 1 : devPhyId_ = 0;
118 1 : initialized_ = false;
119 :
120 1 : HCCL_INFO("[EndpointMonitor] deviceId[%d] UnRegisterToEndpointMonitor completed.", deviceLogicId);
121 1 : return HCCL_SUCCESS;
122 : }
123 :
124 51 : void EndpointMonitor::RemoveEpHandleFromEndpointMonitor(EndpointHandle epHandle)
125 : {
126 51 : if (epHandle == nullptr) {
127 2 : HCCL_ERROR("[EndpointMonitor][%s] epHandle is null", __func__);
128 2 : return;
129 : }
130 :
131 : {
132 49 : std::lock_guard<std::mutex> lock(threadLock_);
133 49 : auto it = epHandleSet_.find(reinterpret_cast<u64>(epHandle));
134 49 : if (it != epHandleSet_.end()) {
135 2 : epHandleSet_.erase(it);
136 2 : HCCL_INFO(
137 : "[EndpointMonitor][%s] epHandle[%p] is remove from deviceId[%d]", __func__, epHandle, deviceLogicId_);
138 : }
139 49 : }
140 : }
141 :
142 265 : HcclResult EndpointMonitor::DeInit(s32 deviceLogicId)
143 : {
144 265 : endpointMonitorThreadFlag_ = false;
145 265 : if (endpointMonitorThread_) {
146 1 : if (endpointMonitorThread_->joinable()) {
147 : try {
148 1 : HCCL_INFO("[EndpointMonitor][%s] deviceId[%d] thread join", __func__, deviceLogicId);
149 1 : endpointMonitorThread_->join();
150 1 : endpointMonitorThread_.reset();
151 0 : } catch (const std::exception& e) {
152 0 : HCCL_ERROR("[EndpointMonitor][%s] deviceId[%d] join failed: %s", __func__, deviceLogicId, e.what());
153 0 : return HCCL_E_INTERNAL;
154 0 : }
155 : }
156 : }
157 265 : return HCCL_SUCCESS;
158 : }
159 :
160 8 : void EndpointMonitor::ProcessUbAsyncEvents()
161 : {
162 8 : std::lock_guard<std::mutex> lock(threadLock_);
163 11 : for (auto it = epHandleSet_.begin(); it != epHandleSet_.end();) {
164 3 : u32 num = ASYNC_EVENT_MAX_NUM;
165 3 : Endpoint* localEpPtr = reinterpret_cast<Endpoint*>(*it);
166 3 : HcclResult ret = localEpPtr->GetAsyncEvents(devPhyId_, events_, num);
167 3 : if (ret != HCCL_SUCCESS) {
168 2 : it = epHandleSet_.erase(it);
169 2 : HCCL_ERROR(
170 : "[EndpointMonitor][%s] deviceId[%d] HcommGetAsyncEvents failed ret[%d], "
171 : "epHandle[%p] removed from monitor",
172 : __func__, deviceLogicId_, ret, localEpPtr);
173 2 : continue;
174 : }
175 :
176 5 : for (u32 i = 0; i < num; ++i) {
177 4 : PrintUbAsyncEventsContext(static_cast<void*>(localEpPtr), i, events_[i]);
178 : }
179 :
180 1 : ++it;
181 : }
182 8 : }
183 :
184 : constexpr u32 SECOND_LAST_OFFSET = 2; // 倒数第二个字节偏移
185 : constexpr u32 LAST_OFFSET = 3; // 倒数第一个字节偏移
186 2 : void EndpointMonitor::PrintUbAsyncEventsContext(void* epHandle, u32 seq, const struct AsyncEvent& event)
187 : {
188 2 : u32 contextLen = event.len;
189 2 : if (contextLen > CONTEXT_MAX_LEN) {
190 1 : HCCL_ERROR(
191 : "[EndpointMonitor][%s] deviceId[%d] epHandle[%p] seq[%u] context len[%u] exceed max[%u]", __func__,
192 : deviceLogicId_, epHandle, seq, contextLen, CONTEXT_MAX_LEN);
193 1 : return;
194 : }
195 :
196 1 : HCCL_ERROR("************************************** ub async event **************************************");
197 1 : HCCL_ERROR(
198 : "deviceId[%d] epHandle[%p] seq[%u] resId[%u] eventType[%u] contextLen[%u]", deviceLogicId_, epHandle, seq,
199 : event.resId, event.eventType, event.len);
200 1 : if (contextLen != 0) {
201 1 : HCCL_ERROR("bytes order: high -> low");
202 : }
203 1 : constexpr u32 bytesPerLine = 4;
204 4 : for (u32 i = 0; i < contextLen; i += bytesPerLine) {
205 3 : u32 endIndex = std::min(i + bytesPerLine, contextLen);
206 3 : HCCL_ERROR(
207 : "context[byte %3u]: %02x%02x%02x%02x", endIndex,
208 : (i + LAST_OFFSET < contextLen) ? event.context[i + LAST_OFFSET] : 0,
209 : (i + SECOND_LAST_OFFSET < contextLen) ? event.context[i + SECOND_LAST_OFFSET] : 0,
210 : (i + 1 < contextLen) ? event.context[i + 1] : 0, event.context[i]);
211 : }
212 1 : HCCL_ERROR("********************************************************************************************");
213 : }
214 :
215 : } // namespace hcomm
|
