Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "dfx/endpoint_monitor.h"
11 : #include "hcom_common.h"
12 :
13 : namespace hcomm {
14 :
15 : constexpr u32 EndpointMonitor::MONITOR_INTERVAL;
16 :
17 198 : EndpointMonitor::~EndpointMonitor()
18 : {
19 198 : DeInit(deviceLogicId_);
20 198 : }
21 :
22 51 : EndpointMonitor &EndpointMonitor::GetInstance(s32 deviceLogicId)
23 : {
24 51 : static std::array<EndpointMonitor, MAX_MODULE_DEVICE_NUM + 1> instances;
25 : uint32_t deviceId;
26 51 : if ((deviceLogicId < 0) || (static_cast<u32>(deviceLogicId) > MAX_MODULE_DEVICE_NUM)) {
27 0 : HCCL_ERROR("[EndpointMonitor][%s] deviceLogicId[%d] not in range [0,%u]", __func__, deviceLogicId,
28 : MAX_MODULE_DEVICE_NUM);
29 0 : deviceId = MAX_MODULE_DEVICE_NUM;
30 : } else {
31 51 : deviceId = static_cast<u32>(deviceLogicId);
32 : }
33 51 : return instances[deviceId];
34 : }
35 :
36 6 : HcclResult EndpointMonitor::RegisterToEndpointMonitor(s32 deviceLogicId, EndpointHandle epHandle)
37 : {
38 6 : if ((deviceLogicId < 0) || (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM)) {
39 2 : HCCL_ERROR("[EndpointMonitor][%s] deviceLogicId[%d] not in range [0,%u)", __func__, deviceLogicId,
40 : MAX_MODULE_DEVICE_NUM);
41 2 : return HCCL_E_PARA;
42 : }
43 4 : CHK_PRT_RET(epHandle == nullptr, HCCL_ERROR("[EndpointMonitor][%s] epHandle is null", __func__), HCCL_E_PTR);
44 :
45 3 : HCCL_INFO("[EndpointMonitor] deviceLogicId[%d] epHandle[%p] RegisterToEndpointMonitor begin.", deviceLogicId,
46 : epHandle);
47 3 : u32 devPhyId{0};
48 3 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId), devPhyId));
49 :
50 : {
51 3 : std::lock_guard<std::mutex> lock(threadLock_);
52 3 : epHandleSet_.emplace(reinterpret_cast<u64>(epHandle));
53 3 : if (!initialized_) {
54 1 : deviceLogicId_ = deviceLogicId;
55 1 : devPhyId_ = devPhyId;
56 1 : CHK_RET(RunMonitorThread());
57 : }
58 3 : }
59 :
60 3 : HCCL_INFO("[EndpointMonitor] deviceLogicId[%d] epHandle[%p] RegisterToEndpointMonitor Completed.", deviceLogicId,
61 : epHandle);
62 3 : return HCCL_SUCCESS;
63 : }
64 :
65 1 : HcclResult EndpointMonitor::RunMonitorThread()
66 : {
67 1 : HCCL_INFO("[EndpointMonitor][%s] deviceLogicId[%d] Start Thread.", __func__, deviceLogicId_);
68 1 : endpointMonitorThreadFlag_ = true;
69 1 : EXCEPTION_CATCH(endpointMonitorThread_ = std::make_unique<std::thread>(&EndpointMonitor::MonitorThread, this),
70 : return HCCL_E_INTERNAL);
71 1 : CHK_SMART_PTR_NULL(endpointMonitorThread_);
72 1 : initialized_ = true;
73 1 : return HCCL_SUCCESS;
74 : }
75 :
76 1 : void EndpointMonitor::MonitorThread()
77 : {
78 1 : SetThreadName("Hccl_Ub_Event_Monitor");
79 :
80 1 : HcclResult ret = hrtSetDevice(deviceLogicId_);
81 1 : if (ret != HCCL_SUCCESS) {
82 0 : HCCL_ERROR("[EndpointMonitor][%s] hrtSetDevice failed, deviceLogicId[%d], ret[%d]",
83 : __func__, deviceLogicId_, ret);
84 0 : return;
85 : }
86 :
87 2 : while (endpointMonitorThreadFlag_) {
88 1 : ProcessUbAsyncEvents();
89 1 : std::this_thread::sleep_for(std::chrono::milliseconds(MONITOR_INTERVAL));
90 : }
91 :
92 1 : ret = hrtResetDevice(deviceLogicId_);
93 1 : if (ret != HCCL_SUCCESS) {
94 0 : HCCL_ERROR("[EndpointMonitor][%s] hrtResetDevice failed, deviceLogicId[%d], ret[%d]",
95 : __func__, deviceLogicId_, ret);
96 : }
97 : }
98 :
99 1 : HcclResult EndpointMonitor::UnRegisterToEndpointMonitor()
100 : {
101 1 : s32 deviceLogicId = deviceLogicId_;
102 1 : HCCL_INFO("[EndpointMonitor] deviceId[%d] UnRegisterToEndpointMonitor begin.", deviceLogicId);
103 : {
104 1 : std::lock_guard<std::mutex> lock(threadLock_);
105 1 : CHK_PRT_RET(!initialized_,
106 : HCCL_WARNING("[EndpointMonitor] deviceId[%d] hcclUbEventMonitor has been destroyed, or not initialized",
107 : deviceLogicId),
108 : HCCL_SUCCESS);
109 1 : epHandleSet_.clear();
110 1 : }
111 :
112 1 : CHK_RET(DeInit(deviceLogicId_));
113 1 : deviceLogicId_ = 0;
114 1 : devPhyId_ = 0;
115 1 : initialized_ = false;
116 :
117 1 : HCCL_INFO("[EndpointMonitor] deviceId[%d] UnRegisterToEndpointMonitor completed.", deviceLogicId);
118 1 : return HCCL_SUCCESS;
119 : }
120 :
121 33 : void EndpointMonitor::RemoveEpHandleFromEndpointMonitor(EndpointHandle epHandle)
122 : {
123 33 : if (epHandle == nullptr) {
124 2 : HCCL_ERROR("[EndpointMonitor][%s] epHandle is null", __func__);
125 2 : return;
126 : }
127 :
128 : {
129 31 : std::lock_guard<std::mutex> lock(threadLock_);
130 31 : auto it = epHandleSet_.find(reinterpret_cast<u64>(epHandle));
131 31 : if (it != epHandleSet_.end()) {
132 2 : epHandleSet_.erase(it);
133 2 : HCCL_INFO("[EndpointMonitor][%s] epHandle[%p] is remove from deviceId[%d]",
134 : __func__, epHandle, deviceLogicId_);
135 : }
136 31 : }
137 : }
138 :
139 199 : HcclResult EndpointMonitor::DeInit(s32 deviceLogicId)
140 : {
141 199 : endpointMonitorThreadFlag_ = false;
142 199 : if (endpointMonitorThread_) {
143 1 : if (endpointMonitorThread_->joinable()) {
144 : try {
145 1 : HCCL_INFO("[EndpointMonitor][%s] deviceId[%d] thread join", __func__, deviceLogicId);
146 1 : endpointMonitorThread_->join();
147 1 : endpointMonitorThread_.reset();
148 0 : } catch (const std::exception &e) {
149 0 : HCCL_ERROR("[EndpointMonitor][%s] deviceId[%d] join failed: %s", __func__, deviceLogicId, e.what());
150 0 : return HCCL_E_INTERNAL;
151 0 : }
152 : }
153 : }
154 199 : return HCCL_SUCCESS;
155 : }
156 :
157 5 : void EndpointMonitor::ProcessUbAsyncEvents()
158 : {
159 5 : std::lock_guard<std::mutex> lock(threadLock_);
160 8 : for (auto it = epHandleSet_.begin(); it != epHandleSet_.end();) {
161 3 : u32 num = ASYNC_EVENT_MAX_NUM;
162 3 : Endpoint *localEpPtr = reinterpret_cast<Endpoint *>(*it);
163 3 : HcclResult ret = localEpPtr->GetAsyncEvents(devPhyId_, events_, num);
164 3 : if (ret != HCCL_SUCCESS) {
165 2 : it = epHandleSet_.erase(it);
166 2 : HCCL_ERROR("[EndpointMonitor][%s] deviceId[%d] HcommGetAsyncEvents failed ret[%d], "
167 : "epHandle[%p] removed from monitor",
168 : __func__, deviceLogicId_, ret, localEpPtr);
169 2 : continue;
170 : }
171 :
172 5 : for (u32 i = 0; i < num; ++i) {
173 4 : PrintUbAsyncEventsContext(static_cast<void *>(localEpPtr), i, events_[i]);
174 : }
175 :
176 1 : ++it;
177 : }
178 5 : }
179 :
180 : constexpr u32 SECOND_LAST_OFFSET = 2; // 倒数第二个字节偏移
181 : constexpr u32 LAST_OFFSET = 3; // 倒数第一个字节偏移
182 2 : void EndpointMonitor::PrintUbAsyncEventsContext(void *epHandle, u32 seq, const struct AsyncEvent &event)
183 : {
184 2 : u32 contextLen = event.len;
185 2 : if (contextLen > CONTEXT_MAX_LEN) {
186 1 : HCCL_ERROR("[EndpointMonitor][%s] deviceId[%d] epHandle[%p] seq[%u] context len[%u] exceed max[%u]",
187 : __func__, deviceLogicId_, epHandle, seq, contextLen, CONTEXT_MAX_LEN);
188 1 : return;
189 : }
190 :
191 1 : HCCL_ERROR("************************************** ub async event **************************************");
192 1 : HCCL_ERROR("deviceId[%d] epHandle[%p] seq[%u] resId[%u] eventType[%u] contextLen[%u]",
193 : deviceLogicId_, epHandle, seq, event.resId, event.eventType, event.len);
194 1 : if (contextLen != 0) {
195 1 : HCCL_ERROR(" bytes order : high -> low");
196 : }
197 1 : constexpr u32 bytesPerLine = 4;
198 4 : for (u32 i = 0; i < contextLen; i += bytesPerLine) {
199 3 : u32 endIndex = std::min(i + bytesPerLine, contextLen);
200 3 : HCCL_ERROR("context[byte %3u]: %02x%02x%02x%02x", endIndex,
201 : (i + LAST_OFFSET < contextLen) ? event.context[i + LAST_OFFSET] : 0,
202 : (i + SECOND_LAST_OFFSET < contextLen) ? event.context[i + SECOND_LAST_OFFSET] : 0,
203 : (i + 1 < contextLen) ? event.context[i + 1] : 0,
204 : event.context[i]);
205 : }
206 1 : HCCL_ERROR("********************************************************************************************");
207 : }
208 :
209 : } // namespace hcomm
|
