Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "channel_logger.h"
12 : #include "comm_addr_logger.h"
13 : #include "endpoint_logger.h"
14 : #include "../endpoint_pairs/channels/channel.h" // ChannelStatus 枚举定义
15 :
16 : namespace hcomm {
17 : namespace logger {
18 :
19 4 : void ChannelLogger::PrintBasicFields(uint32_t idx, const HcclChannelDesc& channelDesc)
20 : {
21 4 : HCCL_INFO("[%s] channelDescs[%u]: remoteRank[%u], channelProtocol[%d], notifyNum[%u], memHandleNum[%u]",
22 : __func__, idx, channelDesc.remoteRank, channelDesc.channelProtocol,
23 : channelDesc.notifyNum, channelDesc.memHandleNum);
24 4 : }
25 :
26 4 : void ChannelLogger::PrintRoceAttributes(uint32_t idx, const HcclChannelDesc& channelDesc)
27 : {
28 4 : if (!IsRoceProtocol(channelDesc)) {
29 2 : return;
30 : }
31 2 : std::string roceAttrStr = FormatRoceAttrDetail(channelDesc);
32 2 : HCCL_INFO("[%s] channelDescs[%u] roceAttr: %s", __func__, idx, roceAttrStr.c_str());
33 2 : }
34 :
35 4 : void ChannelLogger::PrintDescInfo(uint32_t idx, const HcclChannelDesc& channelDesc)
36 : {
37 : // 基本字段
38 4 : PrintBasicFields(idx, channelDesc);
39 :
40 : // 本地端点
41 4 : EndpointLogger::Print(idx, "localEndpoint", channelDesc.localEndpoint);
42 :
43 : // 远端端点
44 4 : EndpointLogger::Print(idx, "remoteEndpoint", channelDesc.remoteEndpoint);
45 :
46 : // ROCE 协议特有属性
47 4 : PrintRoceAttributes(idx, channelDesc);
48 4 : }
49 :
50 1 : void ChannelLogger::PrintDescTableHeader()
51 : {
52 1 : HCCL_INFO(" _________________________________________________CHANNEL_DESCRIPTOR_INFO_________________________________________________");
53 1 : HCCL_INFO(" | idx | remoteRank | Proto | notifyNum | memHandleNum | localAddr | remoteAddr | ROCE Attr |");
54 1 : HCCL_INFO(" |-------|------------|-------|-------------|--------------|-------------------------------------|-------------------------------------|--------------------------------|");
55 1 : }
56 :
57 2 : void ChannelLogger::PrintDescInfoRow(uint32_t idx, const HcclChannelDesc& channelDesc)
58 : {
59 : // 使用类内部辅助函数
60 2 : std::string localAddr, remoteAddr;
61 2 : FormatEndpointAddresses(channelDesc, localAddr, remoteAddr);
62 2 : std::string roceAttrStr = FormatRoceAttrCompact(channelDesc);
63 :
64 : // 输出表格行
65 2 : HCCL_INFO(" | %5u | %10u | %5d | %11u | %12u | %35s | %35s | %30s |",
66 : idx,
67 : channelDesc.remoteRank,
68 : channelDesc.channelProtocol,
69 : channelDesc.notifyNum,
70 : channelDesc.memHandleNum,
71 : localAddr.c_str(),
72 : remoteAddr.c_str(),
73 : roceAttrStr.c_str());
74 2 : }
75 :
76 5 : void ChannelLogger::PrintErrorTableHeader(uint32_t localRank)
77 : {
78 5 : HCCL_ERROR(" ________________________________________________CHANNEL_CONNECT_ERROR_INFO________________________________________________");
79 5 : HCCL_ERROR(" | comm error, localRank[%u]", localRank);
80 5 : HCCL_ERROR(" | idx | localRank | localAddr | remoteRank | remoteAddr | chHandle | Status | TlsStatus | Proto | elapsed |");
81 5 : HCCL_ERROR(" |-------|-----------|-------------------------------------|------------|-------------------------------------|-----------|----------------------------|-----------------|-------|---------|");
82 5 : }
83 :
84 16 : std::string ChannelStatusUtils::ToString(int32_t status)
85 : {
86 16 : ChannelStatus::Value statusValue = static_cast<ChannelStatus::Value>(status);
87 16 : ChannelStatus statusEnum(statusValue);
88 32 : return statusEnum.Describe();
89 : }
90 :
91 11 : void ChannelLogger::PrintErrorInfo(
92 : uint32_t idx,
93 : uint32_t localRank,
94 : const HcclChannelDesc& channelDesc,
95 : ChannelHandle channelHandle,
96 : int32_t status,
97 : uint64_t elapsedMs,
98 : Hccl::TlsStatus tlsStatus)
99 : {
100 : // 复用 FormatEndpointAddresses()
101 11 : std::string localAddr, remoteAddr;
102 11 : FormatEndpointAddresses(channelDesc, localAddr, remoteAddr);
103 11 : std::string statusStr = ChannelStatusUtils::ToString(status);
104 11 : std::string tlsStatusStr = "UNKNOWN";
105 11 : switch (tlsStatus) {
106 1 : case Hccl::TlsStatus::ENABLE:
107 1 : tlsStatusStr = "ENABLE";
108 1 : break;
109 9 : case Hccl::TlsStatus::DISABLE:
110 9 : tlsStatusStr = "DISABLE";
111 9 : break;
112 1 : default:
113 1 : tlsStatusStr = "UNKNOWN";
114 1 : break;
115 : }
116 :
117 11 : HCCL_ERROR(" | %5u | %9u | %35s | %10u | %35s | 0x%08llx | %26s | %15s | %5d | %7llu ms |",
118 : idx,
119 : localRank,
120 : localAddr.c_str(),
121 : channelDesc.remoteRank,
122 : remoteAddr.c_str(),
123 : static_cast<unsigned long long>(channelHandle),
124 : statusStr.c_str(),
125 : tlsStatusStr.c_str(),
126 : channelDesc.channelProtocol,
127 : static_cast<unsigned long long>(elapsedMs));
128 11 : }
129 :
130 4 : void ChannelLogger::PrintChannelErrorDetails(
131 : uint32_t localRank,
132 : uint32_t channelNum,
133 : const HcclChannelDesc* channelDescs,
134 : ChannelHandle* channelHandles,
135 : int32_t* statusList,
136 : int64_t elapsedMs,
137 : Hccl::TlsStatus tlsStatus)
138 : {
139 : // 打印错误详情表格（只打印异常状态的 Channel）
140 4 : PrintErrorTableHeader(localRank);
141 :
142 15 : for (uint32_t i = 0; i < channelNum; ++i) {
143 11 : if (IsAbnormalStatus(statusList[i])) {
144 8 : PrintErrorInfo(i, localRank, channelDescs[i], channelHandles[i], statusList[i], elapsedMs, tlsStatus);
145 : }
146 : }
147 :
148 : // 表格外单独打印详细信息（FAILED 或 TIMEOUT 状态）
149 15 : for (uint32_t i = 0; i < channelNum; ++i) {
150 11 : if (NeedDetailPrint(statusList[i])) {
151 2 : PrintDescInfo(i, channelDescs[i]);
152 : }
153 : }
154 4 : }
155 :
156 : // ========== 私有辅助函数实现 ==========
157 :
158 14 : bool ChannelLogger::IsRoceProtocol(const HcclChannelDesc& channelDesc)
159 : {
160 14 : return channelDesc.channelProtocol == COMM_PROTOCOL_ROCE;
161 : }
162 :
163 4 : std::string ChannelLogger::FormatRoceAttrCompact(const HcclChannelDesc& channelDesc)
164 : {
165 4 : if (!IsRoceProtocol(channelDesc)) {
166 4 : return std::string("-");
167 : }
168 :
169 2 : char roceAttrStr[128] = {0};
170 2 : snprintf_s(roceAttrStr, sizeof(roceAttrStr), sizeof(roceAttrStr) - 1,
171 : "q:%u r:%u ri:%u tc:%u sl:%u",
172 2 : channelDesc.roceAttr.queueNum,
173 2 : channelDesc.roceAttr.retryCnt,
174 2 : channelDesc.roceAttr.retryInterval,
175 2 : channelDesc.roceAttr.tc,
176 2 : channelDesc.roceAttr.sl);
177 :
178 4 : return std::string(roceAttrStr);
179 : }
180 :
181 4 : std::string ChannelLogger::FormatRoceAttrDetail(const HcclChannelDesc& channelDesc)
182 : {
183 4 : if (!IsRoceProtocol(channelDesc)) {
184 2 : return std::string("");
185 : }
186 :
187 3 : char roceAttrStr[256] = {0};
188 3 : snprintf_s(roceAttrStr, sizeof(roceAttrStr), sizeof(roceAttrStr) - 1,
189 : "queueNum[%u], retryCnt[%u], retryInterval[%u], tc[%u], sl[%u]",
190 3 : channelDesc.roceAttr.queueNum,
191 3 : channelDesc.roceAttr.retryCnt,
192 3 : channelDesc.roceAttr.retryInterval,
193 3 : channelDesc.roceAttr.tc,
194 3 : channelDesc.roceAttr.sl);
195 :
196 6 : return std::string(roceAttrStr);
197 : }
198 :
199 14 : bool ChannelLogger::IsAbnormalStatus(int32_t status)
200 : {
201 14 : return status != ChannelStatus::READY;
202 : }
203 :
204 14 : bool ChannelLogger::NeedDetailPrint(int32_t status)
205 : {
206 14 : return status == ChannelStatus::FAILED || status == ChannelStatus::SOCKET_TIMEOUT;
207 : }
208 :
209 14 : void ChannelLogger::FormatEndpointAddresses(
210 : const HcclChannelDesc& channelDesc,
211 : std::string& outLocalAddr,
212 : std::string& outRemoteAddr)
213 : {
214 14 : outLocalAddr = CommAddrLogger::ToString(channelDesc.localEndpoint.commAddr);
215 14 : outRemoteAddr = CommAddrLogger::ToString(channelDesc.remoteEndpoint.commAddr);
216 14 : }
217 :
218 : } // namespace logger
219 : } // namespace hcomm
|
