Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "channel_logger.h"
12 : #include "comm_addr_logger.h"
13 : #include "endpoint_logger.h"
14 : #include "../endpoint_pairs/channels/channel.h" // ChannelStatus 枚举定义
15 :
16 : namespace hcomm {
17 : namespace logger {
18 :
19 5 : void ChannelLogger::PrintBasicFields(uint32_t idx, const HcclChannelDesc& channelDesc)
20 : {
21 5 : HCCL_INFO(
22 : "[%s] channelDescs[%u]: remoteRank[%u], channelProtocol[%d], notifyNum[%u], memHandleNum[%u]", __func__,
23 : idx, channelDesc.remoteRank, channelDesc.channelProtocol, channelDesc.notifyNum, channelDesc.memHandleNum);
24 5 : }
25 :
26 5 : void ChannelLogger::PrintRoceAttributes(uint32_t idx, const HcclChannelDesc& channelDesc)
27 : {
28 5 : if (!IsRoceProtocol(channelDesc)) {
29 3 : return;
30 : }
31 2 : std::string roceAttrStr = FormatRoceAttrDetail(channelDesc);
32 2 : HCCL_INFO("[%s] channelDescs[%u] roceAttr: %s", __func__, idx, roceAttrStr.c_str());
33 2 : }
34 :
35 5 : void ChannelLogger::PrintDescInfo(uint32_t idx, const HcclChannelDesc& channelDesc)
36 : {
37 : // 基本字段
38 5 : PrintBasicFields(idx, channelDesc);
39 :
40 : // 本地端点
41 5 : EndpointLogger::Print(idx, "localEndpoint", channelDesc.localEndpoint);
42 :
43 : // 远端端点
44 5 : EndpointLogger::Print(idx, "remoteEndpoint", channelDesc.remoteEndpoint);
45 :
46 : // ROCE 协议特有属性
47 5 : PrintRoceAttributes(idx, channelDesc);
48 :
49 : // UB_MEM 协议特有属性
50 5 : PrintUbMemAttributes(idx, channelDesc);
51 5 : }
52 :
53 1 : void ChannelLogger::PrintDescTableHeader()
54 : {
55 1 : HCCL_INFO(" "
56 : "_________________________________________________CHANNEL_DESCRIPTOR_INFO____________________________"
57 : "_____________________");
58 1 : HCCL_INFO(" | idx | remoteRank | Proto | notifyNum | memHandleNum | localAddr "
59 : "| remoteAddr | ROCE Attr |");
60 1 : HCCL_INFO(" "
61 : "|-------|------------|-------|-------------|--------------|-------------------------------------|---"
62 : "----------------------------------|--------------------------------|");
63 1 : }
64 :
65 2 : void ChannelLogger::PrintDescInfoRow(uint32_t idx, const HcclChannelDesc& channelDesc)
66 : {
67 : // 使用类内部辅助函数
68 2 : std::string localAddr, remoteAddr;
69 2 : FormatEndpointAddresses(channelDesc, localAddr, remoteAddr);
70 2 : std::string roceAttrStr = FormatRoceAttrCompact(channelDesc);
71 :
72 : // 输出表格行
73 2 : HCCL_INFO(
74 : " | %5u | %10u | %5d | %11u | %12u | %35s | %35s | %30s |", idx, channelDesc.remoteRank,
75 : channelDesc.channelProtocol, channelDesc.notifyNum, channelDesc.memHandleNum, localAddr.c_str(),
76 : remoteAddr.c_str(), roceAttrStr.c_str());
77 2 : }
78 :
79 5 : void ChannelLogger::PrintErrorTableHeader(uint32_t localRank)
80 : {
81 5 : HCCL_ERROR(" "
82 : "________________________________________________CHANNEL_CONNECT_ERROR_INFO_________________________"
83 : "_______________________");
84 5 : HCCL_ERROR(" | comm error, localRank[%u]", localRank);
85 5 : HCCL_ERROR(
86 : " | idx | localRank | localAddr | remoteRank | remoteAddr "
87 : " | chHandle | Status | TlsStatus | Proto | elapsed |");
88 5 : HCCL_ERROR(" "
89 : "|-------|-----------|-------------------------------------|------------|---------------------------"
90 : "----------|-----------|----------------------------|-----------------|-------|---------|");
91 5 : }
92 :
93 16 : std::string ChannelStatusUtils::ToString(int32_t status)
94 : {
95 16 : ChannelStatus::Value statusValue = static_cast<ChannelStatus::Value>(status);
96 16 : ChannelStatus statusEnum(statusValue);
97 32 : return statusEnum.Describe();
98 : }
99 :
100 11 : void ChannelLogger::PrintErrorInfo(
101 : uint32_t idx, uint32_t localRank, const HcclChannelDesc& channelDesc, ChannelHandle channelHandle,
102 : int32_t status, uint64_t elapsedMs, Hccl::TlsStatus tlsStatus)
103 : {
104 : // 复用 FormatEndpointAddresses()
105 11 : std::string localAddr, remoteAddr;
106 11 : FormatEndpointAddresses(channelDesc, localAddr, remoteAddr);
107 11 : std::string statusStr = ChannelStatusUtils::ToString(status);
108 11 : std::string tlsStatusStr = "UNKNOWN";
109 11 : switch (tlsStatus) {
110 1 : case Hccl::TlsStatus::ENABLE:
111 1 : tlsStatusStr = "ENABLE";
112 1 : break;
113 9 : case Hccl::TlsStatus::DISABLE:
114 9 : tlsStatusStr = "DISABLE";
115 9 : break;
116 1 : default:
117 1 : tlsStatusStr = "UNKNOWN";
118 1 : break;
119 : }
120 :
121 11 : HCCL_ERROR(
122 : " | %5u | %9u | %35s | %10u | %35s | 0x%08llx | %26s | %15s | %5d | %7llu ms |", idx, localRank,
123 : localAddr.c_str(), channelDesc.remoteRank, remoteAddr.c_str(),
124 : static_cast<unsigned long long>(channelHandle), statusStr.c_str(), tlsStatusStr.c_str(),
125 : channelDesc.channelProtocol, static_cast<unsigned long long>(elapsedMs));
126 11 : }
127 :
128 4 : void ChannelLogger::PrintChannelErrorDetails(
129 : uint32_t localRank, uint32_t channelNum, const HcclChannelDesc* channelDescs, ChannelHandle* channelHandles,
130 : int32_t* statusList, int64_t elapsedMs, const Hccl::TlsStatus* tlsStatusList)
131 : {
132 : // 打印错误详情表格（只打印异常状态的 Channel）
133 4 : PrintErrorTableHeader(localRank);
134 :
135 15 : for (uint32_t i = 0; i < channelNum; ++i) {
136 11 : if (IsAbnormalStatus(statusList[i])) {
137 8 : PrintErrorInfo(
138 8 : i, localRank, channelDescs[i], channelHandles[i], statusList[i], elapsedMs, tlsStatusList[i]);
139 : }
140 : }
141 :
142 : // 表格外单独打印详细信息（FAILED 或 TIMEOUT 状态）
143 15 : for (uint32_t i = 0; i < channelNum; ++i) {
144 11 : if (NeedDetailPrint(statusList[i])) {
145 2 : PrintDescInfo(i, channelDescs[i]);
146 : }
147 : }
148 4 : }
149 :
150 : // ========== 私有辅助函数实现 ==========
151 :
152 15 : bool ChannelLogger::IsRoceProtocol(const HcclChannelDesc& channelDesc)
153 : {
154 15 : return channelDesc.channelProtocol == COMM_PROTOCOL_ROCE;
155 : }
156 :
157 4 : std::string ChannelLogger::FormatRoceAttrCompact(const HcclChannelDesc& channelDesc)
158 : {
159 4 : if (!IsRoceProtocol(channelDesc)) {
160 4 : return std::string("-");
161 : }
162 :
163 2 : char roceAttrStr[128] = {0};
164 2 : snprintf_s(
165 : roceAttrStr, sizeof(roceAttrStr), sizeof(roceAttrStr) - 1, "q:%u r:%u ri:%u tc:%u sl:%u",
166 2 : channelDesc.roceAttr.queueNum, channelDesc.roceAttr.retryCnt, channelDesc.roceAttr.retryInterval,
167 2 : channelDesc.roceAttr.tc, channelDesc.roceAttr.sl);
168 :
169 4 : return std::string(roceAttrStr);
170 : }
171 :
172 4 : std::string ChannelLogger::FormatRoceAttrDetail(const HcclChannelDesc& channelDesc)
173 : {
174 4 : if (!IsRoceProtocol(channelDesc)) {
175 2 : return std::string("");
176 : }
177 :
178 3 : char roceAttrStr[256] = {0};
179 3 : snprintf_s(
180 : roceAttrStr, sizeof(roceAttrStr), sizeof(roceAttrStr) - 1,
181 3 : "queueNum[%u], retryCnt[%u], retryInterval[%u], tc[%u], sl[%u]", channelDesc.roceAttr.queueNum,
182 3 : channelDesc.roceAttr.retryCnt, channelDesc.roceAttr.retryInterval, channelDesc.roceAttr.tc,
183 3 : channelDesc.roceAttr.sl);
184 :
185 6 : return std::string(roceAttrStr);
186 : }
187 :
188 14 : bool ChannelLogger::IsAbnormalStatus(int32_t status) { return status != ChannelStatus::READY; }
189 :
190 14 : bool ChannelLogger::NeedDetailPrint(int32_t status)
191 : {
192 14 : return status == ChannelStatus::FAILED || status == ChannelStatus::SOCKET_TIMEOUT;
193 : }
194 :
195 14 : void ChannelLogger::FormatEndpointAddresses(
196 : const HcclChannelDesc& channelDesc, std::string& outLocalAddr, std::string& outRemoteAddr)
197 : {
198 14 : outLocalAddr = CommAddrLogger::ToString(channelDesc.localEndpoint.commAddr);
199 14 : outRemoteAddr = CommAddrLogger::ToString(channelDesc.remoteEndpoint.commAddr);
200 14 : }
201 :
202 7 : void ChannelLogger::PrintUbMemAttributes(uint32_t idx, const HcclChannelDesc& channelDesc)
203 : {
204 7 : if (!IsUbMemProtocol(channelDesc)) {
205 5 : return;
206 : }
207 2 : HCCL_INFO("[%s] channelDescs[%u] ubMemAttr: pathMode[%u]", __func__, idx, channelDesc.ubMemAttr.pathMode);
208 : }
209 :
210 10 : bool ChannelLogger::IsUbMemProtocol(const HcclChannelDesc& channelDesc)
211 : {
212 10 : return channelDesc.channelProtocol == COMM_PROTOCOL_UB_MEM;
213 : }
214 :
215 : } // namespace logger
216 : } // namespace hcomm
|
