Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "channel_logger.h"
12 : #include "comm_addr_logger.h"
13 : #include "endpoint_logger.h"
14 : #include "../endpoint_pairs/channels/channel.h" // ChannelStatus 枚举定义
15 :
16 : namespace hcomm {
17 : namespace logger {
18 :
19 5 : void ChannelLogger::PrintBasicFields(uint32_t idx, const HcclChannelDesc& channelDesc)
20 : {
21 5 : HCCL_INFO(
22 : "[%s] channelDescs[%u]: remoteRank[%u], channelProtocol[%d], notifyNum[%u], memHandleNum[%u]", __func__,
23 : idx, channelDesc.remoteRank, channelDesc.channelProtocol, channelDesc.notifyNum, channelDesc.memHandleNum);
24 5 : }
25 :
26 5 : void ChannelLogger::PrintRoceAttributes(uint32_t idx, const HcclChannelDesc& channelDesc)
27 : {
28 5 : if (!IsRoceProtocol(channelDesc)) {
29 3 : return;
30 : }
31 2 : std::string roceAttrStr = FormatRoceAttrDetail(channelDesc);
32 2 : HCCL_INFO("[%s] channelDescs[%u] roceAttr: %s", __func__, idx, roceAttrStr.c_str());
33 2 : }
34 :
35 5 : void ChannelLogger::PrintDescInfo(uint32_t idx, const HcclChannelDesc& channelDesc)
36 : {
37 : // 基本字段
38 5 : PrintBasicFields(idx, channelDesc);
39 :
40 : // 本地端点
41 5 : EndpointLogger::Print(idx, "localEndpoint", channelDesc.localEndpoint);
42 :
43 : // 远端端点
44 5 : EndpointLogger::Print(idx, "remoteEndpoint", channelDesc.remoteEndpoint);
45 :
46 : // ROCE 协议特有属性
47 5 : PrintRoceAttributes(idx, channelDesc);
48 :
49 : // UB_MEM 协议特有属性
50 5 : PrintUbMemAttributes(idx, channelDesc);
51 5 : }
52 :
53 1 : void ChannelLogger::PrintDescTableHeader()
54 : {
55 1 : HCCL_INFO(" "
56 : "_________________________________________________CHANNEL_DESCRIPTOR_INFO____________________________"
57 : "_____________________");
58 1 : HCCL_INFO(" | idx | remoteRank | Proto | notifyNum | memHandleNum | localAddr "
59 : "| remoteAddr | ROCE Attr |");
60 1 : HCCL_INFO(" "
61 : "|-------|------------|-------|-------------|--------------|-------------------------------------|---"
62 : "----------------------------------|--------------------------------|");
63 1 : }
64 :
65 2 : void ChannelLogger::PrintDescInfoRow(uint32_t idx, const HcclChannelDesc& channelDesc)
66 : {
67 : // 使用类内部辅助函数
68 2 : std::string localAddr, remoteAddr;
69 2 : FormatEndpointAddresses(channelDesc, localAddr, remoteAddr);
70 2 : std::string roceAttrStr = FormatRoceAttrCompact(channelDesc);
71 :
72 : // 输出表格行
73 2 : HCCL_INFO(
74 : " | %5u | %10u | %5d | %11u | %12u | %35s | %35s | %30s |", idx, channelDesc.remoteRank,
75 : channelDesc.channelProtocol, channelDesc.notifyNum, channelDesc.memHandleNum, localAddr.c_str(),
76 : remoteAddr.c_str(), roceAttrStr.c_str());
77 2 : }
78 :
79 5 : void ChannelLogger::PrintErrorTableHeader(uint32_t localRank)
80 : {
81 5 : HCCL_ERROR(" "
82 : "________________________________________________CHANNEL_CONNECT_ERROR_INFO_________________________"
83 : "_______________________");
84 5 : HCCL_ERROR(" | comm error, localRank[%u]", localRank);
85 5 : HCCL_ERROR(
86 : " | idx | localRank | localAddr | remoteRank | remoteAddr "
87 : " | chHandle | Status | TlsStatus | Proto | elapsed |");
88 5 : HCCL_ERROR(" "
89 : "|-------|-----------|-------------------------------------|------------|---------------------------"
90 : "----------|-----------|----------------------------|-----------------|-------|---------|");
91 5 : }
92 :
93 16 : std::string ChannelStatusUtils::ToString(int32_t status)
94 : {
95 16 : ChannelStatus::Value statusValue = static_cast<ChannelStatus::Value>(status);
96 16 : ChannelStatus statusEnum(statusValue);
97 32 : return statusEnum.Describe();
98 : }
99 :
100 11 : void ChannelLogger::PrintErrorInfo(
101 : uint32_t idx, uint32_t localRank, const HcclChannelDesc& channelDesc, ChannelHandle channelHandle,
102 : int32_t status, uint64_t elapsedMs, Hccl::TlsStatus tlsStatus)
103 : {
104 : // 复用 FormatEndpointAddresses()
105 11 : std::string localAddr, remoteAddr;
106 11 : FormatEndpointAddresses(channelDesc, localAddr, remoteAddr);
107 11 : std::string statusStr = ChannelStatusUtils::ToString(status);
108 11 : std::string tlsStatusStr = "UNKNOWN";
109 11 : switch (tlsStatus) {
110 1 : case Hccl::TlsStatus::ENABLE:
111 1 : tlsStatusStr = "ENABLE";
112 1 : break;
113 9 : case Hccl::TlsStatus::DISABLE:
114 9 : tlsStatusStr = "DISABLE";
115 9 : break;
116 1 : default:
117 1 : tlsStatusStr = "UNKNOWN";
118 1 : break;
119 : }
120 :
121 11 : HCCL_ERROR(
122 : " | %5u | %9u | %35s | %10u | %35s | 0x%08llx | %26s | %15s | %5d | %7llu ms |", idx, localRank,
123 : localAddr.c_str(), channelDesc.remoteRank, remoteAddr.c_str(),
124 : static_cast<unsigned long long>(channelHandle), statusStr.c_str(), tlsStatusStr.c_str(),
125 : channelDesc.channelProtocol, static_cast<unsigned long long>(elapsedMs));
126 11 : }
127 :
128 4 : void ChannelLogger::PrintChannelErrorDetails(
129 : uint32_t localRank, uint32_t channelNum, const HcclChannelDesc* channelDescs, ChannelHandle* channelHandles,
130 : int32_t* statusList, int64_t elapsedMs, Hccl::TlsStatus tlsStatus)
131 : {
132 : // 打印错误详情表格（只打印异常状态的 Channel）
133 4 : PrintErrorTableHeader(localRank);
134 :
135 15 : for (uint32_t i = 0; i < channelNum; ++i) {
136 11 : if (IsAbnormalStatus(statusList[i])) {
137 8 : PrintErrorInfo(i, localRank, channelDescs[i], channelHandles[i], statusList[i], elapsedMs, tlsStatus);
138 : }
139 : }
140 :
141 : // 表格外单独打印详细信息（FAILED 或 TIMEOUT 状态）
142 15 : for (uint32_t i = 0; i < channelNum; ++i) {
143 11 : if (NeedDetailPrint(statusList[i])) {
144 2 : PrintDescInfo(i, channelDescs[i]);
145 : }
146 : }
147 4 : }
148 :
149 : // ========== 私有辅助函数实现 ==========
150 :
151 15 : bool ChannelLogger::IsRoceProtocol(const HcclChannelDesc& channelDesc)
152 : {
153 15 : return channelDesc.channelProtocol == COMM_PROTOCOL_ROCE;
154 : }
155 :
156 4 : std::string ChannelLogger::FormatRoceAttrCompact(const HcclChannelDesc& channelDesc)
157 : {
158 4 : if (!IsRoceProtocol(channelDesc)) {
159 4 : return std::string("-");
160 : }
161 :
162 2 : char roceAttrStr[128] = {0};
163 2 : snprintf_s(
164 : roceAttrStr, sizeof(roceAttrStr), sizeof(roceAttrStr) - 1, "q:%u r:%u ri:%u tc:%u sl:%u",
165 2 : channelDesc.roceAttr.queueNum, channelDesc.roceAttr.retryCnt, channelDesc.roceAttr.retryInterval,
166 2 : channelDesc.roceAttr.tc, channelDesc.roceAttr.sl);
167 :
168 4 : return std::string(roceAttrStr);
169 : }
170 :
171 4 : std::string ChannelLogger::FormatRoceAttrDetail(const HcclChannelDesc& channelDesc)
172 : {
173 4 : if (!IsRoceProtocol(channelDesc)) {
174 2 : return std::string("");
175 : }
176 :
177 3 : char roceAttrStr[256] = {0};
178 3 : snprintf_s(
179 : roceAttrStr, sizeof(roceAttrStr), sizeof(roceAttrStr) - 1,
180 3 : "queueNum[%u], retryCnt[%u], retryInterval[%u], tc[%u], sl[%u]", channelDesc.roceAttr.queueNum,
181 3 : channelDesc.roceAttr.retryCnt, channelDesc.roceAttr.retryInterval, channelDesc.roceAttr.tc,
182 3 : channelDesc.roceAttr.sl);
183 :
184 6 : return std::string(roceAttrStr);
185 : }
186 :
187 14 : bool ChannelLogger::IsAbnormalStatus(int32_t status) { return status != ChannelStatus::READY; }
188 :
189 14 : bool ChannelLogger::NeedDetailPrint(int32_t status)
190 : {
191 14 : return status == ChannelStatus::FAILED || status == ChannelStatus::SOCKET_TIMEOUT;
192 : }
193 :
194 14 : void ChannelLogger::FormatEndpointAddresses(
195 : const HcclChannelDesc& channelDesc, std::string& outLocalAddr, std::string& outRemoteAddr)
196 : {
197 14 : outLocalAddr = CommAddrLogger::ToString(channelDesc.localEndpoint.commAddr);
198 14 : outRemoteAddr = CommAddrLogger::ToString(channelDesc.remoteEndpoint.commAddr);
199 14 : }
200 :
201 7 : void ChannelLogger::PrintUbMemAttributes(uint32_t idx, const HcclChannelDesc& channelDesc)
202 : {
203 7 : if (!IsUbMemProtocol(channelDesc)) {
204 5 : return;
205 : }
206 2 : HCCL_INFO("[%s] channelDescs[%u] ubMemAttr: pathMode[%u]", __func__, idx, channelDesc.ubMemAttr.pathMode);
207 : }
208 :
209 10 : bool ChannelLogger::IsUbMemProtocol(const HcclChannelDesc& channelDesc)
210 : {
211 10 : return channelDesc.channelProtocol == COMM_PROTOCOL_UB_MEM;
212 : }
213 :
214 : } // namespace logger
215 : } // namespace hcomm
|
