Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "base/err_msg.h"
12 : #include <string>
13 : #include <stdio.h>
14 : namespace {
15 :
16 : const std::string hcomm_g_msg = R"(
17 : {
18 : "error_info_list": [
19 : {
20 : "errClass": "HCCL Errors",
21 : "errTitle": "Config_Error_Invalid_Environment_Variable",
22 : "ErrCode": "EI0001",
23 : "ErrMessage": "Value %s for environment variable %s is invalid. Expected value: %s.",
24 : "Arglist": "value,env,expect",
25 : "suggestion": {
26 : "Possible Cause": "N/A",
27 : "Solution": "N/A"
28 : }
29 : },
30 : {
31 : "errClass": "HCCL Errors",
32 : "errTitle": "Communication_Error_Timeout",
33 : "ErrCode": "EI0002",
34 : "ErrMessage": "An timeout occurs when the Notify register waits for execution. Waiting peer rank: %s; task information: %s; communication operator information: %s; communicator: %s.",
35 : "Arglist": "remote_rankid, base_information, task_information, group_rank_content",
36 : "suggestion": {
37 : "Possible Cause": "1. An exception occurs during the execution on some NPUs in the cluster. As a result, collective communication operation failed.\r\n2. The execution speed on some NPU in the cluster is too slow to complete a communication operation within the timeout interval. (The default timeout interval is 1800s, You can set the interval by using HCCL_EXEC_TIMEOUT.)\r\n3. The number of training samples of each NPU is inconsistent.\r\n4. Packet loss or other connectivity problems occur on the communication link.",
38 : "Solution": "1. If this error is reported on only some ranks, check other ranks for earlier errors and investigate the first reported error.\r\n2. If this error is reported on all ranks, verify that the collective operation type, data count and data type are consistent across all ranks.\r\n3. Check whether the error reporting time difference between ranks exceeds HCCL_EXEC_TIMEOUT (1800s by default). If so, investigate the execution gap between ranks or increase HCCL_EXEC_TIMEOUT if necessary.\r\n4. Check for CQE errors in the plog (grep -rn 'error cqe'). If so, check the network connection status.\r\n5. For detailed troubleshooting guidance, search for the keyword \"EI0002\" on https://www.hiascend.com/document/."
39 : }
40 : },
41 : {
42 : "errClass": "HCCL Errors",
43 : "errTitle": "Invalid_Argument_Collective_Communication_Operator",
44 : "ErrCode": "EI0003",
45 : "ErrMessage": "Failed to verify parameters of operator %s (operator name). Value %s for parameter %s is invalid. The expected value is %s.",
46 : "Arglist": "ccl_op,value,parameter,expect",
47 : "suggestion": {
48 : "Possible Cause": "N/A",
49 : "Solution": "N/A"
50 : }
51 : },
52 : {
53 : "errClass": "HCCL Errors",
54 : "errTitle": "File_Operation_Error_Parse",
55 : "ErrCode": "EI0004",
56 : "ErrMessage": "Failed to parse the ranktable file %s. Reason: %s",
57 : "Arglist": "ranktable_path, error_reason",
58 : "suggestion": {
59 : "Possible Cause": "N/A",
60 : "Solution": "N/A"
61 : }
62 : },
63 : {
64 : "errClass": "HCCL Errors ",
65 : "errTitle": "Invalid_Argument",
66 : "ErrCode": "EI0005",
67 : "ErrMessage": "The arguments for collective communication are inconsistent between ranks, operator %s, group %s, parameter %s, local rank %s, remote rank %s.",
68 : "Arglist": "ccl_op, group, para_name,local_para,remote_para",
69 : "suggestion": {
70 : "Possible Cause": "N/A",
71 : "Solution": "N/A"
72 : }
73 : },
74 : {
75 : "errClass": "HCCL Errors",
76 : "errTitle": "Communication_Error_Get_Socket",
77 : "ErrCode": "EI0006",
78 : "ErrMessage": "Getting socket times out. Reason: %s",
79 : "Arglist": "reason",
80 : "suggestion": {
81 : "Possible Cause": "N/A",
82 : "Solution": "N/A"
83 : }
84 : },
85 : {
86 : "errClass": "HCCL Errors",
87 : "errTitle": "Resource_Error",
88 : "ErrCode": "EI0007",
89 : "ErrMessage": "Failed to allocate resource %s with info %s. Reason: Resources are exhausted.",
90 : "Arglist": "resource_type, resource_info",
91 : "suggestion": {
92 : "Possible Cause": "N/A",
93 : "Solution": "N/A"
94 : }
95 : },
96 : {
97 : "errClass": "HCCL Errors",
98 : "errTitle": "Package_Error_Incorrect_HCCL_Version",
99 : "ErrCode": "EI0008",
100 : "ErrMessage": "The %s versions are inconsistent. The local %s, while the remote %s.",
101 : "Arglist": "inconsistent_package, local_version, remote_version",
102 : "suggestion": {
103 : "Possible Cause": "N/A",
104 : "Solution": "Install the same version. The local Toolkit (cann-hcomm) and ops (cann-hccl) versions must be consistent with the corresponding remote versions."
105 : }
106 : },
107 : {
108 : "errClass": "HCCL Errors",
109 : "errTitle": "Communication_Error_Initialize_Transport",
110 : "ErrCode": "EI0009",
111 : "ErrMessage": "Device %s transport init error. Reason: %s.",
112 : "Arglist": "device_id,reason",
113 : "suggestion": {
114 : "Possible Cause": "N/A",
115 : "Solution": "Use the following hccn_tool commands to check whether the port link is down. (The scope of i represents the number of NPUs of each node. 8 is used as an example.)\n1. Check whether the optical module is in position: for i in {0..7}; do hccn_tool -i $i -optical -g; done | grep present.\n2. Check whether the IP address is configured: for i in {0..7}; do hccn_tool -i $i -ip -g; done.\n3. Check whether the switch is connected: for i in {0..7}; do hccn_tool -i $i -lldp -g; done."
116 : }
117 : },
118 : {
119 : "errClass": "HCCL Errors",
120 : "errTitle": "Communication_Error_P2P",
121 : "ErrCode": "EI0010",
122 : "ErrMessage": "P2P communication failed. Reason: %s",
123 : "Arglist": "reason",
124 : "suggestion": {
125 : "Possible Cause": "N/A",
126 : "Solution": "Ensure that the NPU card is normal and entering environment variables 'export HCCL_INTRA_ROCE_ENABLE=1'."
127 : }
128 : },
129 : {
130 : "errClass": "HCCL Errors",
131 : "errTitle": "Resource_Error_Insufficient_Device_Memory",
132 : "ErrCode": "EI0011",
133 : "ErrMessage": "Failed to allocate %s bytes of NPU memory.",
134 : "Arglist": "memory_size",
135 : "suggestion": {
136 : "Possible Cause": "Allocation failure due to insufficient NPU memory.",
137 : "Solution": "Stop unnecessary processes and ensure the required memory is available."
138 : }
139 : },
140 : {
141 : "errClass": "HCCL Errors",
142 : "errTitle": "Execution_Error_SDMA",
143 : "ErrCode": "EI0012",
144 : "ErrMessage": "SDMA memory copy task exception occurred. Remote rank: %s. Base information: %s. Task information: %s. Communicator information: %s.",
145 : "Arglist": "remote_rankid, base_information, task_information, group_rank_content",
146 : "suggestion": {
147 : "Possible Cause": "1. Network connection exception occurred during the SDMA task execution.\n2. The peer process exits abnormally.\n3. The input or output memory address is not allocated, the actual allocated size is smaller than the input data size, or the memory is freed before the operator execution is complete.",
148 : "Solution": "1. Check whether the network link is abnormal during the execution.\n2. Check whether a process in the cluster exits before an error is reported. If yes, locate the cause of the process exit.\n3. Check whether the input/output memory size is correct and whether the memory or communicator is released prematurely."
149 : }
150 : },
151 : {
152 : "errClass": "HCCL Errors",
153 : "errTitle": "Execution_Error_ROCE_CQE",
154 : "ErrCode": "EI0013",
155 : "ErrMessage": "An error CQE occurred during operator execution. Local information: server %s, device ID %s, device IP %s. Peer information: server %s, device ID %s, device IP %s.",
156 : "Arglist": "localServerId,localDeviceId,localDeviceIp,remoteServerId,remoteDeviceId,remoteDeviceIp",
157 : "suggestion": {
158 : "Possible Cause": "1. The network between two devices is abnormal. For example, the network port is intermittently disconnected. 2. The peer process exits abnormally in advance. As a result, the local end cannot receive the response from the peer end.",
159 : "Solution": "1. Check whether the network devices between the two ends are abnormal. 2. Check whether the peer process exits first. If yes, check the cause of the process exit."
160 : }
161 : },
162 : {
163 : "errClass": "HCCL Errors",
164 : "errTitle": "Config_Error_Ranktable",
165 : "ErrCode": "EI0014",
166 : "ErrMessage": "Value %s for ranktable variable %s is invalid. Expected value: %s.",
167 : "Arglist": "value, variable, expect",
168 : "suggestion": {
169 : "Possible Cause": "Failed to verify the content of the ranktable file, possibly due to inconsistency between the file content and the actual device information.",
170 : "Solution": "Try again with a valid cluster configuration in the ranktable file. Ensure that the configuration matches the operating environment."
171 : }
172 : },
173 : {
174 : "errClass": "HCCL Errors",
175 : "errTitle": "Communication_Error_Ranktable_Detect",
176 : "ErrCode": "EI0015",
177 : "ErrMessage": "Failed to collect cluster information of the communicator based on rootInfo detection. Reason: %s.",
178 : "Arglist": "error_reason",
179 : "suggestion": {
180 : "Possible Cause": "N/A",
181 : "Solution":"1. Check whether all ranks in the communicator have delivered the communicator creation interface. 2. Check the connectivity between the host networks of all nodes and the server node. 3. Check whether the HCCL_SOCKET_IFNAME environment variable of all nodes is correctly configured. 4. Increase the timeout by configuring the HCCL_CONNECT_TIMEOUT environment variable."
182 : }
183 : },
184 : {
185 : "errClass": "HCCL Errors",
186 : "errTitle": "Config_Error",
187 : "ErrCode": "EI0016",
188 : "ErrMessage": "Value %s for config %s is invalid. Expected value: %s.",
189 : "Arglist": "value, variable, expect",
190 : "suggestion": {
191 : "Possible Cause": "N/A",
192 : "Solution": "N/A"
193 : }
194 : },
195 : {
196 : "errClass": "HCCL Errors",
197 : "errTitle": "Config_Error_Ranktable",
198 : "ErrCode": "EI0017",
199 : "ErrMessage": "Config %s is missing in the ranktable file.",
200 : "Arglist": "config",
201 : "suggestion": {
202 : "Possible Cause": "N/A",
203 : "Solution": "N/A"
204 : }
205 : },
206 : {
207 : "errClass": "HCCL Errors",
208 : "errTitle": "Execution_Error_UB_CQE",
209 : "ErrCode": "EI0018",
210 : "ErrMessage": "An error CQE occurred during operator execution. Local information: server %s, device ID %s, device IP %s. Peer information: server %s, device ID %s, device IP %s.",
211 : "Arglist": "localServerId, localDeviceId, localDeviceIp, remoteServerId, remoteDeviceId, remoteDeviceIp",
212 : "suggestion": {
213 : "Possible Cause": "1. The network between two devices is abnormal. For example, the network port is intermittently disconnected. 2. The peer process exits unexpectedly in advance. As a result, the local end cannot receive the response from the peer end. 3.The hardware of the HBM or UB chip processing module of either device is abnormal.",
214 : "Solution": "1. Check whether the network devices between the two ends are abnormal. Generally, packet loss occurs due to intermittent disconnection of the port. If the ping test fails, check whether the port is linkdown or the network configuration is incorrect.2. Check whether the peer process exits first. If yes, check the reason why the process exit.3.Use the RAS fault check mechanism to check whether the hardware of the HBM or UB chip processing module of either device is abnormal.If the hardware is abnormal, contact Huawei technical support."
215 : }
216 : },
217 : {
218 : "errClass": "HCCL Errors",
219 : "errTitle": "Communication_Error_Bind_IP_Port",
220 : "ErrCode": "EI0019",
221 : "ErrMessage": "Failed to enable listening for the host network adapter socket. Reason: %s",
222 : "Arglist": "reason",
223 : "suggestion": {
224 : "Possible Cause": "N/A",
225 : "Solution": "1. Check whether this port has been occupied by another process. If yes, you can make adjustment using the environment variable HCCL_IF_BASE_PORT and use sysctl -w net.ipv4.ip_local_reserved_ports=****-**** to adjust the scope of reserved ports. 2. Check whether the service process is started multiple times on a device during this service."
226 : }
227 : },
228 : {
229 : "errClass": "HCCL Errors",
230 : "errTitle": "Communication_Error_Bind_IP_Port",
231 : "ErrCode": "EI0020",
232 : "ErrMessage": "Failed to enable listening for the NPU network adapter socket. Reason: %s",
233 : "Arglist": "reason",
234 : "suggestion": {
235 : "Possible Cause": "N/A",
236 : "Solution": "Check whether the single-card multi-process scenario is used. If yes, configure the port number using the environment variable HCCL_NPU_SOCKET_PORT_RANGE."
237 : }
238 : }
239 : ]
240 : }
241 : )";
242 : }
243 :
244 47 : REG_FORMAT_ERROR_MSG(hcomm_g_msg.c_str(), hcomm_g_msg.size());
|
