Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef DGW_CLIENT_H
12 : #define DGW_CLIENT_H
13 :
14 : #include <memory>
15 : #include <string>
16 : #include <vector>
17 : #include <utility>
18 : #include <list>
19 : #include "common/type_def.h"
20 : #include "queue_schedule/qs_client.h"
21 :
22 : namespace bqs {
23 : // max ip len
24 : constexpr size_t MAX_IP_LEN = 16UL;
25 : // max tag name len
26 : constexpr size_t MAX_TAG_NAME_LEN = 128UL;
27 :
28 : // config command for datagw
29 : enum class ConfigCmd : int32_t {
30 : DGW_CFG_CMD_BIND_ROUTE = 0,
31 : DGW_CFG_CMD_UNBIND_ROUTE = 1,
32 : DGW_CFG_CMD_QRY_ROUTE = 2,
33 : DGW_CFG_CMD_ADD_GROUP = 3,
34 : DGW_CFG_CMD_DEL_GROUP = 4,
35 : DGW_CFG_CMD_QRY_GROUP = 5,
36 : DGW_CFG_CMD_RESERVED = 6,
37 : DGW_CFG_CMD_UPDATE_PROFILING = 7,
38 : DGW_CFG_CMD_SET_HCCL_PROTOCOL = 8,
39 : DGW_CFG_CMD_INIT_DYNAMIC_SCHEDULE = 9,
40 : DGW_CFG_CMD_STOP_SCHEDULE = 10,
41 : DGW_CFG_CMD_CLEAR_AND_RESTART_SCHEDULE = 11
42 : };
43 :
44 : // endpoint type
45 : enum class EndpointType : int32_t {
46 : QUEUE = 0,
47 : MEM_QUEUE = 1,
48 : GROUP = 2,
49 : COMM_CHANNEL = 3,
50 : };
51 :
52 : // endpoint status
53 : enum class EndpointStatus : int32_t {
54 : AVAILABLE = 0,
55 : UNAVAILABLE = 1,
56 : };
57 :
58 : // route status
59 : enum class RouteStatus : int32_t {
60 : ACTIVE = 0,
61 : INACTIVE = 1,
62 : ERROR = 2,
63 : };
64 :
65 : // group policy
66 : enum class GroupPolicy : int32_t {
67 : HASH = 0,
68 : BROADCAST = 1,
69 : DYNAMIC = 2,
70 : };
71 :
72 : // Query mode
73 : enum class QueryMode : int32_t {
74 : DGW_QUERY_MODE_SRC_ROUTE = 0,
75 : DGW_QUERY_MODE_DST_ROUTE = 1,
76 : DGW_QUERY_MODE_SRC_DST_ROUTE = 2,
77 : DGW_QUERY_MODE_ALL_ROUTE = 3,
78 : DGW_QUERY_MODE_GROUP = 4,
79 : DGW_QUERY_MODE_RESERVED = 5,
80 : };
81 :
82 : // qs scheduling policy
83 : enum class SchedPolicy : uint64_t {
84 : POLICY_UNSUB_F2NF = 1UL,
85 : POLICY_SUB_BUF_EVENT = 2UL,
86 : };
87 :
88 : enum class ProfilingMode : uint32_t {
89 : PROFILING_CLOSE = 0U,
90 : PROFILING_OPEN = 1U,
91 : };
92 :
93 : enum class HcclProtocolType : uint32_t {
94 : RDMA = 0U,
95 : TCP = 1U,
96 : };
97 :
98 : #pragma pack(push, 1)
99 : // queue attr: can not named QueueAttr, duplicatable name with driver
100 : struct FlowQueueAttr {
101 : int32_t queueId; // queue id
102 : };
103 :
104 : struct MemQueueAttr {
105 : int32_t queueId; // queue id
106 : uint32_t queueType; // localQ:0 or clientQ:1
107 : uint32_t rsv[7];
108 : };
109 :
110 : // communication channel attr
111 : struct CommChannelAttr {
112 : uint64_t handle; // hcom handle
113 : uint32_t localTagId; // local tag id
114 : uint32_t peerTagId; // peer tag id
115 : uint32_t localRankId; // local rank id
116 : uint32_t peerRankId; // peer rank id
117 : uint32_t localTagDepth; // local tag depth
118 : uint32_t peerTagDepth; // peer tag depth
119 : uint32_t memType; // memType: 1-dvpp
120 : uint32_t rsv[7]; // reserved field
121 : };
122 :
123 : // group attr
124 : struct GroupAttr {
125 : int32_t groupId; // group id
126 : GroupPolicy policy; // only need set for dstination group
127 : uint32_t endpointNum; // only used for query result
128 : uint32_t rootModelId;
129 : };
130 :
131 : // endpoint: queue, communication channel, group
132 : struct Endpoint {
133 : EndpointType type; // endpoint type
134 : EndpointStatus status; // endpoint status
135 : uint32_t peerNum; // total instances, used by srcGroup
136 : uint32_t localId; // self id, started by 0, used by srcGroup
137 : uint32_t globalId; // endpoint global id
138 : uint32_t modelId;
139 : uint16_t resId; // resId
140 : uint32_t rootModelId;
141 : char_t rsv[10]; // reserved field
142 : union {
143 : FlowQueueAttr queueAttr; // queue attr
144 : MemQueueAttr memQueueAttr; // mem queue attr
145 : CommChannelAttr channelAttr; // communication channel attr
146 : GroupAttr groupAttr; // group attr
147 : } attr;
148 : };
149 :
150 : // group query
151 : struct GroupQuery {
152 : uint32_t endpointNum; // endpoint num, return value
153 : int32_t groupId; // group id
154 : };
155 :
156 : // route query
157 : struct RouteQuery {
158 : uint32_t routeNum; // route num, return value
159 : Endpoint src; // src endpoint
160 : Endpoint dst; // dst endpoint
161 : };
162 :
163 : // route query or group query
164 : struct ConfigQuery {
165 : QueryMode mode; // query mode
166 : union {
167 : GroupQuery groupQry; // group query
168 : RouteQuery routeQry; // route query
169 : } qry;
170 : };
171 :
172 : // group config
173 : struct GroupConfig {
174 : int32_t groupId; // group id, created by datagw server
175 : uint32_t endpointNum; // endpoint number
176 : Endpoint* endpoints; // multi queue or channel, memory malloc by user
177 : };
178 :
179 : // route info
180 : struct Route {
181 : RouteStatus status; // route status
182 : Endpoint src; // src endpoint
183 : Endpoint dst; // dst endpoint
184 : char_t rsv[32]; // reserved param
185 : };
186 :
187 : // routes config
188 : struct RoutesConfig {
189 : uint32_t routeNum; // route number
190 : Route* routes; // routes, memory malloc by user
191 : };
192 :
193 : // profiling config
194 : struct ProfilingConfig {
195 : ProfilingMode profMode; // profiling mode
196 : };
197 :
198 : struct HcclProtocolConfig {
199 : HcclProtocolType protocol;
200 : };
201 :
202 : struct DynamicSchedQueueAttr {
203 : uint32_t queueId;
204 : uint32_t deviceId;
205 : int32_t deviceType;
206 : uint32_t globalLogicId;
207 : bool isClientQ;
208 : };
209 :
210 : struct DynamicSchedConfigV2 {
211 : uint32_t rootModelId;
212 : DynamicSchedQueueAttr requestQ;
213 : DynamicSchedQueueAttr responseQ;
214 : char rsv[26];
215 : };
216 :
217 : struct ReDeployConfig {
218 : uint64_t rootModelIdsAddr; // ptr which point to rootModelIds(uint32_t)
219 : uint32_t rootModelNum; // rootModelId's number
220 : char rsv[4];
221 : };
222 :
223 : // config info, group config or routes config
224 : struct ConfigInfo {
225 : ConfigCmd cmd; // query mode, user donot need fill this param
226 : union {
227 : GroupConfig groupCfg; // group config
228 : RoutesConfig routesCfg; // routes config
229 : ProfilingConfig profCfg; // profiling config
230 : HcclProtocolConfig hcclProtocolCfg;
231 : DynamicSchedConfigV2* dynamicSchedCfgV2;
232 : ReDeployConfig reDeployCfg;
233 : } cfg;
234 : };
235 :
236 : // identify info in mbuf head
237 : struct IdentifyInfo {
238 : uint64_t transId = 0UL; // transaction id
239 : char_t rsv[52]; // reserved param
240 : uint32_t routeLabel;
241 : };
242 :
243 : #pragma pack(pop)
244 :
245 : class __attribute__((visibility("default"))) DgwClient {
246 : public:
247 : /**
248 : * Create instance of dgwClient.
249 : * @param deviceId The id of self cpu.
250 : * @return std::shared_ptr<DgwClient>: DgwClient ptr
251 : */
252 : static std::shared_ptr<DgwClient> GetInstance(const uint32_t deviceId);
253 :
254 : static std::shared_ptr<DgwClient> GetInstance(const uint32_t deviceId, const pid_t qsPid);
255 :
256 : static std::shared_ptr<DgwClient> GetInstance(const uint32_t deviceId, const pid_t qsPid, const bool proxy);
257 :
258 : /**
259 : * Initialize dgw server
260 : * @param dgwPid dgw server process id
261 : * @param procSign procSign
262 : * @return 0:success, other:failed.
263 : */
264 : int32_t Initialize(
265 : const uint32_t dgwPid, const std::string procSign, const bool isProxy = false, const int32_t timeout = -1);
266 :
267 : /**
268 : * Destroy dgw client
269 : * @param dgwPid dgw server process id
270 : * @param procSign procSign
271 : * @return 0:success, other:failed.
272 : */
273 : int32_t Finalize();
274 :
275 : /**
276 : * Create hccl communication handle
277 : * @param rankTable rank table
278 : * @param rankId rank id
279 : * @param reserve reserve params
280 : * @param handle handle
281 : * @return 0:success, other:failed.
282 : */
283 : int32_t CreateHcomHandle(
284 : const std::string& rankTable, const int32_t rankId, const void* const reserve, uint64_t& handle,
285 : const int32_t timeout = -1);
286 :
287 : /**
288 : * Destroy hccl communication handle
289 : * @param handle handle
290 : * @return 0:success, other:failed.
291 : */
292 : int32_t DestroyHcomHandle(const uint64_t handle, const int32_t timeout = -1);
293 :
294 : /**
295 : * Construct a new DgwClient object
296 : * @param deviceId device Id
297 : */
298 : explicit DgwClient(const uint32_t deviceId);
299 :
300 : explicit DgwClient(const uint32_t deviceId, const pid_t qsPid);
301 :
302 : explicit DgwClient(const uint32_t deviceId, const pid_t qsPid, const bool proxy);
303 :
304 : /**
305 : * Destroy the DgwClient object - default method
306 : */
307 17 : ~DgwClient() = default;
308 :
309 : public:
310 : /**
311 : * Update route config or group config
312 : * config routes: any route config failed, return failed
313 : * config group: group config failed, return failed
314 : * @param cfgInfo group config or route config
315 : * @param cfgRets config results
316 : * @return 0:success, other:failed.
317 : */
318 : int32_t UpdateConfig(ConfigInfo& cfgInfo, std::vector<int32_t>& cfgRets, const int32_t timeout = -1);
319 :
320 : /**
321 : * Query Route number or endpoint number in group
322 : * @param query config query info
323 : * @return 0:success, other:failed.
324 : */
325 : int32_t QueryConfigNum(ConfigQuery& query, const int32_t timeout = -1);
326 :
327 : /**
328 : * Query routes or group config
329 : * @param query config query info
330 : * @param cfgInfo routes config or group config
331 : * @return 0:success, other:failed.
332 : */
333 : int32_t QueryConfig(const ConfigQuery& query, ConfigInfo& cfgInfo, const int32_t timeout = -1);
334 :
335 : /**
336 : * wait config effect
337 : * @param timeout(s)
338 : * @return 0:success, other:failed.
339 : */
340 : int32_t WaitConfigEffect(const uint64_t timeout);
341 :
342 : /**
343 : * wait config effect
344 : * @param rsv 0:等待tag建链完成，如果没有tag要建链也直接返回成功
345 : * @param timeout(s)
346 : * @return 0:success, other:failed.
347 : */
348 : int32_t WaitConfigEffect(const int32_t rsv, const int32_t timeout);
349 :
350 : private:
351 : // config params for reducing parameters of API, only used inner
352 : struct ConfigParams {
353 : HcomHandleInfo* info; // hcom handle info, only invalid for create hcom handle
354 : ConfigQuery* query; // config query, only invalid for query
355 : ConfigInfo* cfgInfo; // config info
356 : size_t cfgLen; // config length
357 : size_t totalLen; // total length for mbuf
358 : };
359 :
360 : private:
361 : /**
362 : * Send synchronization event
363 : * @param msg message
364 : * @param msgLen message length
365 : * @param subEventId subevent id
366 : * @param timeout ms
367 : * @param qsProcMsgRsp return value
368 : * @return 0:success, other:failed.
369 : */
370 : int32_t SendEventToQsSync(
371 : const void* const msg, const size_t msgLen, const QueueSubEventType subEventId, QsProcMsgRsp& qsProcMsgRsp,
372 : const int32_t timeout = -1) const;
373 : /**
374 : * Check and calculate config info length
375 : * @param cfgInfo config info
376 : * @param cfgLen config length
377 : * @param dataList data list which need copy to mbuf
378 : * @param spareRoutes spare routes which transform memq to q
379 : * @param spareEndpoints spare endpoint which transform memq to q
380 : * @return 0:success, other:failed.
381 : */
382 : int32_t CalcConfigInfoLen(
383 : const ConfigInfo& cfgInfo, size_t& cfgLen, std::list<std::pair<uintptr_t, size_t>>& dataList,
384 : std::unique_ptr<Route[]>& spareRoutes, std::unique_ptr<Endpoint[]>& spareEndpoints) const;
385 :
386 : /**
387 : * Get operate configuration result
388 : * @param cfgInfo config info
389 : * @param mbufData mbuf data addr
390 : * @param cfgLen config info len
391 : * @param cfgRets config results only for update config
392 : * @param cmdRet command result
393 : * @return 0:success, other:failed.
394 : */
395 : int32_t GetOperateConfigRet(
396 : ConfigInfo& cfgInfo, const uintptr_t mbufData, const size_t cfgLen, std::vector<int32_t>& cfgRets,
397 : int32_t& cmdRet) const;
398 :
399 : /**
400 : * Get operate configuration result
401 : * @param subEventId subevent id
402 : * @param cfgParams config params
403 : * @param dataList data list which need copy to mbuf
404 : * @param cfgRets config results only for update config
405 : * @return 0:success, other:failed.
406 : */
407 : int32_t OperateConfigToServer(
408 : const QueueSubEventType subEventId, const ConfigParams& cfgParams,
409 : std::list<std::pair<uintptr_t, size_t>>& dataList, std::vector<int32_t>& cfgRets, const int32_t timeout = -1);
410 :
411 : /**
412 : * Calculate result length
413 : * @param cfgInfo config info
414 : * @param retLen result length
415 : * @return 0:success, other:failed.
416 : */
417 : int32_t CalcResultLen(const ConfigInfo& cfgInfo, size_t& retLen) const;
418 :
419 : /**
420 : * Get query config num result
421 : * @param query config query
422 : * @param mbufData mbuf data
423 : * @param cmdRet cmd result
424 : * @return 0:success, other:failed.
425 : */
426 : int32_t GetQryConfigNumRet(ConfigQuery& query, const uintptr_t mbufData, int32_t& cmdRet) const;
427 :
428 : /**
429 : * Query and check config num
430 : * @param query config query
431 : * @param cfgInfo config info
432 : * @return 0:success, other:failed.
433 : */
434 : int32_t CheckConfigNum(const ConfigQuery& query, ConfigInfo& cfgInfo);
435 :
436 : /**
437 : * Get operate group result
438 : * @param cfgInfo config info
439 : * @param mbufData mbuf data addr
440 : * @param cfgLen config info len
441 : * @param cfgRets config results only for update config
442 : * @param cmdRet command result
443 : * @return 0:success, other:failed.
444 : */
445 : int32_t GetUpdateGroupRet(
446 : ConfigInfo& cfgInfo, const uintptr_t mbufData, const size_t cfgLen, std::vector<int32_t>& cfgRets,
447 : int32_t& cmdRet) const;
448 :
449 : /**
450 : * Get operate route result
451 : * @param cfgInfo config info
452 : * @param mbufData mbuf data addr
453 : * @param cfgLen config info len
454 : * @param cfgRets config results only for update config
455 : * @param cmdRet command result
456 : * @return 0:success, other:failed.
457 : */
458 : int32_t GetUpdateRouteRet(
459 : const ConfigInfo& cfgInfo, const uintptr_t mbufData, const size_t cfgLen, std::vector<int32_t>& cfgRets,
460 : int32_t& cmdRet) const;
461 :
462 : /**
463 : * Get query route result
464 : * @param cfgInfo config info
465 : * @param mbufData mbuf data addr
466 : * @param cfgLen config info len
467 : * @param cfgRets config results only for update config
468 : * @param cmdRet command result
469 : * @return 0:success, other:failed.
470 : */
471 : int32_t GetQryRouteRet(
472 : const ConfigInfo& cfgInfo, const uintptr_t mbufData, const size_t cfgLen, std::vector<int32_t>& cfgRets,
473 : int32_t& cmdRet) const;
474 :
475 : /**
476 : * Get query group result
477 : * @param cfgInfo config info
478 : * @param mbufData mbuf data addr
479 : * @param cfgLen config info len
480 : * @param cfgRets config results only for update config
481 : * @param cmdRet command result
482 : * @return 0:success, other:failed.
483 : */
484 : int32_t GetQryGroupRet(
485 : const ConfigInfo& cfgInfo, const uintptr_t mbufData, const size_t cfgLen, std::vector<int32_t>& cfgRets,
486 : int32_t& cmdRet) const;
487 :
488 : /**
489 : * Get operate hcom handle result
490 : * @param subEventId sub event id
491 : * @param info hcom handle info
492 : * @param mbufData mbuf data
493 : * @param cfgLen config info len
494 : * @param cmdRet command result
495 : * @return 0:success, other:failed.
496 : */
497 : int32_t GetOperateHcomHandleRet(
498 : const QueueSubEventType subEventId, HcomHandleInfo& info, const uintptr_t mbufData, const size_t cfgLen,
499 : int32_t& cmdRet) const;
500 :
501 : int32_t ProcessEndpointDeviceId(Endpoint& endpoint) const;
502 :
503 : private:
504 : int32_t OperateToServerOnSameSide(
505 : const QueueSubEventType subEventId, const ConfigParams& cfgParams,
506 : std::list<std::pair<uintptr_t, size_t>>& dataList, std::vector<int32_t>& cfgRets, const int32_t timeout);
507 : int32_t OperateToServerOnOtherSide(
508 : const QueueSubEventType subEventId, const ConfigParams& cfgParams,
509 : std::list<std::pair<uintptr_t, size_t>>& dataList, std::vector<int32_t>& cfgRets, const int32_t timeout);
510 : void ExtractRetCode(
511 : const QueueSubEventType subEventId, const ConfigParams& cfgParams, const uintptr_t respPtr,
512 : std::vector<int32_t>& cfgRets, int32_t& cmdRet) const;
513 : int32_t InformServer(const QueueSubEventType subEventId, int32_t& cmdRet, const int32_t timeout);
514 :
515 : static int32_t GetPlatformInfo(const uint32_t deviceId);
516 :
517 : static bool IsNumeric(const std::string& str);
518 :
519 : static bool IsSupportSetVisibleDevices();
520 :
521 : static void SplitString(const std::string& str, std::vector<std::string>& result);
522 :
523 : static bool GetVisibleDevices();
524 :
525 : static int32_t ChangeUserDeviceIdToLogicDeviceId(const uint32_t userDevId, uint32_t& logicDevId);
526 :
527 : static int32_t ChangeDynamicScheduleDeviceId(const ConfigInfo& cfgInfo);
528 :
529 : uint32_t deviceId_;
530 : // dgw server pid
531 : pid_t qsPid_;
532 : // proc sign
533 : std::string procSign_;
534 : // dgw client pid
535 : pid_t curPid_;
536 : // dgw client group id
537 : uint32_t curGroupId_;
538 : // dgw client and qs server pipline queue id
539 : uint32_t piplineQueueId_;
540 : // dgw initialized flag
541 : bool initFlag_;
542 : std::mutex eventMutex_;
543 : std::mutex mutexForWaitConfig;
544 : bool isProxy_;
545 : bool isServerOldVersion_;
546 : };
547 : } // namespace bqs
548 : #endif
|
