Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #include "aicpu_profiling_manager.h"
      12              : 
      13              : #include "aicpu_schedule/aicpu_context.h"
      14              : #include "common/aicpu_sqe_context.h"
      15              : #include "common/aicpu_hccl_common.h"
      16              : #include "profiling_manager_device.h"
      17              : 
      18              : namespace dfx {
      19              : constexpr std::uint32_t HCCLINFO_REPORT_BATCH_NUM = 2;
      20              : uint64_t g_groupNameHashId{0U};
      21              : 
      22            2 : HcclResult AicpuProfilingManager::ReportTaskInfo()
      23              : {
      24            2 :     SqeContext *context = GetSqeContext();
      25            2 :     auto ctx = AicpuGetComContext();
      26            2 :     CHK_PTR_NULL(context->buffPtr);
      27            2 :     MsprofAicpuHcclTaskInfo taskInfos[HCCLINFO_REPORT_BATCH_NUM] = {0};
      28           66 :     for (uint32_t streamId = 0; streamId < AC_MAX_RANK_NUM; streamId++) {
      29           64 :         auto &buff = context->buffPtr[streamId];
      30           64 :         uint16_t &lastSqeIdx = ctx->profilingExtendInfo.lastSqeIdxs[streamId];
      31           64 :         HCCL_INFO("Rank %u stream %u has %u sqes, sqeCnt: %u, lastSqeIdx: %u", ctx->rankId, streamId,
      32              :                   buff.tailSqeIdx, buff.sqeCnt, lastSqeIdx);
      33           64 :         auto endIdx = static_cast<uint32_t>(buff.tailSqeIdx);
      34         2301 :         for (uint32_t idx = lastSqeIdx, batchId = 0; idx < endIdx; ++idx) {
      35         2237 :             auto& taskInfo = taskInfos[batchId++];
      36         2237 :             Ctx2MsprofAicpuMC2HcclInfo(ctx, taskInfo);
      37         2237 :             taskInfo.planeID = streamId;
      38         2237 :             SqeInfo sqeInfo;
      39         2237 :             SqeContextUtils::QuerySqeInfo(
      40         2237 :                 buff.localBuff + idx * AC_SQE_SIZE, buff.sqeType[idx], buff.addInfo[idx], &sqeInfo);
      41         2237 :             ProfilingExtendInfoHelper::SqeInfo2MsprofAicpuMC2HcclInfo(sqeInfo, taskInfo);
      42         2237 :             taskInfo.timeStamp = buff.profTimestap[idx]; // 时间戳
      43         2237 :             ProfilingManager::DumpHcclInfo(taskInfo, batchId, idx);
      44         2237 :             if (batchId == HCCLINFO_REPORT_BATCH_NUM || idx == (endIdx - 1)) {
      45         1119 :                 CHK_PRT(ProfilingManager::CallMsprofReportAdditionInfo(MSPROF_REPORT_AICPU_MC2_BATCH_HCCL_INFO,
      46              :                     0, taskInfos, sizeof(MsprofAicpuHcclTaskInfo) * batchId));
      47         1119 :                 batchId = 0;
      48         1119 :                 memset_s(taskInfos, sizeof(taskInfos), 0, sizeof(taskInfos));
      49              :             }
      50              :         }
      51           64 :         lastSqeIdx = buff.tailSqeIdx;
      52              :     }
      53            2 :     return HCCL_SUCCESS;
      54              : }
      55              : 
      56           48 : HcclResult AicpuProfilingManager::ReportTaskExecTimeLine(AicpuComProf *acprof, u32 turnOffset)
      57              : {
      58           48 :     if (!ProfilingManager::GetProfL1State()) {
      59           44 :         return HCCL_SUCCESS;
      60              :     }
      61            4 :     uint64_t taskId = 0U;
      62            4 :     uint32_t streamId = 0;
      63            4 :     if (AicpuGetStreamId == nullptr || AicpuGetTaskId == nullptr) {
      64            0 :         CHK_PRT_RET(aicpu::GetTaskAndStreamId(taskId, streamId) != aicpu::status_t::AICPU_ERROR_NONE,
      65              :         HCCL_ERROR("Failed to get task id and stream id."),
      66              :         HCCL_E_PARA);
      67              :     } else {
      68            4 :         streamId = AicpuGetStreamId();
      69            4 :         taskId = AicpuGetTaskId();
      70              :     }
      71            4 :     HCCL_INFO("[AicpuProfilingManager] [ReportTaskExecTimeLine] streamId = %u, taskId = %u", streamId, taskId);
      72              : 
      73              :     // 正常一个kfc算子的展开任务不会超过`AC_MAX_PROF_COMM_CNT`轮
      74            4 :     u32 workRcdCnt = acprof->workCnt;
      75            4 :     CHK_PRT_RET(workRcdCnt > AC_MAX_PROF_COMM_CNT,
      76              :         HCCL_ERROR("WorkRcdCnt %u should not bigger than %u,", workRcdCnt, AC_MAX_PROF_COMM_CNT),
      77              :         HCCL_E_PARA);
      78              : 
      79            4 :     for (u32 j = 0; j < workRcdCnt; j++) {
      80            0 :         AicpuComProfCommLoop *commRcd = &acprof->commLoop[j];
      81              :         AicpuKfcProfCommTurn commTurnProf;
      82            0 :         commTurnProf.serverStartTime = acprof->launchEntryTime;
      83            0 :         commTurnProf.waitMsgStartTime = acprof->commInitEndTime;
      84            0 :         commTurnProf.kfcAlgExeStartTime = commRcd->hccExecStartTime;
      85            0 :         commTurnProf.sendTaskStartTime = commRcd->sendTaskStartTime;
      86            0 :         commTurnProf.sendSqeFinishTime = commRcd->sendSqeFinishTime;
      87            0 :         commTurnProf.rtsqExeEndTime = acprof->receiveFinalizeTime;
      88            0 :         commTurnProf.serverEndTime = acprof->endTime;
      89            0 :         commTurnProf.dataLen = commRcd->dataLen;
      90              : 
      91            0 :         commTurnProf.deviceId = acprof->rankId;
      92            0 :         commTurnProf.streamId = streamId;
      93            0 :         commTurnProf.taskId = taskId;
      94            0 :         commTurnProf.commTurn = workRcdCnt;
      95            0 :         commTurnProf.currentTurn = j + turnOffset;
      96            0 :         commTurnProf.version = 0U; // default
      97            0 :         HCCL_INFO("Get one comm prof with details:[%s]", AicpuKfcProfCommTurnToString(commTurnProf).c_str());
      98            0 :         CHK_PRT(ProfilingManager::CallMsprofReportAdditionInfo(
      99              :             MSPROF_REPORT_AICPU_MC2_EXECUTE_COMM_TIME, GetCurCpuTimestamp(true), &commTurnProf, sizeof(commTurnProf)));
     100              :     }
     101            4 :     return HCCL_SUCCESS;
     102              : }
     103              : 
     104           80 : void AicpuProfilingManager::Init(const AicpuComContext *ctx)
     105              : {
     106           80 :     if (AdprofGetHashId == nullptr) {
     107            0 :         HCCL_INFO("AdprofGetHashId is null, just return");
     108            0 :         return;
     109              :     }
     110           80 :     ProfilingExtendInfoHelper::InitProfItemId();
     111              :     // ctx保证了非空
     112           80 :     u32 len = 0;                        // 初始化长度为0
     113          400 :     while (ctx->hcomId[len] != '\0') {  // 遍历数组，直到遇到'\0'
     114          320 :         len++;                          // 每遍历一个字符，长度加1
     115              :     }
     116           80 :     g_groupNameHashId = AdprofGetHashId(ctx->hcomId, len);
     117           80 :     HCCL_DEBUG("Using %s and len %u to get hash %lu", ctx->hcomId, len, g_groupNameHashId);
     118              : }
     119              : 
     120         2237 : void AicpuProfilingManager::Ctx2MsprofAicpuMC2HcclInfo(
     121              :     const AicpuComContext *ctx, MsprofAicpuHcclTaskInfo &msprofAicpuMC2HcclInfo)
     122              : {
     123         2237 :     ProfilingExtendInfoHelper::InitHcclInfo(msprofAicpuMC2HcclInfo);
     124         2237 :     msprofAicpuMC2HcclInfo.groupName = g_groupNameHashId;
     125         2237 :     msprofAicpuMC2HcclInfo.localRank = ctx->rankId;
     126         2237 :     msprofAicpuMC2HcclInfo.remoteRank = ctx->rankId;
     127         2237 :     msprofAicpuMC2HcclInfo.rankSize = ctx->rankNum;
     128         2237 :     msprofAicpuMC2HcclInfo.opType = static_cast<uint32_t>(ctx->reducekind);
     129         2237 : }
     130              : 
     131            0 : std::string AicpuProfilingManager::AicpuKfcProfCommTurnToString(const AicpuKfcProfCommTurn &aicpuKfcProfCommTurn)
     132              : {
     133            0 :     std::stringstream ss;
     134            0 :     ss << "serverStartTime: " << aicpuKfcProfCommTurn.serverStartTime;
     135            0 :     ss << " waitMsgStartTime: " << aicpuKfcProfCommTurn.waitMsgStartTime;
     136            0 :     ss << " kfcAlgExeStartTime: " << aicpuKfcProfCommTurn.kfcAlgExeStartTime;
     137            0 :     ss << " sendTaskStartTime: " << aicpuKfcProfCommTurn.sendTaskStartTime;
     138            0 :     ss << " sendSqeFinishTime: " << aicpuKfcProfCommTurn.sendSqeFinishTime;
     139            0 :     ss << " rtsqExeEndTime: " << aicpuKfcProfCommTurn.rtsqExeEndTime;
     140            0 :     ss << " serverEndTime: " << aicpuKfcProfCommTurn.serverEndTime;
     141            0 :     ss << " dataLen: " << aicpuKfcProfCommTurn.dataLen;
     142            0 :     ss << " deviceId: " << aicpuKfcProfCommTurn.deviceId;
     143            0 :     ss << " streamId: " << aicpuKfcProfCommTurn.streamId;
     144            0 :     ss << " taskId: " << aicpuKfcProfCommTurn.taskId;
     145            0 :     ss << " version: " << aicpuKfcProfCommTurn.version;
     146            0 :     ss << " total commTurn: " << aicpuKfcProfCommTurn.commTurn;
     147            0 :     ss << " currentTurn: " << aicpuKfcProfCommTurn.currentTurn;
     148            0 :     return ss.str();
     149            0 : }
     150              : 
     151              : }  // namespace dfx