Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #include "aicpu_profiling_manager.h"
      12              : 
      13              : #include "aicpu_schedule/aicpu_context.h"
      14              : #include "common/aicpu_sqe_context.h"
      15              : #include "common/aicpu_hccl_common.h"
      16              : #include "profiling_manager_device.h"
      17              : 
      18              : namespace dfx {
      19              : constexpr std::uint32_t HCCLINFO_REPORT_BATCH_NUM = 2;
      20              : uint64_t g_groupNameHashId{0U};
      21              : 
      22            2 : HcclResult AicpuProfilingManager::ReportTaskInfo()
      23              : {
      24            2 :     SqeContext* context = GetSqeContext();
      25            2 :     auto ctx = AicpuGetComContext();
      26            2 :     CHK_PTR_NULL(context->buffPtr);
      27            2 :     MsprofAicpuHcclTaskInfo taskInfos[HCCLINFO_REPORT_BATCH_NUM] = {};
      28           66 :     for (uint32_t streamId = 0; streamId < AC_MAX_RANK_NUM; streamId++) {
      29           64 :         auto& buff = context->buffPtr[streamId];
      30           64 :         uint16_t& lastSqeIdx = ctx->profilingExtendInfo.lastSqeIdxs[streamId];
      31           64 :         HCCL_INFO(
      32              :             "Rank %u stream %u has %u sqes, sqeCnt: %u, lastSqeIdx: %u", ctx->rankId, streamId, buff.tailSqeIdx,
      33              :             buff.sqeCnt, lastSqeIdx);
      34           64 :         auto endIdx = static_cast<uint32_t>(buff.tailSqeIdx);
      35         2301 :         for (uint32_t idx = lastSqeIdx, batchId = 0; idx < endIdx; ++idx) {
      36         2237 :             auto& taskInfo = taskInfos[batchId++];
      37         2237 :             Ctx2MsprofAicpuMC2HcclInfo(ctx, taskInfo);
      38         2237 :             taskInfo.planeID = streamId;
      39         2237 :             SqeInfo sqeInfo;
      40         2237 :             SqeContextUtils::QuerySqeInfo(
      41         2237 :                 buff.localBuff + idx * AC_SQE_SIZE, buff.sqeType[idx], buff.addInfo[idx], &sqeInfo);
      42         2237 :             ProfilingExtendInfoHelper::SqeInfo2MsprofAicpuMC2HcclInfo(sqeInfo, taskInfo);
      43         2237 :             taskInfo.timeStamp = buff.profTimestap[idx]; // 时间戳
      44         2237 :             ProfilingManager::DumpHcclInfo(taskInfo, batchId, idx);
      45         2237 :             if (batchId == HCCLINFO_REPORT_BATCH_NUM || idx == (endIdx - 1)) {
      46         1119 :                 CHK_PRT(ProfilingManager::CallMsprofReportAdditionInfo(
      47              :                     MSPROF_REPORT_AICPU_MC2_BATCH_HCCL_INFO, 0, taskInfos, sizeof(MsprofAicpuHcclTaskInfo) * batchId));
      48         1119 :                 batchId = 0;
      49         1119 :                 memset_s(taskInfos, sizeof(taskInfos), 0, sizeof(taskInfos));
      50              :             }
      51              :         }
      52           64 :         lastSqeIdx = buff.tailSqeIdx;
      53              :     }
      54            2 :     return HCCL_SUCCESS;
      55              : }
      56              : 
      57           48 : HcclResult AicpuProfilingManager::ReportTaskExecTimeLine(AicpuComProf* acprof, u32 turnOffset)
      58              : {
      59           48 :     if (!ProfilingManager::GetProfL1State()) {
      60           44 :         return HCCL_SUCCESS;
      61              :     }
      62            4 :     uint64_t taskId = 0U;
      63            4 :     uint32_t streamId = 0;
      64            4 :     if (AicpuGetStreamId == nullptr || AicpuGetTaskId == nullptr) {
      65            0 :         CHK_PRT_RET(
      66              :             aicpu::GetTaskAndStreamId(taskId, streamId) != aicpu::status_t::AICPU_ERROR_NONE,
      67              :             HCCL_ERROR("Failed to get task id and stream id."), HCCL_E_PARA);
      68              :     } else {
      69            4 :         streamId = AicpuGetStreamId();
      70            4 :         taskId = AicpuGetTaskId();
      71              :     }
      72            4 :     HCCL_INFO("[AicpuProfilingManager] [ReportTaskExecTimeLine] streamId = %u, taskId = %u", streamId, taskId);
      73              : 
      74              :     // 正常一个kfc算子的展开任务不会超过`AC_MAX_PROF_COMM_CNT`轮
      75            4 :     u32 workRcdCnt = acprof->workCnt;
      76            4 :     CHK_PRT_RET(
      77              :         workRcdCnt > AC_MAX_PROF_COMM_CNT,
      78              :         HCCL_ERROR("WorkRcdCnt %u should not bigger than %u,", workRcdCnt, AC_MAX_PROF_COMM_CNT), HCCL_E_PARA);
      79              : 
      80            4 :     for (u32 j = 0; j < workRcdCnt; j++) {
      81            0 :         AicpuComProfCommLoop* commRcd = &acprof->commLoop[j];
      82              :         AicpuKfcProfCommTurn commTurnProf;
      83            0 :         commTurnProf.serverStartTime = acprof->launchEntryTime;
      84            0 :         commTurnProf.waitMsgStartTime = acprof->commInitEndTime;
      85            0 :         commTurnProf.kfcAlgExeStartTime = commRcd->hccExecStartTime;
      86            0 :         commTurnProf.sendTaskStartTime = commRcd->sendTaskStartTime;
      87            0 :         commTurnProf.sendSqeFinishTime = commRcd->sendSqeFinishTime;
      88            0 :         commTurnProf.rtsqExeEndTime = acprof->receiveFinalizeTime;
      89            0 :         commTurnProf.serverEndTime = acprof->endTime;
      90            0 :         commTurnProf.dataLen = commRcd->dataLen;
      91              : 
      92            0 :         commTurnProf.deviceId = acprof->rankId;
      93            0 :         commTurnProf.streamId = streamId;
      94            0 :         commTurnProf.taskId = taskId;
      95            0 :         commTurnProf.commTurn = workRcdCnt;
      96            0 :         commTurnProf.currentTurn = j + turnOffset;
      97            0 :         commTurnProf.version = 0U; // default
      98            0 :         HCCL_INFO("Get one comm prof with details:[%s]", AicpuKfcProfCommTurnToString(commTurnProf).c_str());
      99            0 :         CHK_PRT(ProfilingManager::CallMsprofReportAdditionInfo(
     100              :             MSPROF_REPORT_AICPU_MC2_EXECUTE_COMM_TIME, GetCurCpuTimestamp(true), &commTurnProf, sizeof(commTurnProf)));
     101              :     }
     102            4 :     return HCCL_SUCCESS;
     103              : }
     104              : 
     105           80 : void AicpuProfilingManager::Init(const AicpuComContext* ctx)
     106              : {
     107           80 :     if (AdprofGetHashId == nullptr) {
     108            0 :         HCCL_INFO("AdprofGetHashId is null, just return");
     109            0 :         return;
     110              :     }
     111           80 :     ProfilingExtendInfoHelper::InitProfItemId();
     112              :     // ctx保证了非空
     113           80 :     u32 len = 0;                       // 初始化长度为0
     114          400 :     while (ctx->hcomId[len] != '\0') { // 遍历数组，直到遇到'\0'
     115          320 :         len++;                         // 每遍历一个字符，长度加1
     116              :     }
     117           80 :     g_groupNameHashId = AdprofGetHashId(ctx->hcomId, len);
     118           80 :     HCCL_DEBUG("Using %s and len %u to get hash %lu", ctx->hcomId, len, g_groupNameHashId);
     119              : }
     120              : 
     121         2237 : void AicpuProfilingManager::Ctx2MsprofAicpuMC2HcclInfo(
     122              :     const AicpuComContext* ctx, MsprofAicpuHcclTaskInfo& msprofAicpuMC2HcclInfo)
     123              : {
     124         2237 :     ProfilingExtendInfoHelper::InitHcclInfo(msprofAicpuMC2HcclInfo);
     125         2237 :     msprofAicpuMC2HcclInfo.groupName = g_groupNameHashId;
     126         2237 :     msprofAicpuMC2HcclInfo.localRank = ctx->rankId;
     127         2237 :     msprofAicpuMC2HcclInfo.remoteRank = ctx->rankId;
     128         2237 :     msprofAicpuMC2HcclInfo.rankSize = ctx->rankNum;
     129         2237 :     msprofAicpuMC2HcclInfo.opType = static_cast<uint32_t>(ctx->reducekind);
     130         2237 : }
     131              : 
     132            0 : std::string AicpuProfilingManager::AicpuKfcProfCommTurnToString(const AicpuKfcProfCommTurn& aicpuKfcProfCommTurn)
     133              : {
     134            0 :     std::stringstream ss;
     135            0 :     ss << "serverStartTime: " << aicpuKfcProfCommTurn.serverStartTime;
     136            0 :     ss << " waitMsgStartTime: " << aicpuKfcProfCommTurn.waitMsgStartTime;
     137            0 :     ss << " kfcAlgExeStartTime: " << aicpuKfcProfCommTurn.kfcAlgExeStartTime;
     138            0 :     ss << " sendTaskStartTime: " << aicpuKfcProfCommTurn.sendTaskStartTime;
     139            0 :     ss << " sendSqeFinishTime: " << aicpuKfcProfCommTurn.sendSqeFinishTime;
     140            0 :     ss << " rtsqExeEndTime: " << aicpuKfcProfCommTurn.rtsqExeEndTime;
     141            0 :     ss << " serverEndTime: " << aicpuKfcProfCommTurn.serverEndTime;
     142            0 :     ss << " dataLen: " << aicpuKfcProfCommTurn.dataLen;
     143            0 :     ss << " deviceId: " << aicpuKfcProfCommTurn.deviceId;
     144            0 :     ss << " streamId: " << aicpuKfcProfCommTurn.streamId;
     145            0 :     ss << " taskId: " << aicpuKfcProfCommTurn.taskId;
     146            0 :     ss << " version: " << aicpuKfcProfCommTurn.version;
     147            0 :     ss << " total commTurn: " << aicpuKfcProfCommTurn.commTurn;
     148            0 :     ss << " currentTurn: " << aicpuKfcProfCommTurn.currentTurn;
     149            0 :     return ss.str();
     150            0 : }
     151              : 
     152              : } // namespace dfx