Coverage for /opt/cloud/slavespace/usr1/096471637100f3de0fcfc01072822a80/ut/src/asys/common/device.py: 57%

1#!/usr/bin/env python3

2# -*- coding: utf-8 -*-

3# ----------------------------------------------------------------------------

6# Licensed under the Apache License, Version 2.0 (the "License");

7# you may not use this file except in compliance with the License.

8# You may obtain a copy of the License at

10# http://www.apache.org/licenses/LICENSE-2.0

11#

12# Unless required by applicable law or agreed to in writing, software

13# distributed under the License is distributed on an "AS IS" BASIS,

14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

15# See the License for the specific language governing permissions and

16# limitations under the License.

17# ----------------------------------------------------------------------------

19import ctypes

20from common import RetCode

21from common import log_debug, log_error

22from common.const import AI_CORE_FREQUENCY, HBM_BANDWIDTH_USE, NOT_SUPPORT, UNKNOWN

23import common.interface as interface

24from drv import LoadSoType

26SOC_TEMP_ID = 11

28MAX_CHIP_NAME = 32

29MODULE_TYPE_AICORE = 4

30MODULE_TYPE_VECTOR_CORE = 7

31MODULE_TYPE_AICPU = 1

32MODULE_TYPE_CCPU = 2

33INFO_TYPE_CORE_NUM = 3

35DSMI_DEVICE_TYPE_HBM = 2

36DSMI_MAIN_CMD_MEMORY = 9

37DSMI_SUB_CMD_HBM_MEMORY = 35

38DSMI_MEMORY_RESERVE_LEN = 38

40DSMI_TAG_SENSOR_TEMP_LEN = 2

41DSMI_TAG_SENSOR_NTC_TEMP_LEN = 4

42SENSOR_DATA_MAX_LEN = 16

44ACL_DEV_ATTR_NPU_ARCH = 601

46MODULE_TYPE_SYSTEM = 0

47INFO_TYPE_MASTERID = 2

49MEMEORY_CONVERT_RATIO = 1024

51DSMI_ERROR_CORE = {

52 1: "the device does not exist.",

53 2: "invalid device ID.",

54 3: "invalid value",

55 4: "invalid handle",

56 6: "out of memory",

57 7: "inner err",

58 8: "para error",

59 10: "repeated init",

60 11: "not exist",

61 13: "device busy",

62 14: "device nor resources",

63 16: "wait timeout",

64 17: "iocrl failed",

65 27: "send mesg failed",

66 37: "over limit",

67 38: "file ops",

68 46: "oper not permitted",

69 51: "try again",

70 58: "memory opt fai",

71 86: "partitio not right",

72 87: "resource occupied",

73 89: "resume",

74 91: "bist hw err",

75 92: "bist sw err",

76 93: "dup config",

77 94: "power of fail",

78 65534: "not support"

79}

81class DsmiChipInfoStru(ctypes.Structure):

82 _fields_ = [

83 ("chip_type", ctypes.c_char * MAX_CHIP_NAME),

84 ("chip_name", ctypes.c_char * MAX_CHIP_NAME),

85 ("chip_ver", ctypes.c_char * MAX_CHIP_NAME),

86 ]

89class DsmiAicpuInfoStru(ctypes.Structure):

90 _fields_ = [

91 ("maxFreq", ctypes.c_int),

92 ("curFreq", ctypes.c_int),

93 ("aicpuNum", ctypes.c_int),

94 ("utilRate", ctypes.c_int * 16)

95 ]

98class DsmiMemoryInfoStru(ctypes.Structure):

99 _fields_ = [

100 ("memory_size", ctypes.c_ulonglong),

101 ("freq", ctypes.c_uint),

102 ("utiliza", ctypes.c_uint),

103 ]

104

105

106class DsmiNormalMemoryInfoStru(ctypes.Structure):

107 _fields_ = [

108 ("total_size", ctypes.c_uint),

109 ("used_size", ctypes.c_uint),

110 ("reserve", ctypes.c_char * DSMI_MEMORY_RESERVE_LEN),

111 ]

112

113

114class DsmiHBMInfoStru(ctypes.Structure):

115 _fields_ = [

116 ("memory_size", ctypes.c_ulonglong),

117 ("freq", ctypes.c_uint),

118 ("memory_usage", ctypes.c_ulonglong),

119 ("temp", ctypes.c_int),

120 ("bandwith_util_rate", ctypes.c_uint),

121 ]

122

123

124class DsmiPowerInfoStru(ctypes.Structure):

125 _fields_ = [

126 ("power", ctypes.c_short)

127 ]

128

129

130class DsmiTagSensorInfo(ctypes.Union):

131 _fields_ = [

132 ("uchar", ctypes.c_char),

133 ("ushort", ctypes.c_ushort),

134 ("uint", ctypes.c_uint),

135 ("iint", ctypes.c_int),

136 ("temp", ctypes.c_char * DSMI_TAG_SENSOR_TEMP_LEN),

137 ("ntc_temp", ctypes.c_int * DSMI_TAG_SENSOR_NTC_TEMP_LEN),

138 ("data", ctypes.c_uint * SENSOR_DATA_MAX_LEN),

139 ]

140

141

142class AmlCpuInfo(ctypes.Structure):

143 _fields_ = [

144 ("aicpuCount", ctypes.c_uint16),

145 ("ccpuCount", ctypes.c_uint16),

146 ("ccpuVoltage", ctypes.c_uint16),

147 ("ccpuFrequency", ctypes.c_uint16)

148 ]

149

150

151class AmlAicoreInfo(ctypes.Structure):

152 _fields_ = [

153 ("aicCount", ctypes.c_uint16),

154 ("aicVoltage", ctypes.c_uint16),

155 ("aicFrequency", ctypes.c_uint16)

156 ]

157

158

159class AmlBusInfo(ctypes.Structure):

160 _fields_ = [

161 ("busVoltage", ctypes.c_uint16),

162 ("ringFrequency", ctypes.c_uint16),

163 ("cpuFrequency", ctypes.c_uint16),

164 ("mataFrequency", ctypes.c_uint16),

165 ("l2bufferFrequency", ctypes.c_uint16),

166 ]

167

168

169class AmlHbmInfo(ctypes.Structure):

170 _fields_ = [

171 ("hbmVoltage", ctypes.c_uint16),

172 ("hbmFrequency", ctypes.c_uint16)

173 ]

174

175

176class DsmiEccPageStru(ctypes.Structure):

177 _fields_ = [

178 ("correctedEccErrorsTotal", ctypes.c_int),

179 ("ucorrectedEccErrorsTotal", ctypes.c_int),

180 ("isolatedSingleBitError", ctypes.c_int),

181 ("isolatedDoubleBitError", ctypes.c_int),

182 ]

183

184

185class DeviceInfo:

186

187 UNSUPPORTED_KEY_WORDS = [NOT_SUPPORT] # 查询信息返回不支持的关键字数组

188

189 def __init__(self):

190 self.dsmi_handle = LoadSoType().get_drvdsmi_env_type()

191 self.hal_handle = LoadSoType().get_drvhal_env_type()

192 self.ascend_ml = LoadSoType().get_ascend_ml()

193 self.ascend_cl = LoadSoType().get_ascend_cl()

194

195 @staticmethod

196 def check_status(ret, msg="Failed to query data"):

197 if ret == 0:

198 return True

199 msg += ", %s" % DSMI_ERROR_CORE.get(ret)

200 log_debug(msg)

201 return False

202

203 @staticmethod

204 def get_device_info_loop(device_num, func, err):

205 for device_id in range(device_num):

206 ret = func(device_id)

207 if ret and ret != err:

208 return ret

209 return err

210

211 def get_device_cpu_info(self, device_id):

212 if self.ascend_ml == RetCode.FAILED:

213 return [NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT]

214 cpu_info = ctypes.pointer(AmlCpuInfo())

215 try:

216 ret = self.ascend_ml.AmlDeviceGetCpuInfo(device_id, cpu_info)

217 except Exception as e:

218 log_error(f"Get device cpu info failed, error_msg: {e}")

219 return [NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT]

220

221 if not self.check_status(ret, "get device cpu info failed"):

222 return [NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT]

223

224 ai_cpu_c = cpu_info.contents.aicpuCount

225 c_cpu_c = cpu_info.contents.ccpuCount

226 c_cpu_v = cpu_info.contents.ccpuVoltage

227 c_cpu_f = cpu_info.contents.ccpuFrequency

228

229 return [ai_cpu_c, c_cpu_c, c_cpu_v, c_cpu_f]

230

231 def get_device_aic_info(self, device_id):

232 if self.ascend_ml == RetCode.FAILED:

233 return [NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT]

234 aic_info = ctypes.pointer(AmlAicoreInfo())

235 try:

236 ret = self.ascend_ml.AmlDeviceGetAicoreInfo(device_id, aic_info)

237 except Exception as e:

238 log_error(f"Get device aic info failed, error_msg: {e}")

239 return [NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT]

240

241 if not self.check_status(ret, "get device aic info failed"):

242 return [NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT]

243

244 aic_c = aic_info.contents.aicCount

245 aic_v = aic_info.contents.aicVoltage

246 aic_f = aic_info.contents.aicFrequency

247

248 return [aic_c, aic_v, aic_f]

249

250 def get_device_bus_info(self, device_id):

251 if self.ascend_ml == RetCode.FAILED:

252 return [NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT]

253 bus_info = ctypes.pointer(AmlBusInfo())

254 try:

255 ret = self.ascend_ml.AmlDeviceGetBusInfo(device_id, bus_info)

256 except Exception as e:

257 log_error(f"Get device bus info failed, error_msg: {e}")

258 return [NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT]

259

260 if not self.check_status(ret, "get device bus info failed"):

261 return [NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT]

262

263 bus_v = bus_info.contents.busVoltage

264 ring_f = bus_info.contents.ringFrequency

265 cpu_f = bus_info.contents.cpuFrequency

266 mate_f = bus_info.contents.mataFrequency

267 l2_buf_f = bus_info.contents.l2bufferFrequency

268

269 return [bus_v, ring_f, cpu_f, mate_f, l2_buf_f]

270

271 def get_device_hbm_volt_freq(self, device_id):

272 if self.ascend_ml == RetCode.FAILED:

273 return [NOT_SUPPORT, NOT_SUPPORT]

274 hbm_info = ctypes.pointer(AmlHbmInfo())

275 try:

276 ret = self.ascend_ml.AmlDeviceGetHbmInfo(device_id, hbm_info)

277 except Exception as e:

278 log_error(f"Get device hbm volt & freq failed, error_msg: {e}")

279 return [NOT_SUPPORT, NOT_SUPPORT]

280

281 if not self.check_status(ret, "get device hbm volt & freq failed"):

282 return [NOT_SUPPORT, NOT_SUPPORT]

283

284 hbm_v = hbm_info.contents.hbmVoltage

285 hbm_f = hbm_info.contents.hbmFrequency

286

287 return [hbm_v, hbm_f]

288

289 def get_device_count(self):

290 """Obtains the number of devices."""

291 p_device_count = ctypes.pointer(ctypes.c_int())

292 try:

293 ret = self.dsmi_handle.dsmi_get_device_count(p_device_count)

294 except AttributeError:

295 return 0

296 if not self.check_status(ret, "Get device count failed"):

297 return 0

298 return p_device_count.contents.value

299

300 def get_chip_info(self, device_id):

301 """Obtains device chip information."""

302 p_chip_info = ctypes.pointer(DsmiChipInfoStru())

303 try:

304 ret = self.hal_handle.halGetChipInfo(device_id, p_chip_info)

305 except AttributeError:

306 return UNKNOWN

307 if not self.check_status(ret, "Get chip info failed"):

308 return UNKNOWN

309 chip_type = p_chip_info.contents.chip_type.decode()

310 chip_name = p_chip_info.contents.chip_name.decode()

311 chip_ver = p_chip_info.contents.chip_ver.decode()

312 return " ".join([chip_type, chip_name, chip_ver])

313

314 def get_aicpu_count(self, device_id):

315 """Obtains device aicpu information."""

316 module_type_aicpu = ctypes.c_int(MODULE_TYPE_AICPU)

317 type_core_num = ctypes.c_int(INFO_TYPE_CORE_NUM)

318 p_aicpu_count = ctypes.pointer(ctypes.c_int())

319 try:

320 ret = self.hal_handle.halGetDeviceInfo(device_id, module_type_aicpu, type_core_num, p_aicpu_count)

321 except AttributeError:

322 return NOT_SUPPORT

323 if not self.check_status(ret, "Get aicpu count failed"):

324 return NOT_SUPPORT

325 return p_aicpu_count.contents.value

326

327 def get_ccpu_count(self, device_id):

328 module_type_ccpu = ctypes.c_int(MODULE_TYPE_CCPU)

329 type_core_num = ctypes.c_int(INFO_TYPE_CORE_NUM)

330 p_ccpu_count = ctypes.pointer(ctypes.c_int())

331 try:

332 ret = self.hal_handle.halGetDeviceInfo(device_id, module_type_ccpu, type_core_num, p_ccpu_count)

333 except AttributeError:

334 return NOT_SUPPORT

335

336 if not self.check_status(ret, "Get device cpu count failed"):

337 return NOT_SUPPORT

338

339 return p_ccpu_count.contents.value

340

341 def get_device_health(self, device_id):

342 """Obtaining the device health status"""

343 device_health_status = {0: "Healthy", 1: "Warning", 2: "Alarm", 3: "Critical"}

344

345 p_health_count = ctypes.pointer(ctypes.c_int())

346 try:

347 ret = self.dsmi_handle.dsmi_get_device_health(device_id, p_health_count)

348 except AttributeError:

349 return UNKNOWN

350 if not self.check_status(ret, "Get device health failed"):

351 return UNKNOWN

352 device_health_count = p_health_count.contents.value

353 if device_health_count in device_health_status.keys():

354 return device_health_status.get(device_health_count)

355 return UNKNOWN

356

357 def get_device_errorcode(self, device_id):

358 """Obtaining the device error code"""

359 error_list = list()

360 pyarray = [0]

361 perrorcode = (ctypes.c_uint * 128)(*pyarray)

362 perrorinfo = (ctypes.c_char * 255)(*pyarray)

363 p_error_count = ctypes.pointer(ctypes.c_int())

364 try:

365 ret = self.dsmi_handle.dsmi_get_device_errorcode(device_id, p_error_count, perrorcode)

366 except AttributeError:

367 return error_list

368 if not self.check_status(ret, "Get error code failed"):

369 return error_list

370 error_code = p_error_count.contents.value

371 for i in range(error_code):

372 ret = self.dsmi_handle.dsmi_query_errorstring(device_id, perrorcode[i], perrorinfo, 256)

373 error_info = ""

374 for info in perrorinfo:

375 error_info += str(info.decode())

376 if not self.check_status(ret, "Get error code failed"):

377 error_list.append([hex(perrorcode[i]), "NA"])

378 else:

379 error_list.append([hex(perrorcode[i]), str(error_info.strip("\x00"))])

380 return error_list

381

382 def get_aicore_count(self, device_id):

383 module_type_aicore = ctypes.c_int(MODULE_TYPE_AICORE)

384 type_core_num = ctypes.c_int(INFO_TYPE_CORE_NUM)

385 p_aicore_count = ctypes.pointer(ctypes.c_int())

386 try:

387 ret = self.hal_handle.halGetDeviceInfo(device_id, module_type_aicore, type_core_num, p_aicore_count)

388 except AttributeError:

389 return NOT_SUPPORT

390

391 if not self.check_status(ret, "Get device aicore count failed"):

392 return NOT_SUPPORT

393

394 return p_aicore_count.contents.value

395

396 def get_veccore_count(self, device_id):

397 type_vector_core = ctypes.c_int(MODULE_TYPE_VECTOR_CORE)

398 type_core_num = ctypes.c_int(INFO_TYPE_CORE_NUM)

399 p_veccore_count = ctypes.pointer(ctypes.c_int())

400 try:

401 ret = self.hal_handle.halGetDeviceInfo(device_id, type_vector_core, type_core_num, p_veccore_count)

402 except AttributeError:

403 return NOT_SUPPORT

404

405 if not self.check_status(ret, "Get device veccore count failed"):

406 return NOT_SUPPORT

407

408 return p_veccore_count.contents.value

409

410 def get_npu_arch(self, device_id):

411 npu_arch = ctypes.c_int64()

412 try:

413 ret = self.ascend_cl.aclrtGetDeviceInfo(device_id, ACL_DEV_ATTR_NPU_ARCH, ctypes.byref(npu_arch))

414 except AttributeError:

415 return NOT_SUPPORT

416

417 if not self.check_status(ret, "Get npu arch info failed"):

418 return NOT_SUPPORT

419 return npu_arch.value

420

421 def get_device_power(self, device_id):

422 p_power_info = ctypes.pointer(DsmiPowerInfoStru())

423 try:

424 ret = self.dsmi_handle.dsmi_get_device_power_info(device_id, p_power_info)

425 except AttributeError:

426 return NOT_SUPPORT

427 if not self.check_status(ret, "Get power info failed"):

428 return NOT_SUPPORT

429 return round(p_power_info.contents.power * 0.1, 1) # power unit is W

430

431 def get_device_temperature(self, device_id):

432 p_temperature = ctypes.pointer(ctypes.c_int())

433 try:

434 ret = self.dsmi_handle.dsmi_get_device_temperature(ctypes.c_int32(device_id), p_temperature)

435 except AttributeError:

436 return NOT_SUPPORT

437 if not self.check_status(ret, "Get temperature info failed"):

438 return NOT_SUPPORT

439 return p_temperature.contents.value

440

441 def get_device_frequency(self, device_id, device_type):

442 """

443 device_id: device id

444 device_type:

445 1 memory Frequency

446 2 Control CPU Frequency

447 3 HBM

448 4 AI CORE Current frequency

449 5 AI CORE rated frequency

450 6 Vector CORE Current frequency

451 """

452 p_frequency = ctypes.pointer(ctypes.c_int())

453 try:

454 ret = self.dsmi_handle.dsmi_get_device_frequency(device_id, device_type, p_frequency)

455 except AttributeError:

456 return NOT_SUPPORT

457 if not self.check_status(ret, "Get frequency info failed"):

458 return NOT_SUPPORT

459 return p_frequency.contents.value

460

461 def get_device_voltage(self, device_id):

462 """

463 device_id device_id

464 return voltage pvoltage * 0.01V * 1000 MV

465 """

466

467 p_voltage = ctypes.pointer(ctypes.c_uint())

468 try:

469 ret = self.dsmi_handle.dsmi_get_device_voltage(device_id, p_voltage)

470 except AttributeError:

471 return NOT_SUPPORT

472 if not self.check_status(ret, "Get voltage info failed"):

473 return NOT_SUPPORT

474 return int(p_voltage.contents.value * 0.01 * 1000)

475

476 def get_device_utilization_rate(self, device_id, device_type):

477 """

478 device_id: Device Id

479 device_type:

480 1: memory 2: Ai Core 3: Ai Cpu 4: Control CPU

481 5: memory Bandwidth 6: HBM 8: DDR 10: HBM Bandwidth

482 12: Vector Core

483 return:

484 utilization

485 """

486

487 p_utilization = ctypes.pointer(ctypes.c_uint())

488 try:

489 ret = self.dsmi_handle.dsmi_get_device_utilization_rate(device_id, device_type, p_utilization)

490 except AttributeError:

491 return NOT_SUPPORT

492 if not self.check_status(ret, "Get utilization info failed"):

493 return NOT_SUPPORT

494 return p_utilization.contents.value

495

496 def get_device_memory_info(self, device_id):

497 p_memory_info = ctypes.pointer(DsmiMemoryInfoStru())

498 try:

499 ret = self.dsmi_handle.dsmi_get_memory_info(device_id, p_memory_info)

500 except AttributeError:

501 return NOT_SUPPORT, NOT_SUPPORT

502 if not self.check_status(ret, "Get ddr memory info failed"):

503 return NOT_SUPPORT, NOT_SUPPORT

504

505 if "310 " in self.get_chip_info(device_id):

506 memory_size = p_memory_info.contents.memory_size

507 else:

508 memory_size = p_memory_info.contents.memory_size // MEMEORY_CONVERT_RATIO

509 utiliza = p_memory_info.contents.utiliza

510 return memory_size, round(memory_size * utiliza / 100, 2)

511

512 def get_device_hbm_info(self, device_id):

513 p_memory_info = ctypes.pointer(DsmiHBMInfoStru())

514 try:

515 ret = self.dsmi_handle.dsmi_get_hbm_info(device_id, p_memory_info)

516 except AttributeError:

517 return [NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT]

518 if not self.check_status(ret, "Get hbm memory info failed"):

519 return [NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT, NOT_SUPPORT]

520

521 memory_size = p_memory_info.contents.memory_size // MEMEORY_CONVERT_RATIO

522 usage = p_memory_info.contents.memory_usage

523 bandwidth = p_memory_info.contents.bandwith_util_rate

524 temp = p_memory_info.contents.temp

525 return [memory_size, round(usage / MEMEORY_CONVERT_RATIO, 2), temp, bandwidth]

526

527 def get_ecc_isolated_page(self, device_id):

528 p_device_ecc_info = ctypes.pointer(DsmiEccPageStru())

529 dsmi_device_type_hbm = ctypes.c_int(DSMI_DEVICE_TYPE_HBM)

530 try:

531 ret = self.dsmi_handle.dsmi_get_total_ecc_isolated_pages_info(device_id,

532 dsmi_device_type_hbm, p_device_ecc_info)

533 except AttributeError:

534 return "-"

535 if not self.check_status(ret, "Aggregation of uncorrected ECC errors failed."):

536 return "-"

537 return p_device_ecc_info.contents.ucorrectedEccErrorsTotal

538

539 def clear_ecc_isolated(self, device_id):

540 """

541 Clears historical ECC error statistics and isolation page information.

542 """

543 try:

544 ret = self.dsmi_handle.dsmi_clear_ecc_isolated_statistics_info(device_id)

545 if ret != 0:

546 log_error("Clear ecc isolated failed")

547 except AttributeError as e:

548 log_error(e)

549

550 def get_phyid_from_logicid(self, device_id):

551 phyid = ctypes.pointer(ctypes.c_uint32(0))

552 try:

553 ret = self.hal_handle.drvDeviceGetPhyIdByIndex(ctypes.c_int32(device_id), phyid)

554 except AttributeError as e:

555 log_error(f"Get PhyId by device:{device_id} failed, error_msg: {e}")

556 return RetCode.FAILED

557

558 if ret != 0:

559 log_error(f"Get PhyId by device:{device_id} failed")

560 return RetCode.FAILED

561

562 return phyid.contents.value

563

564 def get_masterid_from_phyid(self, phyid):

565 master_id = ctypes.pointer(ctypes.c_int64(0))

566 try:

567 ret = self.hal_handle.halGetDeviceInfo(

568 ctypes.c_uint32(phyid),

569 ctypes.c_int32(MODULE_TYPE_SYSTEM),

570 ctypes.c_int32(INFO_TYPE_MASTERID),

571 master_id

572 )

573 except AttributeError as e:

574 log_error(f"Get MasterId by PhyId:{phyid} failed, error_msg: {e}")

575 return RetCode.FAILED

576

577 if ret != 0:

578 log_error(f"Get MasterId by PhyId:{phyid} failed")

579 return RetCode.FAILED

580

581 return master_id.contents.value

582

583 def get_device_aicore_frequency(self, device_id):

584 """get aicore frequency"""

585 return self.get_device_frequency(device_id, AI_CORE_FREQUENCY)

586

587 def run_diagnose(self, device_obj, diagnose_devices, run_mode):

588 return interface.run_diagnose(device_obj, diagnose_devices, run_mode)