hisi_acc_vfio_pci: register debugfs for hisilicon migration driver

On the debugfs framework of VFIO, if the CONFIG_VFIO_DEBUGFS macro is
enabled, the debug function is registered for the live migration driver
of the HiSilicon accelerator device.

After registering the HiSilicon accelerator device on the debugfs
framework of live migration of vfio, a directory file "hisi_acc"
of debugfs is created, and then three debug function files are
created in this directory:

vfio
|
+---<dev_name1>
| +---migration
| +--state
| +--hisi_acc
| +--dev_data
| +--migf_data
| +--cmd_state
|
+---<dev_name2>
+---migration
+--state
+--hisi_acc
+--dev_data
+--migf_data
+--cmd_state

dev_data file: read device data that needs to be migrated from the
current device in real time
migf_data file: read the migration data of the last live migration
from the current driver.
cmd_state: used to get the cmd channel state for the device.

+----------------+ +--------------+ +---------------+
| migration dev | | src dev | | dst dev |
+-------+--------+ +------+-------+ +-------+-------+
| | |
| +------v-------+ +-------v-------+
| | saving_migf | | resuming_migf |
read | | file | | file |
| +------+-------+ +-------+-------+
| | copy |
| +------------+----------+
| |
+-------v--------+ +-------v--------+
| data buffer | | debug_migf |
+-------+--------+ +-------+--------+
| |
cat | cat |
+-------v--------+ +-------v--------+
| dev_data | | migf_data |
+----------------+ +----------------+

When accessing debugfs, user can obtain the most recent status data
of the device through the "dev_data" file. It can read recent
complete status data of the device. If the current device is being
migrated, it will wait for it to complete.
The data for the last completed migration function will be stored
in debug_migf. Users can read it via "migf_data".

Signed-off-by: Longfang Liu <liulongfang@huawei.com>
Reviewed-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Link: https://lore.kernel.org/r/20241112073322.54550-4-liulongfang@huawei.com
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>

authored by

Longfang Liu and committed by

Alex Williamson 2 years ago b398f917 19629206

+210

2 changed files

expand all

drivers

vfio

pci

hisilicon

hisi_acc_vfio_pci.c

hisi_acc_vfio_pci.h

+191

drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c

··· 627 627 mutex_unlock(&migf->lock); 628 628 } 629 629 630 + static void 631 + hisi_acc_debug_migf_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev, 632 + struct hisi_acc_vf_migration_file *src_migf) 633 + { 634 + struct hisi_acc_vf_migration_file *dst_migf = hisi_acc_vdev->debug_migf; 635 + 636 + if (!dst_migf) 637 + return; 638 + 639 + dst_migf->total_length = src_migf->total_length; 640 + memcpy(&dst_migf->vf_data, &src_migf->vf_data, 641 + sizeof(struct acc_vf_data)); 642 + } 643 + 630 644 static void hisi_acc_vf_disable_fds(struct hisi_acc_vf_core_device *hisi_acc_vdev) 631 645 { 632 646 if (hisi_acc_vdev->resuming_migf) { 647 + hisi_acc_debug_migf_copy(hisi_acc_vdev, hisi_acc_vdev->resuming_migf); 633 648 hisi_acc_vf_disable_fd(hisi_acc_vdev->resuming_migf); 634 649 fput(hisi_acc_vdev->resuming_migf->filp); 635 650 hisi_acc_vdev->resuming_migf = NULL; 636 651 } 637 652 638 653 if (hisi_acc_vdev->saving_migf) { 654 + hisi_acc_debug_migf_copy(hisi_acc_vdev, hisi_acc_vdev->saving_migf); 639 655 hisi_acc_vf_disable_fd(hisi_acc_vdev->saving_migf); 640 656 fput(hisi_acc_vdev->saving_migf->filp); 641 657 hisi_acc_vdev->saving_migf = NULL; ··· 1308 1292 return vfio_pci_core_ioctl(core_vdev, cmd, arg); 1309 1293 } 1310 1294 1295 + static int hisi_acc_vf_debug_check(struct seq_file *seq, struct vfio_device *vdev) 1296 + { 1297 + struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev); 1298 + struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; 1299 + int ret; 1300 + 1301 + lockdep_assert_held(&hisi_acc_vdev->open_mutex); 1302 + /* 1303 + * When the device is not opened, the io_base is not mapped. 1304 + * The driver cannot perform device read and write operations. 1305 + */ 1306 + if (!hisi_acc_vdev->dev_opened) { 1307 + seq_puts(seq, "device not opened!\n"); 1308 + return -EINVAL; 1309 + } 1310 + 1311 + ret = qm_wait_dev_not_ready(vf_qm); 1312 + if (ret) { 1313 + seq_puts(seq, "VF device not ready!\n"); 1314 + return -EBUSY; 1315 + } 1316 + 1317 + return 0; 1318 + } 1319 + 1320 + static int hisi_acc_vf_debug_cmd(struct seq_file *seq, void *data) 1321 + { 1322 + struct device *vf_dev = seq->private; 1323 + struct vfio_pci_core_device *core_device = dev_get_drvdata(vf_dev); 1324 + struct vfio_device *vdev = &core_device->vdev; 1325 + struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev); 1326 + struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; 1327 + u64 value; 1328 + int ret; 1329 + 1330 + mutex_lock(&hisi_acc_vdev->open_mutex); 1331 + ret = hisi_acc_vf_debug_check(seq, vdev); 1332 + if (ret) { 1333 + mutex_unlock(&hisi_acc_vdev->open_mutex); 1334 + return ret; 1335 + } 1336 + 1337 + value = readl(vf_qm->io_base + QM_MB_CMD_SEND_BASE); 1338 + if (value == QM_MB_CMD_NOT_READY) { 1339 + mutex_unlock(&hisi_acc_vdev->open_mutex); 1340 + seq_puts(seq, "mailbox cmd channel not ready!\n"); 1341 + return -EINVAL; 1342 + } 1343 + mutex_unlock(&hisi_acc_vdev->open_mutex); 1344 + seq_puts(seq, "mailbox cmd channel ready!\n"); 1345 + 1346 + return 0; 1347 + } 1348 + 1349 + static int hisi_acc_vf_dev_read(struct seq_file *seq, void *data) 1350 + { 1351 + struct device *vf_dev = seq->private; 1352 + struct vfio_pci_core_device *core_device = dev_get_drvdata(vf_dev); 1353 + struct vfio_device *vdev = &core_device->vdev; 1354 + struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev); 1355 + size_t vf_data_sz = offsetofend(struct acc_vf_data, padding); 1356 + struct acc_vf_data *vf_data; 1357 + int ret; 1358 + 1359 + mutex_lock(&hisi_acc_vdev->open_mutex); 1360 + ret = hisi_acc_vf_debug_check(seq, vdev); 1361 + if (ret) { 1362 + mutex_unlock(&hisi_acc_vdev->open_mutex); 1363 + return ret; 1364 + } 1365 + 1366 + mutex_lock(&hisi_acc_vdev->state_mutex); 1367 + vf_data = kzalloc(sizeof(*vf_data), GFP_KERNEL); 1368 + if (!vf_data) { 1369 + ret = -ENOMEM; 1370 + goto mutex_release; 1371 + } 1372 + 1373 + vf_data->vf_qm_state = hisi_acc_vdev->vf_qm_state; 1374 + ret = vf_qm_read_data(&hisi_acc_vdev->vf_qm, vf_data); 1375 + if (ret) 1376 + goto migf_err; 1377 + 1378 + seq_hex_dump(seq, "Dev Data:", DUMP_PREFIX_OFFSET, 16, 1, 1379 + (const void *)vf_data, vf_data_sz, false); 1380 + 1381 + seq_printf(seq, 1382 + "guest driver load: %u\n" 1383 + "data size: %lu\n", 1384 + hisi_acc_vdev->vf_qm_state, 1385 + sizeof(struct acc_vf_data)); 1386 + 1387 + migf_err: 1388 + kfree(vf_data); 1389 + mutex_release: 1390 + mutex_unlock(&hisi_acc_vdev->state_mutex); 1391 + mutex_unlock(&hisi_acc_vdev->open_mutex); 1392 + 1393 + return ret; 1394 + } 1395 + 1396 + static int hisi_acc_vf_migf_read(struct seq_file *seq, void *data) 1397 + { 1398 + struct device *vf_dev = seq->private; 1399 + struct vfio_pci_core_device *core_device = dev_get_drvdata(vf_dev); 1400 + struct vfio_device *vdev = &core_device->vdev; 1401 + struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev); 1402 + size_t vf_data_sz = offsetofend(struct acc_vf_data, padding); 1403 + struct hisi_acc_vf_migration_file *debug_migf = hisi_acc_vdev->debug_migf; 1404 + 1405 + /* Check whether the live migration operation has been performed */ 1406 + if (debug_migf->total_length < QM_MATCH_SIZE) { 1407 + seq_puts(seq, "device not migrated!\n"); 1408 + return -EAGAIN; 1409 + } 1410 + 1411 + seq_hex_dump(seq, "Mig Data:", DUMP_PREFIX_OFFSET, 16, 1, 1412 + (const void *)&debug_migf->vf_data, vf_data_sz, false); 1413 + seq_printf(seq, "migrate data length: %lu\n", debug_migf->total_length); 1414 + 1415 + return 0; 1416 + } 1417 + 1311 1418 static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev) 1312 1419 { 1313 1420 struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(core_vdev); ··· 1442 1303 return ret; 1443 1304 1444 1305 if (core_vdev->mig_ops) { 1306 + mutex_lock(&hisi_acc_vdev->open_mutex); 1445 1307 ret = hisi_acc_vf_qm_init(hisi_acc_vdev); 1446 1308 if (ret) { 1309 + mutex_unlock(&hisi_acc_vdev->open_mutex); 1447 1310 vfio_pci_core_disable(vdev); 1448 1311 return ret; 1449 1312 } 1450 1313 hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; 1314 + hisi_acc_vdev->dev_opened = true; 1315 + mutex_unlock(&hisi_acc_vdev->open_mutex); 1451 1316 } 1452 1317 1453 1318 vfio_pci_core_finish_enable(vdev); ··· 1463 1320 struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(core_vdev); 1464 1321 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; 1465 1322 1323 + mutex_lock(&hisi_acc_vdev->open_mutex); 1324 + hisi_acc_vdev->dev_opened = false; 1466 1325 iounmap(vf_qm->io_base); 1326 + mutex_unlock(&hisi_acc_vdev->open_mutex); 1467 1327 vfio_pci_core_close_device(core_vdev); 1468 1328 } 1469 1329 ··· 1486 1340 hisi_acc_vdev->pf_qm = pf_qm; 1487 1341 hisi_acc_vdev->vf_dev = pdev; 1488 1342 mutex_init(&hisi_acc_vdev->state_mutex); 1343 + mutex_init(&hisi_acc_vdev->open_mutex); 1489 1344 1490 1345 core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY; 1491 1346 core_vdev->mig_ops = &hisi_acc_vfio_pci_migrn_state_ops; ··· 1532 1385 .detach_ioas = vfio_iommufd_physical_detach_ioas, 1533 1386 }; 1534 1387 1388 + static void hisi_acc_vfio_debug_init(struct hisi_acc_vf_core_device *hisi_acc_vdev) 1389 + { 1390 + struct vfio_device *vdev = &hisi_acc_vdev->core_device.vdev; 1391 + struct hisi_acc_vf_migration_file *migf; 1392 + struct dentry *vfio_dev_migration; 1393 + struct dentry *vfio_hisi_acc; 1394 + struct device *dev = vdev->dev; 1395 + 1396 + if (!debugfs_initialized() || 1397 + !IS_ENABLED(CONFIG_VFIO_DEBUGFS)) 1398 + return; 1399 + 1400 + if (vdev->ops != &hisi_acc_vfio_pci_migrn_ops) 1401 + return; 1402 + 1403 + vfio_dev_migration = debugfs_lookup("migration", vdev->debug_root); 1404 + if (!vfio_dev_migration) { 1405 + dev_err(dev, "failed to lookup migration debugfs file!\n"); 1406 + return; 1407 + } 1408 + 1409 + migf = kzalloc(sizeof(*migf), GFP_KERNEL); 1410 + if (!migf) 1411 + return; 1412 + hisi_acc_vdev->debug_migf = migf; 1413 + 1414 + vfio_hisi_acc = debugfs_create_dir("hisi_acc", vfio_dev_migration); 1415 + debugfs_create_devm_seqfile(dev, "dev_data", vfio_hisi_acc, 1416 + hisi_acc_vf_dev_read); 1417 + debugfs_create_devm_seqfile(dev, "migf_data", vfio_hisi_acc, 1418 + hisi_acc_vf_migf_read); 1419 + debugfs_create_devm_seqfile(dev, "cmd_state", vfio_hisi_acc, 1420 + hisi_acc_vf_debug_cmd); 1421 + } 1422 + 1423 + static void hisi_acc_vf_debugfs_exit(struct hisi_acc_vf_core_device *hisi_acc_vdev) 1424 + { 1425 + kfree(hisi_acc_vdev->debug_migf); 1426 + hisi_acc_vdev->debug_migf = NULL; 1427 + } 1428 + 1535 1429 static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1536 1430 { 1537 1431 struct hisi_acc_vf_core_device *hisi_acc_vdev; ··· 1599 1411 ret = vfio_pci_core_register_device(&hisi_acc_vdev->core_device); 1600 1412 if (ret) 1601 1413 goto out_put_vdev; 1414 + 1415 + hisi_acc_vfio_debug_init(hisi_acc_vdev); 1602 1416 return 0; 1603 1417 1604 1418 out_put_vdev: ··· 1613 1423 struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); 1614 1424 1615 1425 vfio_pci_core_unregister_device(&hisi_acc_vdev->core_device); 1426 + hisi_acc_vf_debugfs_exit(hisi_acc_vdev); 1616 1427 vfio_put_device(&hisi_acc_vdev->core_device.vdev); 1617 1428 } 1618 1429

+19

drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h

··· 32 32 #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) 33 33 #define QM_SQC_VFT_NUM_SHIFT_V2 45 34 34 #define QM_SQC_VFT_NUM_MASK_V2 GENMASK(9, 0) 35 + #define QM_MB_CMD_NOT_READY 0xffffffff 35 36 36 37 /* RW regs */ 37 38 #define QM_REGS_MAX_LEN 7 ··· 100 99 struct hisi_acc_vf_core_device { 101 100 struct vfio_pci_core_device core_device; 102 101 u8 match_done; 102 + /* 103 + * io_base is only valid when dev_opened is true, 104 + * which is protected by open_mutex. 105 + */ 106 + bool dev_opened; 107 + /* Ensure the accuracy of dev_opened operation */ 108 + struct mutex open_mutex; 103 109 104 110 /* For migration state */ 105 111 struct mutex state_mutex; ··· 115 107 struct pci_dev *vf_dev; 116 108 struct hisi_qm *pf_qm; 117 109 struct hisi_qm vf_qm; 110 + /* 111 + * vf_qm_state represents the QM_VF_STATE register value. 112 + * It is set by Guest driver for the ACC VF dev indicating 113 + * the driver has loaded and configured the dev correctly. 114 + */ 118 115 u32 vf_qm_state; 119 116 int vf_id; 120 117 struct hisi_acc_vf_migration_file *resuming_migf; 121 118 struct hisi_acc_vf_migration_file *saving_migf; 119 + 120 + /* 121 + * It holds migration data corresponding to the last migration 122 + * and is used by the debugfs interface to report it. 123 + */ 124 + struct hisi_acc_vf_migration_file *debug_migf; 122 125 }; 123 126 #endif /* HISI_ACC_VFIO_PCI_H */

Configure Feed

Configure Feed