tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

RDMA/mlx5: Create ODP EQ only when ODP MR is created

There is no need to create the ODP EQ if the user doesn't use ODP MRs.
Hence, create it only when the first ODP MR is created. This EQ will be
destroyed only when the device is unloaded.
This will decrease the number of EQs created per device. for example: If
we creates 1K devices (SF/VF/etc'), than we will decrease the num of EQs
by 1K.

Link: https://lore.kernel.org/r/20210314125418.179716-1-leon@kernel.org
Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Maor Gottlieb <maorg@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

authored by

Shay Drory and committed by
Jason Gunthorpe ad50294d 783cf673

+29 -10
+7
drivers/infiniband/hw/mlx5/mlx5_ib.h
··· 1080 1080 struct mutex slow_path_mutex; 1081 1081 struct ib_odp_caps odp_caps; 1082 1082 u64 odp_max_size; 1083 + struct mutex odp_eq_mutex; 1083 1084 struct mlx5_ib_pf_eq odp_pf_eq; 1084 1085 1085 1086 struct xarray odp_mkeys; ··· 1359 1358 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1360 1359 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev); 1361 1360 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev); 1361 + int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq); 1362 1362 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev); 1363 1363 int __init mlx5_ib_odp_init(void); 1364 1364 void mlx5_ib_odp_cleanup(void); ··· 1379 1377 } 1380 1378 1381 1379 static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; } 1380 + static inline int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, 1381 + struct mlx5_ib_pf_eq *eq) 1382 + { 1383 + return 0; 1384 + } 1382 1385 static inline void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev) {} 1383 1386 static inline int mlx5_ib_odp_init(void) { return 0; } 1384 1387 static inline void mlx5_ib_odp_cleanup(void) {}
+3
drivers/infiniband/hw/mlx5/mr.c
··· 1500 1500 if (!IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) 1501 1501 return ERR_PTR(-EOPNOTSUPP); 1502 1502 1503 + err = mlx5r_odp_create_eq(dev, &dev->odp_pf_eq); 1504 + if (err) 1505 + return ERR_PTR(err); 1503 1506 if (!start && length == U64_MAX) { 1504 1507 if (iova != 0) 1505 1508 return ERR_PTR(-EINVAL);
+19 -10
drivers/infiniband/hw/mlx5/odp.c
··· 1531 1531 MLX5_IB_NUM_PF_DRAIN = 64, 1532 1532 }; 1533 1533 1534 - static int 1535 - mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1534 + int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1536 1535 { 1537 1536 struct mlx5_eq_param param = {}; 1538 - int err; 1537 + int err = 0; 1539 1538 1539 + mutex_lock(&dev->odp_eq_mutex); 1540 + if (eq->core) 1541 + goto unlock; 1540 1542 INIT_WORK(&eq->work, mlx5_ib_eq_pf_action); 1541 1543 spin_lock_init(&eq->lock); 1542 1544 eq->dev = dev; 1543 1545 1544 1546 eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN, 1545 1547 sizeof(struct mlx5_pagefault)); 1546 - if (!eq->pool) 1547 - return -ENOMEM; 1548 + if (!eq->pool) { 1549 + err = -ENOMEM; 1550 + goto unlock; 1551 + } 1548 1552 1549 1553 eq->wq = alloc_workqueue("mlx5_ib_page_fault", 1550 1554 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, ··· 1559 1555 } 1560 1556 1561 1557 eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int; 1562 - param = (struct mlx5_eq_param) { 1558 + param = (struct mlx5_eq_param){ 1563 1559 .irq_index = 0, 1564 1560 .nent = MLX5_IB_NUM_PF_EQE, 1565 1561 }; ··· 1575 1571 goto err_eq; 1576 1572 } 1577 1573 1574 + mutex_unlock(&dev->odp_eq_mutex); 1578 1575 return 0; 1579 1576 err_eq: 1580 1577 mlx5_eq_destroy_generic(dev->mdev, eq->core); 1581 1578 err_wq: 1579 + eq->core = NULL; 1582 1580 destroy_workqueue(eq->wq); 1583 1581 err_mempool: 1584 1582 mempool_destroy(eq->pool); 1583 + unlock: 1584 + mutex_unlock(&dev->odp_eq_mutex); 1585 1585 return err; 1586 1586 } 1587 1587 1588 1588 static int 1589 - mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1589 + mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1590 1590 { 1591 1591 int err; 1592 1592 1593 + if (!eq->core) 1594 + return 0; 1593 1595 mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb); 1594 1596 err = mlx5_eq_destroy_generic(dev->mdev, eq->core); 1595 1597 cancel_work_sync(&eq->work); ··· 1652 1642 } 1653 1643 } 1654 1644 1655 - ret = mlx5_ib_create_pf_eq(dev, &dev->odp_pf_eq); 1656 - 1645 + mutex_init(&dev->odp_eq_mutex); 1657 1646 return ret; 1658 1647 } 1659 1648 ··· 1661 1652 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1662 1653 return; 1663 1654 1664 - mlx5_ib_destroy_pf_eq(dev, &dev->odp_pf_eq); 1655 + mlx5_ib_odp_destroy_eq(dev, &dev->odp_pf_eq); 1665 1656 } 1666 1657 1667 1658 int mlx5_ib_odp_init(void)