nvme-pci: align io queue count with allocted nvme_queue in nvme_probe

Since commit 147b27e4bd08 ("nvme-pci: allocate device queues storage
space at probe"), nvme_alloc_queue does not alloc the nvme queues
itself anymore.

If the write/poll_queues module parameters are changed at runtime to
values larger than the number of allocated queues in nvme_probe,
nvme_alloc_queue will access unallocated memory.

Add a new nr_allocated_queues member to struct nvme_dev to record how
many queues were alloctated in nvme_probe to avoid using more than the
allocated queues after a reset following a change to the
write/poll_queues module parameters.

Also add nr_write_queues and nr_poll_queues members to allow refreshing
the number of write and poll queues based on a change to the module
parameters when resetting the controller.

Fixes: 147b27e4bd08 ("nvme-pci: allocate device queues storage space at probe")
Signed-off-by: Weiping Zhang <zhangweiping@didiglobal.com>
Reviewed-by: Keith Busch <kbusch@kernel.org>
Reviewed-by: Max Gurtovoy <maxg@mellanox.com>
[hch: add nvme_max_io_queues, update the commit message]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Weiping Zhang and committed by

Jens Axboe 6 years ago 2a5bcfdd 54b2fcee

+33 -24

1 changed file

expand all

drivers

nvme

host

pci.c

+33 -24

drivers/nvme/host/pci.c

··· 128 128 dma_addr_t host_mem_descs_dma; 129 129 struct nvme_host_mem_buf_desc *host_mem_descs; 130 130 void **host_mem_desc_bufs; 131 + unsigned int nr_allocated_queues; 132 + unsigned int nr_write_queues; 133 + unsigned int nr_poll_queues; 131 134 }; 132 135 133 136 static int io_queue_depth_set(const char *val, const struct kernel_param *kp) ··· 211 208 struct scatterlist *sg; 212 209 }; 213 210 214 - static unsigned int max_io_queues(void) 211 + static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev) 215 212 { 216 - return num_possible_cpus() + write_queues + poll_queues; 217 - } 218 - 219 - static unsigned int max_queue_count(void) 220 - { 221 - /* IO queues + admin queue */ 222 - return 1 + max_io_queues(); 223 - } 224 - 225 - static inline unsigned int nvme_dbbuf_size(u32 stride) 226 - { 227 - return (max_queue_count() * 8 * stride); 213 + return dev->nr_allocated_queues * 8 * dev->db_stride; 228 214 } 229 215 230 216 static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev) 231 217 { 232 - unsigned int mem_size = nvme_dbbuf_size(dev->db_stride); 218 + unsigned int mem_size = nvme_dbbuf_size(dev); 233 219 234 220 if (dev->dbbuf_dbs) 235 221 return 0; ··· 243 251 244 252 static void nvme_dbbuf_dma_free(struct nvme_dev *dev) 245 253 { 246 - unsigned int mem_size = nvme_dbbuf_size(dev->db_stride); 254 + unsigned int mem_size = nvme_dbbuf_size(dev); 247 255 248 256 if (dev->dbbuf_dbs) { 249 257 dma_free_coherent(dev->dev, mem_size, ··· 1973 1981 static void nvme_calc_irq_sets(struct irq_affinity *affd, unsigned int nrirqs) 1974 1982 { 1975 1983 struct nvme_dev *dev = affd->priv; 1976 - unsigned int nr_read_queues; 1984 + unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues; 1977 1985 1978 1986 /* 1979 1987 * If there is no interupt available for queues, ensure that ··· 1989 1997 if (!nrirqs) { 1990 1998 nrirqs = 1; 1991 1999 nr_read_queues = 0; 1992 - } else if (nrirqs == 1 || !write_queues) { 2000 + } else if (nrirqs == 1 || !nr_write_queues) { 1993 2001 nr_read_queues = 0; 1994 - } else if (write_queues >= nrirqs) { 2002 + } else if (nr_write_queues >= nrirqs) { 1995 2003 nr_read_queues = 1; 1996 2004 } else { 1997 - nr_read_queues = nrirqs - write_queues; 2005 + nr_read_queues = nrirqs - nr_write_queues; 1998 2006 } 1999 2007 2000 2008 dev->io_queues[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues; ··· 2018 2026 * Poll queues don't need interrupts, but we need at least one IO 2019 2027 * queue left over for non-polled IO. 2020 2028 */ 2021 - this_p_queues = poll_queues; 2029 + this_p_queues = dev->nr_poll_queues; 2022 2030 if (this_p_queues >= nr_io_queues) { 2023 2031 this_p_queues = nr_io_queues - 1; 2024 2032 irq_queues = 1; ··· 2048 2056 __nvme_disable_io_queues(dev, nvme_admin_delete_cq); 2049 2057 } 2050 2058 2059 + static unsigned int nvme_max_io_queues(struct nvme_dev *dev) 2060 + { 2061 + return num_possible_cpus() + dev->nr_write_queues + dev->nr_poll_queues; 2062 + } 2063 + 2051 2064 static int nvme_setup_io_queues(struct nvme_dev *dev) 2052 2065 { 2053 2066 struct nvme_queue *adminq = &dev->queues[0]; 2054 2067 struct pci_dev *pdev = to_pci_dev(dev->dev); 2055 - int result, nr_io_queues; 2068 + unsigned int nr_io_queues; 2056 2069 unsigned long size; 2070 + int result; 2057 2071 2058 - nr_io_queues = max_io_queues(); 2072 + /* 2073 + * Sample the module parameters once at reset time so that we have 2074 + * stable values to work with. 2075 + */ 2076 + dev->nr_write_queues = write_queues; 2077 + dev->nr_poll_queues = poll_queues; 2059 2078 2060 2079 /* 2061 2080 * If tags are shared with admin queue (Apple bug), then ··· 2074 2071 */ 2075 2072 if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) 2076 2073 nr_io_queues = 1; 2074 + else 2075 + nr_io_queues = min(nvme_max_io_queues(dev), 2076 + dev->nr_allocated_queues - 1); 2077 2077 2078 2078 result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues); 2079 2079 if (result < 0) ··· 2751 2745 if (!dev) 2752 2746 return -ENOMEM; 2753 2747 2754 - dev->queues = kcalloc_node(max_queue_count(), sizeof(struct nvme_queue), 2755 - GFP_KERNEL, node); 2748 + dev->nr_write_queues = write_queues; 2749 + dev->nr_poll_queues = poll_queues; 2750 + dev->nr_allocated_queues = nvme_max_io_queues(dev) + 1; 2751 + dev->queues = kcalloc_node(dev->nr_allocated_queues, 2752 + sizeof(struct nvme_queue), GFP_KERNEL, node); 2756 2753 if (!dev->queues) 2757 2754 goto free; 2758 2755

Configure Feed

Configure Feed