-18

Documentation/ABI/testing/debugfs-pktcdvd

reviewed

··· 1 1 - What: /sys/kernel/debug/pktcdvd/pktcdvd[0-7] 2 2 - Date: Oct. 2006 3 3 - KernelVersion: 2.6.20 4 4 - Contact: Thomas Maier <balagi@justmail.de> 5 5 - Description: 6 6 - 7 7 - The pktcdvd module (packet writing driver) creates 8 8 - these files in debugfs: 9 9 - 10 10 - /sys/kernel/debug/pktcdvd/pktcdvd[0-7]/ 11 11 - 12 12 - ==== ====== ==================================== 13 13 - info 0444 Lots of driver statistics and infos. 14 14 - ==== ====== ==================================== 15 15 - 16 16 - Example:: 17 17 - 18 18 - cat /sys/kernel/debug/pktcdvd/pktcdvd0/info

-97

Documentation/ABI/testing/sysfs-class-pktcdvd

reviewed

··· 1 1 - sysfs interface 2 2 - --------------- 3 3 - The pktcdvd module (packet writing driver) creates the following files in the 4 4 - sysfs: (<devid> is in the format major:minor) 5 5 - 6 6 - What: /sys/class/pktcdvd/add 7 7 - What: /sys/class/pktcdvd/remove 8 8 - What: /sys/class/pktcdvd/device_map 9 9 - Date: Oct. 2006 10 10 - KernelVersion: 2.6.20 11 11 - Contact: Thomas Maier <balagi@justmail.de> 12 12 - Description: 13 13 - 14 14 - ========== ============================================== 15 15 - add (WO) Write a block device id (major:minor) to 16 16 - create a new pktcdvd device and map it to the 17 17 - block device. 18 18 - 19 19 - remove (WO) Write the pktcdvd device id (major:minor) 20 20 - to remove the pktcdvd device. 21 21 - 22 22 - device_map (RO) Shows the device mapping in format: 23 23 - pktcdvd[0-7] <pktdevid> <blkdevid> 24 24 - ========== ============================================== 25 25 - 26 26 - 27 27 - What: /sys/class/pktcdvd/pktcdvd[0-7]/dev 28 28 - What: /sys/class/pktcdvd/pktcdvd[0-7]/uevent 29 29 - Date: Oct. 2006 30 30 - KernelVersion: 2.6.20 31 31 - Contact: Thomas Maier <balagi@justmail.de> 32 32 - Description: 33 33 - dev: (RO) Device id 34 34 - 35 35 - uevent: (WO) To send a uevent 36 36 - 37 37 - 38 38 - What: /sys/class/pktcdvd/pktcdvd[0-7]/stat/packets_started 39 39 - What: /sys/class/pktcdvd/pktcdvd[0-7]/stat/packets_finished 40 40 - What: /sys/class/pktcdvd/pktcdvd[0-7]/stat/kb_written 41 41 - What: /sys/class/pktcdvd/pktcdvd[0-7]/stat/kb_read 42 42 - What: /sys/class/pktcdvd/pktcdvd[0-7]/stat/kb_read_gather 43 43 - What: /sys/class/pktcdvd/pktcdvd[0-7]/stat/reset 44 44 - Date: Oct. 2006 45 45 - KernelVersion: 2.6.20 46 46 - Contact: Thomas Maier <balagi@justmail.de> 47 47 - Description: 48 48 - packets_started: (RO) Number of started packets. 49 49 - 50 50 - packets_finished: (RO) Number of finished packets. 51 51 - 52 52 - kb_written: (RO) kBytes written. 53 53 - 54 54 - kb_read: (RO) kBytes read. 55 55 - 56 56 - kb_read_gather: (RO) kBytes read to fill write packets. 57 57 - 58 58 - reset: (WO) Write any value to it to reset 59 59 - pktcdvd device statistic values, like 60 60 - bytes read/written. 61 61 - 62 62 - 63 63 - What: /sys/class/pktcdvd/pktcdvd[0-7]/write_queue/size 64 64 - What: /sys/class/pktcdvd/pktcdvd[0-7]/write_queue/congestion_off 65 65 - What: /sys/class/pktcdvd/pktcdvd[0-7]/write_queue/congestion_on 66 66 - Date: Oct. 2006 67 67 - KernelVersion: 2.6.20 68 68 - Contact: Thomas Maier <balagi@justmail.de> 69 69 - Description: 70 70 - ============== ================================================ 71 71 - size (RO) Contains the size of the bio write queue. 72 72 - 73 73 - congestion_off (RW) If bio write queue size is below this mark, 74 74 - accept new bio requests from the block layer. 75 75 - 76 76 - congestion_on (RW) If bio write queue size is higher as this 77 77 - mark, do no longer accept bio write requests 78 78 - from the block layer and wait till the pktcdvd 79 79 - device has processed enough bio's so that bio 80 80 - write queue size is below congestion off mark. 81 81 - A value of <= 0 disables congestion control. 82 82 - ============== ================================================ 83 83 - 84 84 - 85 85 - Example: 86 86 - -------- 87 87 - To use the pktcdvd sysfs interface directly, you can do:: 88 88 - 89 89 - # create a new pktcdvd device mapped to /dev/hdc 90 90 - echo "22:0" >/sys/class/pktcdvd/add 91 91 - cat /sys/class/pktcdvd/device_map 92 92 - # assuming device pktcdvd0 was created, look at stat's 93 93 - cat /sys/class/pktcdvd/pktcdvd0/stat/kb_written 94 94 - # print the device id of the mapped block device 95 95 - fgrep pktcdvd0 /sys/class/pktcdvd/device_map 96 96 - # remove device, using pktcdvd0 device id 253:0 97 97 - echo "253:0" >/sys/class/pktcdvd/remove

-1

Documentation/cdrom/cdrom-standard.rst

reviewed

··· 273 273 __u8 media_written; /* dirty flag, DVD+RW bookkeeping */ 274 274 unsigned short mmc3_profile; /* current MMC3 profile */ 275 275 int for_data; /* unknown:TBD */ 276 276 - int (*exit)(struct cdrom_device_info *);/* unknown:TBD */ 277 276 int mrw_mode_page; /* which MRW mode page is in use */ 278 277 }; 279 278

-1

Documentation/cdrom/index.rst

reviewed

··· 8 8 :maxdepth: 1 9 9 10 10 cdrom-standard 11 11 - packet-writing 12 11 13 12 .. only:: subproject and html 14 13

-139

Documentation/cdrom/packet-writing.rst

reviewed

··· 1 1 - ============== 2 2 - Packet writing 3 3 - ============== 4 4 - 5 5 - Getting started quick 6 6 - --------------------- 7 7 - 8 8 - - Select packet support in the block device section and UDF support in 9 9 - the file system section. 10 10 - 11 11 - - Compile and install kernel and modules, reboot. 12 12 - 13 13 - - You need the udftools package (pktsetup, mkudffs, cdrwtool). 14 14 - Download from https://github.com/pali/udftools 15 15 - 16 16 - - Grab a new CD-RW disc and format it (assuming CD-RW is hdc, substitute 17 17 - as appropriate):: 18 18 - 19 19 - # cdrwtool -d /dev/hdc -q 20 20 - 21 21 - - Setup your writer:: 22 22 - 23 23 - # pktsetup dev_name /dev/hdc 24 24 - 25 25 - - Now you can mount /dev/pktcdvd/dev_name and copy files to it. Enjoy:: 26 26 - 27 27 - # mount /dev/pktcdvd/dev_name /cdrom -t udf -o rw,noatime 28 28 - 29 29 - 30 30 - Packet writing for DVD-RW media 31 31 - ------------------------------- 32 32 - 33 33 - DVD-RW discs can be written to much like CD-RW discs if they are in 34 34 - the so called "restricted overwrite" mode. To put a disc in restricted 35 35 - overwrite mode, run:: 36 36 - 37 37 - # dvd+rw-format /dev/hdc 38 38 - 39 39 - You can then use the disc the same way you would use a CD-RW disc:: 40 40 - 41 41 - # pktsetup dev_name /dev/hdc 42 42 - # mount /dev/pktcdvd/dev_name /cdrom -t udf -o rw,noatime 43 43 - 44 44 - 45 45 - Packet writing for DVD+RW media 46 46 - ------------------------------- 47 47 - 48 48 - According to the DVD+RW specification, a drive supporting DVD+RW discs 49 49 - shall implement "true random writes with 2KB granularity", which means 50 50 - that it should be possible to put any filesystem with a block size >= 51 51 - 2KB on such a disc. For example, it should be possible to do:: 52 52 - 53 53 - # dvd+rw-format /dev/hdc (only needed if the disc has never 54 54 - been formatted) 55 55 - # mkudffs /dev/hdc 56 56 - # mount /dev/hdc /cdrom -t udf -o rw,noatime 57 57 - 58 58 - However, some drives don't follow the specification and expect the 59 59 - host to perform aligned writes at 32KB boundaries. Other drives do 60 60 - follow the specification, but suffer bad performance problems if the 61 61 - writes are not 32KB aligned. 62 62 - 63 63 - Both problems can be solved by using the pktcdvd driver, which always 64 64 - generates aligned writes:: 65 65 - 66 66 - # dvd+rw-format /dev/hdc 67 67 - # pktsetup dev_name /dev/hdc 68 68 - # mkudffs /dev/pktcdvd/dev_name 69 69 - # mount /dev/pktcdvd/dev_name /cdrom -t udf -o rw,noatime 70 70 - 71 71 - 72 72 - Packet writing for DVD-RAM media 73 73 - -------------------------------- 74 74 - 75 75 - DVD-RAM discs are random writable, so using the pktcdvd driver is not 76 76 - necessary. However, using the pktcdvd driver can improve performance 77 77 - in the same way it does for DVD+RW media. 78 78 - 79 79 - 80 80 - Notes 81 81 - ----- 82 82 - 83 83 - - CD-RW media can usually not be overwritten more than about 1000 84 84 - times, so to avoid unnecessary wear on the media, you should always 85 85 - use the noatime mount option. 86 86 - 87 87 - - Defect management (ie automatic remapping of bad sectors) has not 88 88 - been implemented yet, so you are likely to get at least some 89 89 - filesystem corruption if the disc wears out. 90 90 - 91 91 - - Since the pktcdvd driver makes the disc appear as a regular block 92 92 - device with a 2KB block size, you can put any filesystem you like on 93 93 - the disc. For example, run:: 94 94 - 95 95 - # /sbin/mke2fs /dev/pktcdvd/dev_name 96 96 - 97 97 - to create an ext2 filesystem on the disc. 98 98 - 99 99 - 100 100 - Using the pktcdvd sysfs interface 101 101 - --------------------------------- 102 102 - 103 103 - Since Linux 2.6.20, the pktcdvd module has a sysfs interface 104 104 - and can be controlled by it. For example the "pktcdvd" tool uses 105 105 - this interface. (see http://tom.ist-im-web.de/linux/software/pktcdvd ) 106 106 - 107 107 - "pktcdvd" works similar to "pktsetup", e.g.:: 108 108 - 109 109 - # pktcdvd -a dev_name /dev/hdc 110 110 - # mkudffs /dev/pktcdvd/dev_name 111 111 - # mount -t udf -o rw,noatime /dev/pktcdvd/dev_name /dvdram 112 112 - # cp files /dvdram 113 113 - # umount /dvdram 114 114 - # pktcdvd -r dev_name 115 115 - 116 116 - 117 117 - For a description of the sysfs interface look into the file: 118 118 - 119 119 - Documentation/ABI/testing/sysfs-class-pktcdvd 120 120 - 121 121 - 122 122 - Using the pktcdvd debugfs interface 123 123 - ----------------------------------- 124 124 - 125 125 - To read pktcdvd device infos in human readable form, do:: 126 126 - 127 127 - # cat /sys/kernel/debug/pktcdvd/pktcdvd[0-7]/info 128 128 - 129 129 - For a description of the debugfs interface look into the file: 130 130 - 131 131 - Documentation/ABI/testing/debugfs-pktcdvd 132 132 - 133 133 - 134 134 - 135 135 - Links 136 136 - ----- 137 137 - 138 138 - See http://fy.chalmers.se/~appro/linux/DVD+RW/ for more information 139 139 - about DVD writing.

+11 -11

Documentation/nvme/nvme-pci-endpoint-target.rst

reviewed

··· 6 6 7 7 :Author: Damien Le Moal <dlemoal@kernel.org> 8 8 9 9 - The NVMe PCI endpoint function target driver implements a NVMe PCIe controller 10 10 - using a NVMe fabrics target controller configured with the PCI transport type. 9 9 + The NVMe PCI endpoint function target driver implements an NVMe PCIe controller 10 10 + using an NVMe fabrics target controller configured with the PCI transport type. 11 11 12 12 Overview 13 13 ======== 14 14 15 15 - The NVMe PCI endpoint function target driver allows exposing a NVMe target 15 15 + The NVMe PCI endpoint function target driver allows exposing an NVMe target 16 16 controller over a PCIe link, thus implementing an NVMe PCIe device similar to a 17 17 regular M.2 SSD. The target controller is created in the same manner as when 18 18 using NVMe over fabrics: the controller represents the interface to an NVMe 19 19 subsystem using a port. The port transfer type must be configured to be 20 20 "pci". The subsystem can be configured to have namespaces backed by regular 21 21 files or block devices, or can use NVMe passthrough to expose to the PCI host an 22 22 - existing physical NVMe device or a NVMe fabrics host controller (e.g. a NVMe TCP 23 23 - host controller). 22 22 + existing physical NVMe device or an NVMe fabrics host controller (e.g. a NVMe 23 23 + TCP host controller). 24 24 25 25 The NVMe PCI endpoint function target driver relies as much as possible on the 26 26 NVMe target core code to parse and execute NVMe commands submitted by the PCIe ··· 181 181 subsystem and port must be defined. Second, the NVMe PCI endpoint device must 182 182 be setup and bound to the subsystem and port created. 183 183 184 184 - Creating a NVMe Subsystem and Port 185 185 - ---------------------------------- 184 184 + Creating an NVMe Subsystem and Port 185 185 + ----------------------------------- 186 186 187 187 - Details about how to configure a NVMe target subsystem and port are outside the 187 187 + Details about how to configure an NVMe target subsystem and port are outside the 188 188 scope of this document. The following only provides a simple example of a port 189 189 and subsystem with a single namespace backed by a null_blk device. 190 190 ··· 234 234 # ln -s /sys/kernel/config/nvmet/subsystems/nvmepf.0.nqn \ 235 235 /sys/kernel/config/nvmet/ports/1/subsystems/nvmepf.0.nqn 236 236 237 237 - Creating a NVMe PCI Endpoint Device 238 238 - ----------------------------------- 237 237 + Creating an NVMe PCI Endpoint Device 238 238 + ------------------------------------ 239 239 240 240 With the NVMe target subsystem and port ready for use, the NVMe PCI endpoint 241 241 device can now be created and enabled. The NVMe PCI endpoint target driver ··· 303 303 304 304 nvmet_pci_epf nvmet_pci_epf.0: Enabling controller 305 305 306 306 - On the host side, the NVMe PCI endpoint function target device will is 306 306 + On the host side, the NVMe PCI endpoint function target device is 307 307 discoverable as a PCI device, with the vendor ID and device ID as configured:: 308 308 309 309 # lspci -n

-1

Documentation/userspace-api/ioctl/ioctl-number.rst

reviewed

··· 220 220 include/linux/falloc.h, 221 221 linux/fs.h, 222 222 'X' all fs/ocfs2/ocfs_fs.h conflict! 223 223 - 'X' 01 linux/pktcdvd.h conflict! 224 223 'Z' 14-15 drivers/message/fusion/mptctl.h 225 224 '[' 00-3F linux/usb/tmc.h USB Test and Measurement Devices 226 225 <mailto:gregkh@linuxfoundation.org>

-7

MAINTAINERS

reviewed

··· 19706 19706 F: Documentation/devicetree/bindings/input/pine64,pinephone-keyboard.yaml 19707 19707 F: drivers/input/keyboard/pinephone-keyboard.c 19708 19708 19709 19709 - PKTCDVD DRIVER 19710 19710 - M: linux-block@vger.kernel.org 19711 19711 - S: Orphan 19712 19712 - F: drivers/block/pktcdvd.c 19713 19713 - F: include/linux/pktcdvd.h 19714 19714 - F: include/uapi/linux/pktcdvd.h 19715 19715 - 19716 19709 PLANTOWER PMS7003 AIR POLLUTION SENSOR DRIVER 19717 19710 M: Tomasz Duszynski <tduszyns@gmail.com> 19718 19711 S: Maintained

+3

block/bio-integrity.c

reviewed

··· 128 128 if (bip->bip_vcnt > 0) { 129 129 struct bio_vec *bv = &bip->bip_vec[bip->bip_vcnt - 1]; 130 130 131 131 + if (!zone_device_pages_have_same_pgmap(bv->bv_page, page)) 132 132 + return 0; 133 133 + 131 134 if (bvec_try_merge_hw_page(q, bv, page, len, offset)) { 132 135 bip->bip_iter.bi_size += len; 133 136 return len;

+13 -7

block/bio.c

reviewed

··· 930 930 return false; 931 931 if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) 932 932 return false; 933 933 - if (!zone_device_pages_have_same_pgmap(bv->bv_page, page)) 934 934 - return false; 935 933 936 934 if ((vec_end_addr & PAGE_MASK) != ((page_addr + off) & PAGE_MASK)) { 937 935 if (IS_ENABLED(CONFIG_KMSAN)) ··· 980 982 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); 981 983 WARN_ON_ONCE(bio_full(bio, len)); 982 984 985 985 + if (is_pci_p2pdma_page(page)) 986 986 + bio->bi_opf |= REQ_P2PDMA | REQ_NOMERGE; 987 987 + 983 988 bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt], page, len, off); 984 989 bio->bi_iter.bi_size += len; 985 990 bio->bi_vcnt++; ··· 1023 1022 if (bio->bi_iter.bi_size > UINT_MAX - len) 1024 1023 return 0; 1025 1024 1026 1026 - if (bio->bi_vcnt > 0 && 1027 1027 - bvec_try_merge_page(&bio->bi_io_vec[bio->bi_vcnt - 1], 1028 1028 - page, len, offset)) { 1029 1029 - bio->bi_iter.bi_size += len; 1030 1030 - return len; 1025 1025 + if (bio->bi_vcnt > 0) { 1026 1026 + struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; 1027 1027 + 1028 1028 + if (!zone_device_pages_have_same_pgmap(bv->bv_page, page)) 1029 1029 + return 0; 1030 1030 + 1031 1031 + if (bvec_try_merge_page(bv, page, len, offset)) { 1032 1032 + bio->bi_iter.bi_size += len; 1033 1033 + return len; 1034 1034 + } 1031 1035 } 1032 1036 1033 1037 if (bio->bi_vcnt >= bio->bi_max_vecs)

+43 -3

block/blk-mq-cpumap.c

reviewed

··· 12 12 #include <linux/cpu.h> 13 13 #include <linux/group_cpus.h> 14 14 #include <linux/device/bus.h> 15 15 + #include <linux/sched/isolation.h> 15 16 16 17 #include "blk.h" 17 18 #include "blk-mq.h" 18 19 20 20 + static unsigned int blk_mq_num_queues(const struct cpumask *mask, 21 21 + unsigned int max_queues) 22 22 + { 23 23 + unsigned int num; 24 24 + 25 25 + num = cpumask_weight(mask); 26 26 + return min_not_zero(num, max_queues); 27 27 + } 28 28 + 29 29 + /** 30 30 + * blk_mq_num_possible_queues - Calc nr of queues for multiqueue devices 31 31 + * @max_queues: The maximum number of queues the hardware/driver 32 32 + * supports. If max_queues is 0, the argument is 33 33 + * ignored. 34 34 + * 35 35 + * Calculates the number of queues to be used for a multiqueue 36 36 + * device based on the number of possible CPUs. 37 37 + */ 38 38 + unsigned int blk_mq_num_possible_queues(unsigned int max_queues) 39 39 + { 40 40 + return blk_mq_num_queues(cpu_possible_mask, max_queues); 41 41 + } 42 42 + EXPORT_SYMBOL_GPL(blk_mq_num_possible_queues); 43 43 + 44 44 + /** 45 45 + * blk_mq_num_online_queues - Calc nr of queues for multiqueue devices 46 46 + * @max_queues: The maximum number of queues the hardware/driver 47 47 + * supports. If max_queues is 0, the argument is 48 48 + * ignored. 49 49 + * 50 50 + * Calculates the number of queues to be used for a multiqueue 51 51 + * device based on the number of online CPUs. 52 52 + */ 53 53 + unsigned int blk_mq_num_online_queues(unsigned int max_queues) 54 54 + { 55 55 + return blk_mq_num_queues(cpu_online_mask, max_queues); 56 56 + } 57 57 + EXPORT_SYMBOL_GPL(blk_mq_num_online_queues); 58 58 + 19 59 void blk_mq_map_queues(struct blk_mq_queue_map *qmap) 20 60 { 21 61 const struct cpumask *masks; 22 22 - unsigned int queue, cpu; 62 62 + unsigned int queue, cpu, nr_masks; 23 63 24 24 - masks = group_cpus_evenly(qmap->nr_queues); 64 64 + masks = group_cpus_evenly(qmap->nr_queues, &nr_masks); 25 65 if (!masks) { 26 66 for_each_possible_cpu(cpu) 27 67 qmap->mq_map[cpu] = qmap->queue_offset; ··· 69 29 } 70 30 71 31 for (queue = 0; queue < qmap->nr_queues; queue++) { 72 72 - for_each_cpu(cpu, &masks[queue]) 32 32 + for_each_cpu(cpu, &masks[queue % nr_masks]) 73 33 qmap->mq_map[cpu] = qmap->queue_offset + queue; 74 34 } 75 35 kfree(masks);

+161

block/blk-mq-dma.c

reviewed

··· 2 2 /* 3 3 * Copyright (C) 2025 Christoph Hellwig 4 4 */ 5 5 + #include <linux/blk-mq-dma.h> 5 6 #include "blk.h" 6 7 7 8 struct phys_vec { ··· 61 60 vec->len = bv.bv_len; 62 61 return true; 63 62 } 63 63 + 64 64 + /* 65 65 + * The IOVA-based DMA API wants to be able to coalesce at the minimal IOMMU page 66 66 + * size granularity (which is guaranteed to be <= PAGE_SIZE and usually 4k), so 67 67 + * we need to ensure our segments are aligned to this as well. 68 68 + * 69 69 + * Note that there is no point in using the slightly more complicated IOVA based 70 70 + * path for single segment mappings. 71 71 + */ 72 72 + static inline bool blk_can_dma_map_iova(struct request *req, 73 73 + struct device *dma_dev) 74 74 + { 75 75 + return !((queue_virt_boundary(req->q) + 1) & 76 76 + dma_get_merge_boundary(dma_dev)); 77 77 + } 78 78 + 79 79 + static bool blk_dma_map_bus(struct blk_dma_iter *iter, struct phys_vec *vec) 80 80 + { 81 81 + iter->addr = pci_p2pdma_bus_addr_map(&iter->p2pdma, vec->paddr); 82 82 + iter->len = vec->len; 83 83 + return true; 84 84 + } 85 85 + 86 86 + static bool blk_dma_map_direct(struct request *req, struct device *dma_dev, 87 87 + struct blk_dma_iter *iter, struct phys_vec *vec) 88 88 + { 89 89 + iter->addr = dma_map_page(dma_dev, phys_to_page(vec->paddr), 90 90 + offset_in_page(vec->paddr), vec->len, rq_dma_dir(req)); 91 91 + if (dma_mapping_error(dma_dev, iter->addr)) { 92 92 + iter->status = BLK_STS_RESOURCE; 93 93 + return false; 94 94 + } 95 95 + iter->len = vec->len; 96 96 + return true; 97 97 + } 98 98 + 99 99 + static bool blk_rq_dma_map_iova(struct request *req, struct device *dma_dev, 100 100 + struct dma_iova_state *state, struct blk_dma_iter *iter, 101 101 + struct phys_vec *vec) 102 102 + { 103 103 + enum dma_data_direction dir = rq_dma_dir(req); 104 104 + unsigned int mapped = 0; 105 105 + int error; 106 106 + 107 107 + iter->addr = state->addr; 108 108 + iter->len = dma_iova_size(state); 109 109 + 110 110 + do { 111 111 + error = dma_iova_link(dma_dev, state, vec->paddr, mapped, 112 112 + vec->len, dir, 0); 113 113 + if (error) 114 114 + break; 115 115 + mapped += vec->len; 116 116 + } while (blk_map_iter_next(req, &iter->iter, vec)); 117 117 + 118 118 + error = dma_iova_sync(dma_dev, state, 0, mapped); 119 119 + if (error) { 120 120 + iter->status = errno_to_blk_status(error); 121 121 + return false; 122 122 + } 123 123 + 124 124 + return true; 125 125 + } 126 126 + 127 127 + /** 128 128 + * blk_rq_dma_map_iter_start - map the first DMA segment for a request 129 129 + * @req: request to map 130 130 + * @dma_dev: device to map to 131 131 + * @state: DMA IOVA state 132 132 + * @iter: block layer DMA iterator 133 133 + * 134 134 + * Start DMA mapping @req to @dma_dev. @state and @iter are provided by the 135 135 + * caller and don't need to be initialized. @state needs to be stored for use 136 136 + * at unmap time, @iter is only needed at map time. 137 137 + * 138 138 + * Returns %false if there is no segment to map, including due to an error, or 139 139 + * %true ft it did map a segment. 140 140 + * 141 141 + * If a segment was mapped, the DMA address for it is returned in @iter.addr and 142 142 + * the length in @iter.len. If no segment was mapped the status code is 143 143 + * returned in @iter.status. 144 144 + * 145 145 + * The caller can call blk_rq_dma_map_coalesce() to check if further segments 146 146 + * need to be mapped after this, or go straight to blk_rq_dma_map_iter_next() 147 147 + * to try to map the following segments. 148 148 + */ 149 149 + bool blk_rq_dma_map_iter_start(struct request *req, struct device *dma_dev, 150 150 + struct dma_iova_state *state, struct blk_dma_iter *iter) 151 151 + { 152 152 + unsigned int total_len = blk_rq_payload_bytes(req); 153 153 + struct phys_vec vec; 154 154 + 155 155 + iter->iter.bio = req->bio; 156 156 + iter->iter.iter = req->bio->bi_iter; 157 157 + memset(&iter->p2pdma, 0, sizeof(iter->p2pdma)); 158 158 + iter->status = BLK_STS_OK; 159 159 + 160 160 + /* 161 161 + * Grab the first segment ASAP because we'll need it to check for P2P 162 162 + * transfers. 163 163 + */ 164 164 + if (!blk_map_iter_next(req, &iter->iter, &vec)) 165 165 + return false; 166 166 + 167 167 + if (IS_ENABLED(CONFIG_PCI_P2PDMA) && (req->cmd_flags & REQ_P2PDMA)) { 168 168 + switch (pci_p2pdma_state(&iter->p2pdma, dma_dev, 169 169 + phys_to_page(vec.paddr))) { 170 170 + case PCI_P2PDMA_MAP_BUS_ADDR: 171 171 + return blk_dma_map_bus(iter, &vec); 172 172 + case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: 173 173 + /* 174 174 + * P2P transfers through the host bridge are treated the 175 175 + * same as non-P2P transfers below and during unmap. 176 176 + */ 177 177 + req->cmd_flags &= ~REQ_P2PDMA; 178 178 + break; 179 179 + default: 180 180 + iter->status = BLK_STS_INVAL; 181 181 + return false; 182 182 + } 183 183 + } 184 184 + 185 185 + if (blk_can_dma_map_iova(req, dma_dev) && 186 186 + dma_iova_try_alloc(dma_dev, state, vec.paddr, total_len)) 187 187 + return blk_rq_dma_map_iova(req, dma_dev, state, iter, &vec); 188 188 + return blk_dma_map_direct(req, dma_dev, iter, &vec); 189 189 + } 190 190 + EXPORT_SYMBOL_GPL(blk_rq_dma_map_iter_start); 191 191 + 192 192 + /** 193 193 + * blk_rq_dma_map_iter_next - map the next DMA segment for a request 194 194 + * @req: request to map 195 195 + * @dma_dev: device to map to 196 196 + * @state: DMA IOVA state 197 197 + * @iter: block layer DMA iterator 198 198 + * 199 199 + * Iterate to the next mapping after a previous call to 200 200 + * blk_rq_dma_map_iter_start(). See there for a detailed description of the 201 201 + * arguments. 202 202 + * 203 203 + * Returns %false if there is no segment to map, including due to an error, or 204 204 + * %true ft it did map a segment. 205 205 + * 206 206 + * If a segment was mapped, the DMA address for it is returned in @iter.addr and 207 207 + * the length in @iter.len. If no segment was mapped the status code is 208 208 + * returned in @iter.status. 209 209 + */ 210 210 + bool blk_rq_dma_map_iter_next(struct request *req, struct device *dma_dev, 211 211 + struct dma_iova_state *state, struct blk_dma_iter *iter) 212 212 + { 213 213 + struct phys_vec vec; 214 214 + 215 215 + if (!blk_map_iter_next(req, &iter->iter, &vec)) 216 216 + return false; 217 217 + 218 218 + if (iter->p2pdma.map == PCI_P2PDMA_MAP_BUS_ADDR) 219 219 + return blk_dma_map_bus(iter, &vec); 220 220 + return blk_dma_map_direct(req, dma_dev, iter, &vec); 221 221 + } 222 222 + EXPORT_SYMBOL_GPL(blk_rq_dma_map_iter_next); 64 223 65 224 static inline struct scatterlist * 66 225 blk_next_sg(struct scatterlist **sg, struct scatterlist *sglist)

+83 -13

block/blk-mq.c

reviewed

··· 883 883 /* Completion has already been traced */ 884 884 bio_clear_flag(bio, BIO_TRACE_COMPLETION); 885 885 886 886 - blk_zone_update_request_bio(req, bio); 886 886 + if (blk_req_bio_is_zone_append(req, bio)) 887 887 + blk_zone_append_update_request_bio(req, bio); 887 888 888 889 if (!is_flush) 889 890 bio_endio(bio); ··· 983 982 984 983 /* Don't actually finish bio if it's part of flush sequence */ 985 984 if (!bio->bi_iter.bi_size) { 986 986 - blk_zone_update_request_bio(req, bio); 985 985 + if (blk_req_bio_is_zone_append(req, bio)) 986 986 + blk_zone_append_update_request_bio(req, bio); 987 987 if (!is_flush) 988 988 bio_endio(bio); 989 989 } ··· 3171 3169 if (blk_mq_attempt_bio_merge(q, bio, nr_segs)) 3172 3170 goto queue_exit; 3173 3171 3174 3174 - if (blk_queue_is_zoned(q) && blk_zone_plug_bio(bio, nr_segs)) 3175 3175 - goto queue_exit; 3172 3172 + if (bio_needs_zone_write_plugging(bio)) { 3173 3173 + if (blk_zone_plug_bio(bio, nr_segs)) 3174 3174 + goto queue_exit; 3175 3175 + } 3176 3176 3177 3177 new_request: 3178 3178 if (rq) { ··· 4970 4966 return ret; 4971 4967 } 4972 4968 4969 4969 + /* 4970 4970 + * Switch back to the elevator type stored in the xarray. 4971 4971 + */ 4972 4972 + static void blk_mq_elv_switch_back(struct request_queue *q, 4973 4973 + struct xarray *elv_tbl) 4974 4974 + { 4975 4975 + struct elevator_type *e = xa_load(elv_tbl, q->id); 4976 4976 + 4977 4977 + /* The elv_update_nr_hw_queues unfreezes the queue. */ 4978 4978 + elv_update_nr_hw_queues(q, e); 4979 4979 + 4980 4980 + /* Drop the reference acquired in blk_mq_elv_switch_none. */ 4981 4981 + if (e) 4982 4982 + elevator_put(e); 4983 4983 + } 4984 4984 + 4985 4985 + /* 4986 4986 + * Stores elevator type in xarray and set current elevator to none. It uses 4987 4987 + * q->id as an index to store the elevator type into the xarray. 4988 4988 + */ 4989 4989 + static int blk_mq_elv_switch_none(struct request_queue *q, 4990 4990 + struct xarray *elv_tbl) 4991 4991 + { 4992 4992 + int ret = 0; 4993 4993 + 4994 4994 + lockdep_assert_held_write(&q->tag_set->update_nr_hwq_lock); 4995 4995 + 4996 4996 + /* 4997 4997 + * Accessing q->elevator without holding q->elevator_lock is safe here 4998 4998 + * because we're called from nr_hw_queue update which is protected by 4999 4999 + * set->update_nr_hwq_lock in the writer context. So, scheduler update/ 5000 5000 + * switch code (which acquires the same lock in the reader context) 5001 5001 + * can't run concurrently. 5002 5002 + */ 5003 5003 + if (q->elevator) { 5004 5004 + 5005 5005 + ret = xa_insert(elv_tbl, q->id, q->elevator->type, GFP_KERNEL); 5006 5006 + if (WARN_ON_ONCE(ret)) 5007 5007 + return ret; 5008 5008 + 5009 5009 + /* 5010 5010 + * Before we switch elevator to 'none', take a reference to 5011 5011 + * the elevator module so that while nr_hw_queue update is 5012 5012 + * running, no one can remove elevator module. We'd put the 5013 5013 + * reference to elevator module later when we switch back 5014 5014 + * elevator. 5015 5015 + */ 5016 5016 + __elevator_get(q->elevator->type); 5017 5017 + 5018 5018 + elevator_set_none(q); 5019 5019 + } 5020 5020 + return ret; 5021 5021 + } 5022 5022 + 4973 5023 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, 4974 5024 int nr_hw_queues) 4975 5025 { ··· 5031 4973 int prev_nr_hw_queues = set->nr_hw_queues; 5032 4974 unsigned int memflags; 5033 4975 int i; 4976 4976 + struct xarray elv_tbl; 5034 4977 5035 4978 lockdep_assert_held(&set->tag_list_lock); 5036 4979 ··· 5043 4984 return; 5044 4985 5045 4986 memflags = memalloc_noio_save(); 4987 4987 + 4988 4988 + xa_init(&elv_tbl); 4989 4989 + 5046 4990 list_for_each_entry(q, &set->tag_list, tag_set_list) { 5047 4991 blk_mq_debugfs_unregister_hctxs(q); 5048 4992 blk_mq_sysfs_unregister_hctxs(q); ··· 5054 4992 list_for_each_entry(q, &set->tag_list, tag_set_list) 5055 4993 blk_mq_freeze_queue_nomemsave(q); 5056 4994 5057 5057 - if (blk_mq_realloc_tag_set_tags(set, nr_hw_queues) < 0) { 5058 5058 - list_for_each_entry(q, &set->tag_list, tag_set_list) 5059 5059 - blk_mq_unfreeze_queue_nomemrestore(q); 5060 5060 - goto reregister; 5061 5061 - } 4995 4995 + /* 4996 4996 + * Switch IO scheduler to 'none', cleaning up the data associated 4997 4997 + * with the previous scheduler. We will switch back once we are done 4998 4998 + * updating the new sw to hw queue mappings. 4999 4999 + */ 5000 5000 + list_for_each_entry(q, &set->tag_list, tag_set_list) 5001 5001 + if (blk_mq_elv_switch_none(q, &elv_tbl)) 5002 5002 + goto switch_back; 5003 5003 + 5004 5004 + if (blk_mq_realloc_tag_set_tags(set, nr_hw_queues) < 0) 5005 5005 + goto switch_back; 5062 5006 5063 5007 fallback: 5064 5008 blk_mq_update_queue_map(set); ··· 5084 5016 } 5085 5017 blk_mq_map_swqueue(q); 5086 5018 } 5087 5087 - 5088 5088 - /* elv_update_nr_hw_queues() unfreeze queue for us */ 5019 5019 + switch_back: 5020 5020 + /* The blk_mq_elv_switch_back unfreezes queue for us. */ 5089 5021 list_for_each_entry(q, &set->tag_list, tag_set_list) 5090 5090 - elv_update_nr_hw_queues(q); 5022 5022 + blk_mq_elv_switch_back(q, &elv_tbl); 5091 5023 5092 5092 - reregister: 5093 5024 list_for_each_entry(q, &set->tag_list, tag_set_list) { 5094 5025 blk_mq_sysfs_register_hctxs(q); 5095 5026 blk_mq_debugfs_register_hctxs(q); ··· 5096 5029 blk_mq_remove_hw_queues_cpuhp(q); 5097 5030 blk_mq_add_hw_queues_cpuhp(q); 5098 5031 } 5032 5032 + 5033 5033 + xa_destroy(&elv_tbl); 5034 5034 + 5099 5035 memalloc_noio_restore(memflags); 5100 5036 5101 5037 /* Free the excess tags when nr_hw_queues shrink. */

+41 -25

block/blk-settings.c

reviewed

··· 221 221 static void blk_validate_atomic_write_limits(struct queue_limits *lim) 222 222 { 223 223 unsigned int boundary_sectors; 224 224 + unsigned int atomic_write_hw_max_sectors = 225 225 + lim->atomic_write_hw_max >> SECTOR_SHIFT; 224 226 225 227 if (!(lim->features & BLK_FEAT_ATOMIC_WRITES)) 226 228 goto unsupported; ··· 242 240 243 241 if (WARN_ON_ONCE(lim->atomic_write_hw_unit_max > 244 242 lim->atomic_write_hw_max)) 243 243 + goto unsupported; 244 244 + 245 245 + if (WARN_ON_ONCE(lim->chunk_sectors && 246 246 + atomic_write_hw_max_sectors > lim->chunk_sectors)) 245 247 goto unsupported; 246 248 247 249 boundary_sectors = lim->atomic_write_hw_boundary >> SECTOR_SHIFT; ··· 642 636 return true; 643 637 } 644 638 639 639 + static void blk_stack_atomic_writes_chunk_sectors(struct queue_limits *t) 640 640 + { 641 641 + unsigned int chunk_bytes; 642 642 + 643 643 + if (!t->chunk_sectors) 644 644 + return; 645 645 + 646 646 + /* 647 647 + * If chunk sectors is so large that its value in bytes overflows 648 648 + * UINT_MAX, then just shift it down so it definitely will fit. 649 649 + * We don't support atomic writes of such a large size anyway. 650 650 + */ 651 651 + if (check_shl_overflow(t->chunk_sectors, SECTOR_SHIFT, &chunk_bytes)) 652 652 + chunk_bytes = t->chunk_sectors; 653 653 + 654 654 + /* 655 655 + * Find values for limits which work for chunk size. 656 656 + * b->atomic_write_hw_unit_{min, max} may not be aligned with chunk 657 657 + * size, as the chunk size is not restricted to a power-of-2. 658 658 + * So we need to find highest power-of-2 which works for the chunk 659 659 + * size. 660 660 + * As an example scenario, we could have t->unit_max = 16K and 661 661 + * t->chunk_sectors = 24KB. For this case, reduce t->unit_max to a 662 662 + * value aligned with both limits, i.e. 8K in this example. 663 663 + */ 664 664 + t->atomic_write_hw_unit_max = min(t->atomic_write_hw_unit_max, 665 665 + max_pow_of_two_factor(chunk_bytes)); 666 666 + 667 667 + t->atomic_write_hw_unit_min = min(t->atomic_write_hw_unit_min, 668 668 + t->atomic_write_hw_unit_max); 669 669 + t->atomic_write_hw_max = min(t->atomic_write_hw_max, chunk_bytes); 670 670 + } 645 671 646 672 /* Check stacking of first bottom device */ 647 673 static bool blk_stack_atomic_writes_head(struct queue_limits *t, ··· 683 645 !blk_stack_atomic_writes_boundary_head(t, b)) 684 646 return false; 685 647 686 686 - if (t->io_min <= SECTOR_SIZE) { 687 687 - /* No chunk sectors, so use bottom device values directly */ 688 688 - t->atomic_write_hw_unit_max = b->atomic_write_hw_unit_max; 689 689 - t->atomic_write_hw_unit_min = b->atomic_write_hw_unit_min; 690 690 - t->atomic_write_hw_max = b->atomic_write_hw_max; 691 691 - return true; 692 692 - } 693 693 - 694 694 - /* 695 695 - * Find values for limits which work for chunk size. 696 696 - * b->atomic_write_hw_unit_{min, max} may not be aligned with chunk 697 697 - * size (t->io_min), as chunk size is not restricted to a power-of-2. 698 698 - * So we need to find highest power-of-2 which works for the chunk 699 699 - * size. 700 700 - * As an example scenario, we could have b->unit_max = 16K and 701 701 - * t->io_min = 24K. For this case, reduce t->unit_max to a value 702 702 - * aligned with both limits, i.e. 8K in this example. 703 703 - */ 704 648 t->atomic_write_hw_unit_max = b->atomic_write_hw_unit_max; 705 705 - while (t->io_min % t->atomic_write_hw_unit_max) 706 706 - t->atomic_write_hw_unit_max /= 2; 707 707 - 708 708 - t->atomic_write_hw_unit_min = min(b->atomic_write_hw_unit_min, 709 709 - t->atomic_write_hw_unit_max); 710 710 - t->atomic_write_hw_max = min(b->atomic_write_hw_max, t->io_min); 711 711 - 649 649 + t->atomic_write_hw_unit_min = b->atomic_write_hw_unit_min; 650 650 + t->atomic_write_hw_max = b->atomic_write_hw_max; 712 651 return true; 713 652 } 714 653 ··· 713 698 714 699 if (!blk_stack_atomic_writes_head(t, b)) 715 700 goto unsupported; 701 701 + blk_stack_atomic_writes_chunk_sectors(t); 716 702 return; 717 703 718 704 unsupported:

+24 -19

block/blk-zoned.c

reviewed

··· 17 17 #include <linux/refcount.h> 18 18 #include <linux/mempool.h> 19 19 20 20 + #include <trace/events/block.h> 21 21 + 20 22 #include "blk.h" 21 23 #include "blk-mq-sched.h" 22 24 #include "blk-mq-debugfs.h" ··· 179 177 struct bio bio; 180 178 181 179 bio_init(&bio, bdev, NULL, 0, REQ_OP_ZONE_RESET_ALL | REQ_SYNC); 180 180 + trace_blkdev_zone_mgmt(&bio, 0); 182 181 return submit_bio_wait(&bio); 183 182 } 184 183 ··· 243 240 cond_resched(); 244 241 } 245 242 243 243 + trace_blkdev_zone_mgmt(bio, nr_sectors); 246 244 ret = submit_bio_wait(bio); 247 245 bio_put(bio); 248 246 ··· 822 818 * at the tail of the list to preserve the sequential write order. 823 819 */ 824 820 bio_list_add(&zwplug->bio_list, bio); 821 821 + trace_disk_zone_wplug_add_bio(zwplug->disk->queue, zwplug->zone_no, 822 822 + bio->bi_iter.bi_sector, bio_sectors(bio)); 825 823 826 824 zwplug->flags |= BLK_ZONE_WPLUG_PLUGGED; 827 825 ··· 1122 1116 { 1123 1117 struct block_device *bdev = bio->bi_bdev; 1124 1118 1125 1125 - if (!bdev->bd_disk->zone_wplugs_hash) 1126 1126 - return false; 1127 1127 - 1128 1128 - /* 1129 1129 - * If the BIO already has the plugging flag set, then it was already 1130 1130 - * handled through this path and this is a submission from the zone 1131 1131 - * plug bio submit work. 1132 1132 - */ 1133 1133 - if (bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING)) 1134 1134 - return false; 1135 1135 - 1136 1136 - /* 1137 1137 - * We do not need to do anything special for empty flush BIOs, e.g 1138 1138 - * BIOs such as issued by blkdev_issue_flush(). The is because it is 1139 1139 - * the responsibility of the user to first wait for the completion of 1140 1140 - * write operations for flush to have any effect on the persistence of 1141 1141 - * the written data. 1142 1142 - */ 1143 1143 - if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) 1119 1119 + if (WARN_ON_ONCE(!bdev->bd_disk->zone_wplugs_hash)) 1144 1120 return false; 1145 1121 1146 1122 /* ··· 1191 1203 disk_remove_zone_wplug(disk, zwplug); 1192 1204 1193 1205 spin_unlock_irqrestore(&zwplug->lock, flags); 1206 1206 + } 1207 1207 + 1208 1208 + void blk_zone_append_update_request_bio(struct request *rq, struct bio *bio) 1209 1209 + { 1210 1210 + /* 1211 1211 + * For zone append requests, the request sector indicates the location 1212 1212 + * at which the BIO data was written. Return this value to the BIO 1213 1213 + * issuer through the BIO iter sector. 1214 1214 + * For plugged zone writes, which include emulated zone append, we need 1215 1215 + * the original BIO sector so that blk_zone_write_plug_bio_endio() can 1216 1216 + * lookup the zone write plug. 1217 1217 + */ 1218 1218 + bio->bi_iter.bi_sector = rq->__sector; 1219 1219 + trace_blk_zone_append_update_request_bio(rq); 1194 1220 } 1195 1221 1196 1222 void blk_zone_write_plug_bio_endio(struct bio *bio) ··· 1300 1298 spin_unlock_irqrestore(&zwplug->lock, flags); 1301 1299 goto put_zwplug; 1302 1300 } 1301 1301 + 1302 1302 + trace_blk_zone_wplug_bio(zwplug->disk->queue, zwplug->zone_no, 1303 1303 + bio->bi_iter.bi_sector, bio_sectors(bio)); 1303 1304 1304 1305 if (!blk_zone_wplug_prepare_bio(zwplug, bio)) { 1305 1306 blk_zone_wplug_bio_io_error(zwplug, bio);

+24 -18

block/blk.h

reviewed

··· 13 13 14 14 struct elevator_type; 15 15 16 16 + /* 17 17 + * Default upper limit for the software max_sectors limit used for regular I/Os. 18 18 + * This can be increased through sysfs. 19 19 + * 20 20 + * This should not be confused with the max_hw_sector limit that is entirely 21 21 + * controlled by the block device driver, usually based on hardware limits. 22 22 + */ 23 23 + #define BLK_DEF_MAX_SECTORS_CAP (SZ_4M >> SECTOR_SHIFT) 24 24 + 16 25 #define BLK_DEV_MAX_SECTORS (LLONG_MAX >> 9) 17 26 #define BLK_MIN_SEGMENT_SIZE 4096 18 27 ··· 330 321 331 322 bool blk_insert_flush(struct request *rq); 332 323 333 333 - void elv_update_nr_hw_queues(struct request_queue *q); 324 324 + void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e); 334 325 void elevator_set_default(struct request_queue *q); 335 326 void elevator_set_none(struct request_queue *q); 336 327 ··· 476 467 { 477 468 return bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING); 478 469 } 470 470 + static inline bool blk_req_bio_is_zone_append(struct request *rq, 471 471 + struct bio *bio) 472 472 + { 473 473 + return req_op(rq) == REQ_OP_ZONE_APPEND || 474 474 + bio_flagged(bio, BIO_EMULATES_ZONE_APPEND); 475 475 + } 479 476 void blk_zone_write_plug_bio_merged(struct bio *bio); 480 477 void blk_zone_write_plug_init_request(struct request *rq); 481 481 - static inline void blk_zone_update_request_bio(struct request *rq, 482 482 - struct bio *bio) 483 483 - { 484 484 - /* 485 485 - * For zone append requests, the request sector indicates the location 486 486 - * at which the BIO data was written. Return this value to the BIO 487 487 - * issuer through the BIO iter sector. 488 488 - * For plugged zone writes, which include emulated zone append, we need 489 489 - * the original BIO sector so that blk_zone_write_plug_bio_endio() can 490 490 - * lookup the zone write plug. 491 491 - */ 492 492 - if (req_op(rq) == REQ_OP_ZONE_APPEND || 493 493 - bio_flagged(bio, BIO_EMULATES_ZONE_APPEND)) 494 494 - bio->bi_iter.bi_sector = rq->__sector; 495 495 - } 478 478 + void blk_zone_append_update_request_bio(struct request *rq, struct bio *bio); 496 479 void blk_zone_write_plug_bio_endio(struct bio *bio); 497 480 static inline void blk_zone_bio_endio(struct bio *bio) 498 481 { ··· 517 516 { 518 517 return false; 519 518 } 519 519 + static inline bool blk_req_bio_is_zone_append(struct request *req, 520 520 + struct bio *bio) 521 521 + { 522 522 + return false; 523 523 + } 520 524 static inline void blk_zone_write_plug_bio_merged(struct bio *bio) 521 525 { 522 526 } 523 527 static inline void blk_zone_write_plug_init_request(struct request *rq) 524 528 { 525 529 } 526 526 - static inline void blk_zone_update_request_bio(struct request *rq, 527 527 - struct bio *bio) 530 530 + static inline void blk_zone_append_update_request_bio(struct request *rq, 531 531 + struct bio *bio) 528 532 { 529 533 } 530 534 static inline void blk_zone_bio_endio(struct bio *bio)

+5 -5

block/elevator.c

reviewed

··· 689 689 * The I/O scheduler depends on the number of hardware queues, this forces a 690 690 * reattachment when nr_hw_queues changes. 691 691 */ 692 692 - void elv_update_nr_hw_queues(struct request_queue *q) 692 692 + void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e) 693 693 { 694 694 struct elv_change_ctx ctx = {}; 695 695 int ret = -ENODEV; 696 696 697 697 WARN_ON_ONCE(q->mq_freeze_depth == 0); 698 698 699 699 - mutex_lock(&q->elevator_lock); 700 700 - if (q->elevator && !blk_queue_dying(q) && blk_queue_registered(q)) { 701 701 - ctx.name = q->elevator->type->elevator_name; 699 699 + if (e && !blk_queue_dying(q) && blk_queue_registered(q)) { 700 700 + ctx.name = e->elevator_name; 702 701 702 702 + mutex_lock(&q->elevator_lock); 703 703 /* force to reattach elevator after nr_hw_queue is updated */ 704 704 ret = elevator_switch(q, &ctx); 705 705 + mutex_unlock(&q->elevator_lock); 705 706 } 706 706 - mutex_unlock(&q->elevator_lock); 707 707 blk_mq_unfreeze_queue_nomemrestore(q); 708 708 if (!ret) 709 709 WARN_ON_ONCE(elevator_change_done(q, &ctx));

-43

drivers/block/Kconfig

reviewed

··· 256 256 The default value is 4096 kilobytes. Only change this if you know 257 257 what you are doing. 258 258 259 259 - config CDROM_PKTCDVD 260 260 - tristate "Packet writing on CD/DVD media (DEPRECATED)" 261 261 - depends on !UML 262 262 - depends on SCSI 263 263 - select CDROM 264 264 - help 265 265 - Note: This driver is deprecated and will be removed from the 266 266 - kernel in the near future! 267 267 - 268 268 - If you have a CDROM/DVD drive that supports packet writing, say 269 269 - Y to include support. It should work with any MMC/Mt Fuji 270 270 - compliant ATAPI or SCSI drive, which is just about any newer 271 271 - DVD/CD writer. 272 272 - 273 273 - Currently only writing to CD-RW, DVD-RW, DVD+RW and DVDRAM discs 274 274 - is possible. 275 275 - DVD-RW disks must be in restricted overwrite mode. 276 276 - 277 277 - See the file <file:Documentation/cdrom/packet-writing.rst> 278 278 - for further information on the use of this driver. 279 279 - 280 280 - To compile this driver as a module, choose M here: the 281 281 - module will be called pktcdvd. 282 282 - 283 283 - config CDROM_PKTCDVD_BUFFERS 284 284 - int "Free buffers for data gathering" 285 285 - depends on CDROM_PKTCDVD 286 286 - default "8" 287 287 - help 288 288 - This controls the maximum number of active concurrent packets. More 289 289 - concurrent packets can increase write performance, but also require 290 290 - more memory. Each concurrent packet will require approximately 64Kb 291 291 - of non-swappable kernel memory, memory which will be allocated when 292 292 - a disc is opened for writing. 293 293 - 294 294 - config CDROM_PKTCDVD_WCACHE 295 295 - bool "Enable write caching" 296 296 - depends on CDROM_PKTCDVD 297 297 - help 298 298 - If enabled, write caching will be set for the CD-R/W device. For now 299 299 - this option is dangerous unless the CD-RW media is known good, as we 300 300 - don't do deferred write error handling yet. 301 301 - 302 259 config ATA_OVER_ETH 303 260 tristate "ATA over Ethernet support" 304 261 depends on NET

-1

drivers/block/Makefile

reviewed

··· 23 23 obj-$(CONFIG_N64CART) += n64cart.o 24 24 obj-$(CONFIG_BLK_DEV_RAM) += brd.o 25 25 obj-$(CONFIG_BLK_DEV_LOOP) += loop.o 26 26 - obj-$(CONFIG_CDROM_PKTCDVD) += pktcdvd.o 27 26 obj-$(CONFIG_SUNVDC) += sunvdc.o 28 27 29 28 obj-$(CONFIG_BLK_DEV_NBD) += nbd.o

+5 -1

drivers/block/drbd/drbd_receiver.c

reviewed

··· 2500 2500 peer_req->w.cb = superseded ? e_send_superseded : 2501 2501 e_send_retry_write; 2502 2502 list_add_tail(&peer_req->w.list, &device->done_ee); 2503 2503 - queue_work(connection->ack_sender, &peer_req->peer_device->send_acks_work); 2503 2503 + /* put is in drbd_send_acks_wf() */ 2504 2504 + kref_get(&device->kref); 2505 2505 + if (!queue_work(connection->ack_sender, 2506 2506 + &peer_req->peer_device->send_acks_work)) 2507 2507 + kref_put(&device->kref, drbd_destroy_device); 2504 2508 2505 2509 err = -ENOENT; 2506 2510 goto out;

+1 -1

drivers/block/floppy.c

reviewed

··· 3411 3411 struct floppy_max_errors max_errors; 3412 3412 struct floppy_drive_params dp; 3413 3413 } inparam; /* parameters coming from user space */ 3414 3414 - const void *outparam; /* parameters passed back to user space */ 3414 3414 + const void *outparam = NULL; /* parameters passed back to user space */ 3415 3415 3416 3416 /* convert compatibility eject ioctls into floppy eject ioctl. 3417 3417 * We do this in order to provide a means to eject floppy disks before

+30 -8

drivers/block/loop.c

reviewed

··· 1431 1431 return 0; 1432 1432 } 1433 1433 1434 1434 - static int loop_set_block_size(struct loop_device *lo, unsigned long arg) 1434 1434 + static int loop_set_block_size(struct loop_device *lo, blk_mode_t mode, 1435 1435 + struct block_device *bdev, unsigned long arg) 1435 1436 { 1436 1437 struct queue_limits lim; 1437 1438 unsigned int memflags; 1438 1439 int err = 0; 1439 1440 1440 1440 - if (lo->lo_state != Lo_bound) 1441 1441 - return -ENXIO; 1441 1441 + /* 1442 1442 + * If we don't hold exclusive handle for the device, upgrade to it 1443 1443 + * here to avoid changing device under exclusive owner. 1444 1444 + */ 1445 1445 + if (!(mode & BLK_OPEN_EXCL)) { 1446 1446 + err = bd_prepare_to_claim(bdev, loop_set_block_size, NULL); 1447 1447 + if (err) 1448 1448 + return err; 1449 1449 + } 1450 1450 + 1451 1451 + err = mutex_lock_killable(&lo->lo_mutex); 1452 1452 + if (err) 1453 1453 + goto abort_claim; 1454 1454 + 1455 1455 + if (lo->lo_state != Lo_bound) { 1456 1456 + err = -ENXIO; 1457 1457 + goto unlock; 1458 1458 + } 1442 1459 1443 1460 if (lo->lo_queue->limits.logical_block_size == arg) 1444 1444 - return 0; 1461 1461 + goto unlock; 1445 1462 1446 1463 sync_blockdev(lo->lo_device); 1447 1464 invalidate_bdev(lo->lo_device); ··· 1471 1454 loop_update_dio(lo); 1472 1455 blk_mq_unfreeze_queue(lo->lo_queue, memflags); 1473 1456 1457 1457 + unlock: 1458 1458 + mutex_unlock(&lo->lo_mutex); 1459 1459 + abort_claim: 1460 1460 + if (!(mode & BLK_OPEN_EXCL)) 1461 1461 + bd_abort_claiming(bdev, loop_set_block_size); 1474 1462 return err; 1475 1463 } 1476 1464 ··· 1493 1471 break; 1494 1472 case LOOP_SET_DIRECT_IO: 1495 1473 err = loop_set_dio(lo, arg); 1496 1496 - break; 1497 1497 - case LOOP_SET_BLOCK_SIZE: 1498 1498 - err = loop_set_block_size(lo, arg); 1499 1474 break; 1500 1475 default: 1501 1476 err = -EINVAL; ··· 1548 1529 break; 1549 1530 case LOOP_GET_STATUS64: 1550 1531 return loop_get_status64(lo, argp); 1532 1532 + case LOOP_SET_BLOCK_SIZE: 1533 1533 + if (!(mode & BLK_OPEN_WRITE) && !capable(CAP_SYS_ADMIN)) 1534 1534 + return -EPERM; 1535 1535 + return loop_set_block_size(lo, mode, bdev, arg); 1551 1536 case LOOP_SET_CAPACITY: 1552 1537 case LOOP_SET_DIRECT_IO: 1553 1553 - case LOOP_SET_BLOCK_SIZE: 1554 1538 if (!(mode & BLK_OPEN_WRITE) && !capable(CAP_SYS_ADMIN)) 1555 1539 return -EPERM; 1556 1540 fallthrough;

+17 -10

drivers/block/mtip32xx/mtip32xx.c

reviewed

··· 2040 2040 * @dir Direction (read or write) 2041 2041 * 2042 2042 * return value 2043 2043 - * None 2043 2043 + * 0 The IO completed successfully. 2044 2044 + * -ENOMEM The DMA mapping failed. 2044 2045 */ 2045 2045 - static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq, 2046 2046 - struct mtip_cmd *command, 2047 2047 - struct blk_mq_hw_ctx *hctx) 2046 2046 + static int mtip_hw_submit_io(struct driver_data *dd, struct request *rq, 2047 2047 + struct mtip_cmd *command, 2048 2048 + struct blk_mq_hw_ctx *hctx) 2048 2049 { 2049 2050 struct mtip_cmd_hdr *hdr = 2050 2051 dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag; ··· 2057 2056 unsigned int nents; 2058 2057 2059 2058 /* Map the scatter list for DMA access */ 2060 2060 - nents = blk_rq_map_sg(rq, command->sg); 2061 2061 - nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); 2059 2059 + command->scatter_ents = blk_rq_map_sg(rq, command->sg); 2060 2060 + nents = dma_map_sg(&dd->pdev->dev, command->sg, 2061 2061 + command->scatter_ents, dma_dir); 2062 2062 + if (!nents) 2063 2063 + return -ENOMEM; 2064 2064 + 2062 2065 2063 2066 prefetch(&port->flags); 2064 2064 - 2065 2065 - command->scatter_ents = nents; 2066 2067 2067 2068 /* 2068 2069 * The number of retries for this command before it is ··· 2115 2112 if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) { 2116 2113 set_bit(rq->tag, port->cmds_to_issue); 2117 2114 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); 2118 2118 - return; 2115 2115 + return 0; 2119 2116 } 2120 2117 2121 2118 /* Issue the command to the hardware */ 2122 2119 mtip_issue_ncq_command(port, rq->tag); 2120 2120 + 2121 2121 + return 0; 2123 2122 } 2124 2123 2125 2124 /* ··· 3320 3315 3321 3316 blk_mq_start_request(rq); 3322 3317 3323 3323 - mtip_hw_submit_io(dd, rq, cmd, hctx); 3318 3318 + if (mtip_hw_submit_io(dd, rq, cmd, hctx)) 3319 3319 + return BLK_STS_IOERR; 3320 3320 + 3324 3321 return BLK_STS_OK; 3325 3322 } 3326 3323

+11 -1

drivers/block/nbd.c

reviewed

··· 1473 1473 return -EINVAL; 1474 1474 } 1475 1475 1476 1476 - blk_mq_update_nr_hw_queues(&nbd->tag_set, config->num_connections); 1476 1476 + retry: 1477 1477 + mutex_unlock(&nbd->config_lock); 1478 1478 + blk_mq_update_nr_hw_queues(&nbd->tag_set, num_connections); 1479 1479 + mutex_lock(&nbd->config_lock); 1480 1480 + 1481 1481 + /* if another code path updated nr_hw_queues, retry until succeed */ 1482 1482 + if (num_connections != config->num_connections) { 1483 1483 + num_connections = config->num_connections; 1484 1484 + goto retry; 1485 1485 + } 1486 1486 + 1477 1487 nbd->pid = task_pid_nr(current); 1478 1488 1479 1489 nbd_parse_flags(nbd);

-2916

drivers/block/pktcdvd.c

reviewed

··· 1 1 - /* 2 2 - * Copyright (C) 2000 Jens Axboe <axboe@suse.de> 3 3 - * Copyright (C) 2001-2004 Peter Osterlund <petero2@telia.com> 4 4 - * Copyright (C) 2006 Thomas Maier <balagi@justmail.de> 5 5 - * 6 6 - * May be copied or modified under the terms of the GNU General Public 7 7 - * License. See linux/COPYING for more information. 8 8 - * 9 9 - * Packet writing layer for ATAPI and SCSI CD-RW, DVD+RW, DVD-RW and 10 10 - * DVD-RAM devices. 11 11 - * 12 12 - * Theory of operation: 13 13 - * 14 14 - * At the lowest level, there is the standard driver for the CD/DVD device, 15 15 - * such as drivers/scsi/sr.c. This driver can handle read and write requests, 16 16 - * but it doesn't know anything about the special restrictions that apply to 17 17 - * packet writing. One restriction is that write requests must be aligned to 18 18 - * packet boundaries on the physical media, and the size of a write request 19 19 - * must be equal to the packet size. Another restriction is that a 20 20 - * GPCMD_FLUSH_CACHE command has to be issued to the drive before a read 21 21 - * command, if the previous command was a write. 22 22 - * 23 23 - * The purpose of the packet writing driver is to hide these restrictions from 24 24 - * higher layers, such as file systems, and present a block device that can be 25 25 - * randomly read and written using 2kB-sized blocks. 26 26 - * 27 27 - * The lowest layer in the packet writing driver is the packet I/O scheduler. 28 28 - * Its data is defined by the struct packet_iosched and includes two bio 29 29 - * queues with pending read and write requests. These queues are processed 30 30 - * by the pkt_iosched_process_queue() function. The write requests in this 31 31 - * queue are already properly aligned and sized. This layer is responsible for 32 32 - * issuing the flush cache commands and scheduling the I/O in a good order. 33 33 - * 34 34 - * The next layer transforms unaligned write requests to aligned writes. This 35 35 - * transformation requires reading missing pieces of data from the underlying 36 36 - * block device, assembling the pieces to full packets and queuing them to the 37 37 - * packet I/O scheduler. 38 38 - * 39 39 - * At the top layer there is a custom ->submit_bio function that forwards 40 40 - * read requests directly to the iosched queue and puts write requests in the 41 41 - * unaligned write queue. A kernel thread performs the necessary read 42 42 - * gathering to convert the unaligned writes to aligned writes and then feeds 43 43 - * them to the packet I/O scheduler. 44 44 - * 45 45 - *************************************************************************/ 46 46 - 47 47 - #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 48 48 - 49 49 - #include <linux/backing-dev.h> 50 50 - #include <linux/compat.h> 51 51 - #include <linux/debugfs.h> 52 52 - #include <linux/device.h> 53 53 - #include <linux/errno.h> 54 54 - #include <linux/file.h> 55 55 - #include <linux/freezer.h> 56 56 - #include <linux/kernel.h> 57 57 - #include <linux/kthread.h> 58 58 - #include <linux/miscdevice.h> 59 59 - #include <linux/module.h> 60 60 - #include <linux/mutex.h> 61 61 - #include <linux/nospec.h> 62 62 - #include <linux/pktcdvd.h> 63 63 - #include <linux/proc_fs.h> 64 64 - #include <linux/seq_file.h> 65 65 - #include <linux/slab.h> 66 66 - #include <linux/spinlock.h> 67 67 - #include <linux/types.h> 68 68 - #include <linux/uaccess.h> 69 69 - 70 70 - #include <scsi/scsi.h> 71 71 - #include <scsi/scsi_cmnd.h> 72 72 - #include <scsi/scsi_ioctl.h> 73 73 - 74 74 - #include <linux/unaligned.h> 75 75 - 76 76 - #define DRIVER_NAME "pktcdvd" 77 77 - 78 78 - #define MAX_SPEED 0xffff 79 79 - 80 80 - static DEFINE_MUTEX(pktcdvd_mutex); 81 81 - static struct pktcdvd_device *pkt_devs[MAX_WRITERS]; 82 82 - static struct proc_dir_entry *pkt_proc; 83 83 - static int pktdev_major; 84 84 - static int write_congestion_on = PKT_WRITE_CONGESTION_ON; 85 85 - static int write_congestion_off = PKT_WRITE_CONGESTION_OFF; 86 86 - static struct mutex ctl_mutex; /* Serialize open/close/setup/teardown */ 87 87 - static mempool_t psd_pool; 88 88 - static struct bio_set pkt_bio_set; 89 89 - 90 90 - /* /sys/class/pktcdvd */ 91 91 - static struct class class_pktcdvd; 92 92 - static struct dentry *pkt_debugfs_root = NULL; /* /sys/kernel/debug/pktcdvd */ 93 93 - 94 94 - /* forward declaration */ 95 95 - static int pkt_setup_dev(dev_t dev, dev_t* pkt_dev); 96 96 - static int pkt_remove_dev(dev_t pkt_dev); 97 97 - 98 98 - static sector_t get_zone(sector_t sector, struct pktcdvd_device *pd) 99 99 - { 100 100 - return (sector + pd->offset) & ~(sector_t)(pd->settings.size - 1); 101 101 - } 102 102 - 103 103 - /********************************************************** 104 104 - * sysfs interface for pktcdvd 105 105 - * by (C) 2006 Thomas Maier <balagi@justmail.de> 106 106 - 107 107 - /sys/class/pktcdvd/pktcdvd[0-7]/ 108 108 - stat/reset 109 109 - stat/packets_started 110 110 - stat/packets_finished 111 111 - stat/kb_written 112 112 - stat/kb_read 113 113 - stat/kb_read_gather 114 114 - write_queue/size 115 115 - write_queue/congestion_off 116 116 - write_queue/congestion_on 117 117 - **********************************************************/ 118 118 - 119 119 - static ssize_t packets_started_show(struct device *dev, 120 120 - struct device_attribute *attr, char *buf) 121 121 - { 122 122 - struct pktcdvd_device *pd = dev_get_drvdata(dev); 123 123 - 124 124 - return sysfs_emit(buf, "%lu\n", pd->stats.pkt_started); 125 125 - } 126 126 - static DEVICE_ATTR_RO(packets_started); 127 127 - 128 128 - static ssize_t packets_finished_show(struct device *dev, 129 129 - struct device_attribute *attr, char *buf) 130 130 - { 131 131 - struct pktcdvd_device *pd = dev_get_drvdata(dev); 132 132 - 133 133 - return sysfs_emit(buf, "%lu\n", pd->stats.pkt_ended); 134 134 - } 135 135 - static DEVICE_ATTR_RO(packets_finished); 136 136 - 137 137 - static ssize_t kb_written_show(struct device *dev, 138 138 - struct device_attribute *attr, char *buf) 139 139 - { 140 140 - struct pktcdvd_device *pd = dev_get_drvdata(dev); 141 141 - 142 142 - return sysfs_emit(buf, "%lu\n", pd->stats.secs_w >> 1); 143 143 - } 144 144 - static DEVICE_ATTR_RO(kb_written); 145 145 - 146 146 - static ssize_t kb_read_show(struct device *dev, 147 147 - struct device_attribute *attr, char *buf) 148 148 - { 149 149 - struct pktcdvd_device *pd = dev_get_drvdata(dev); 150 150 - 151 151 - return sysfs_emit(buf, "%lu\n", pd->stats.secs_r >> 1); 152 152 - } 153 153 - static DEVICE_ATTR_RO(kb_read); 154 154 - 155 155 - static ssize_t kb_read_gather_show(struct device *dev, 156 156 - struct device_attribute *attr, char *buf) 157 157 - { 158 158 - struct pktcdvd_device *pd = dev_get_drvdata(dev); 159 159 - 160 160 - return sysfs_emit(buf, "%lu\n", pd->stats.secs_rg >> 1); 161 161 - } 162 162 - static DEVICE_ATTR_RO(kb_read_gather); 163 163 - 164 164 - static ssize_t reset_store(struct device *dev, struct device_attribute *attr, 165 165 - const char *buf, size_t len) 166 166 - { 167 167 - struct pktcdvd_device *pd = dev_get_drvdata(dev); 168 168 - 169 169 - if (len > 0) { 170 170 - pd->stats.pkt_started = 0; 171 171 - pd->stats.pkt_ended = 0; 172 172 - pd->stats.secs_w = 0; 173 173 - pd->stats.secs_rg = 0; 174 174 - pd->stats.secs_r = 0; 175 175 - } 176 176 - return len; 177 177 - } 178 178 - static DEVICE_ATTR_WO(reset); 179 179 - 180 180 - static struct attribute *pkt_stat_attrs[] = { 181 181 - &dev_attr_packets_finished.attr, 182 182 - &dev_attr_packets_started.attr, 183 183 - &dev_attr_kb_read.attr, 184 184 - &dev_attr_kb_written.attr, 185 185 - &dev_attr_kb_read_gather.attr, 186 186 - &dev_attr_reset.attr, 187 187 - NULL, 188 188 - }; 189 189 - 190 190 - static const struct attribute_group pkt_stat_group = { 191 191 - .name = "stat", 192 192 - .attrs = pkt_stat_attrs, 193 193 - }; 194 194 - 195 195 - static ssize_t size_show(struct device *dev, 196 196 - struct device_attribute *attr, char *buf) 197 197 - { 198 198 - struct pktcdvd_device *pd = dev_get_drvdata(dev); 199 199 - int n; 200 200 - 201 201 - spin_lock(&pd->lock); 202 202 - n = sysfs_emit(buf, "%d\n", pd->bio_queue_size); 203 203 - spin_unlock(&pd->lock); 204 204 - return n; 205 205 - } 206 206 - static DEVICE_ATTR_RO(size); 207 207 - 208 208 - static void init_write_congestion_marks(int* lo, int* hi) 209 209 - { 210 210 - if (*hi > 0) { 211 211 - *hi = max(*hi, 500); 212 212 - *hi = min(*hi, 1000000); 213 213 - if (*lo <= 0) 214 214 - *lo = *hi - 100; 215 215 - else { 216 216 - *lo = min(*lo, *hi - 100); 217 217 - *lo = max(*lo, 100); 218 218 - } 219 219 - } else { 220 220 - *hi = -1; 221 221 - *lo = -1; 222 222 - } 223 223 - } 224 224 - 225 225 - static ssize_t congestion_off_show(struct device *dev, 226 226 - struct device_attribute *attr, char *buf) 227 227 - { 228 228 - struct pktcdvd_device *pd = dev_get_drvdata(dev); 229 229 - int n; 230 230 - 231 231 - spin_lock(&pd->lock); 232 232 - n = sysfs_emit(buf, "%d\n", pd->write_congestion_off); 233 233 - spin_unlock(&pd->lock); 234 234 - return n; 235 235 - } 236 236 - 237 237 - static ssize_t congestion_off_store(struct device *dev, 238 238 - struct device_attribute *attr, 239 239 - const char *buf, size_t len) 240 240 - { 241 241 - struct pktcdvd_device *pd = dev_get_drvdata(dev); 242 242 - int val, ret; 243 243 - 244 244 - ret = kstrtoint(buf, 10, &val); 245 245 - if (ret) 246 246 - return ret; 247 247 - 248 248 - spin_lock(&pd->lock); 249 249 - pd->write_congestion_off = val; 250 250 - init_write_congestion_marks(&pd->write_congestion_off, &pd->write_congestion_on); 251 251 - spin_unlock(&pd->lock); 252 252 - return len; 253 253 - } 254 254 - static DEVICE_ATTR_RW(congestion_off); 255 255 - 256 256 - static ssize_t congestion_on_show(struct device *dev, 257 257 - struct device_attribute *attr, char *buf) 258 258 - { 259 259 - struct pktcdvd_device *pd = dev_get_drvdata(dev); 260 260 - int n; 261 261 - 262 262 - spin_lock(&pd->lock); 263 263 - n = sysfs_emit(buf, "%d\n", pd->write_congestion_on); 264 264 - spin_unlock(&pd->lock); 265 265 - return n; 266 266 - } 267 267 - 268 268 - static ssize_t congestion_on_store(struct device *dev, 269 269 - struct device_attribute *attr, 270 270 - const char *buf, size_t len) 271 271 - { 272 272 - struct pktcdvd_device *pd = dev_get_drvdata(dev); 273 273 - int val, ret; 274 274 - 275 275 - ret = kstrtoint(buf, 10, &val); 276 276 - if (ret) 277 277 - return ret; 278 278 - 279 279 - spin_lock(&pd->lock); 280 280 - pd->write_congestion_on = val; 281 281 - init_write_congestion_marks(&pd->write_congestion_off, &pd->write_congestion_on); 282 282 - spin_unlock(&pd->lock); 283 283 - return len; 284 284 - } 285 285 - static DEVICE_ATTR_RW(congestion_on); 286 286 - 287 287 - static struct attribute *pkt_wq_attrs[] = { 288 288 - &dev_attr_congestion_on.attr, 289 289 - &dev_attr_congestion_off.attr, 290 290 - &dev_attr_size.attr, 291 291 - NULL, 292 292 - }; 293 293 - 294 294 - static const struct attribute_group pkt_wq_group = { 295 295 - .name = "write_queue", 296 296 - .attrs = pkt_wq_attrs, 297 297 - }; 298 298 - 299 299 - static const struct attribute_group *pkt_groups[] = { 300 300 - &pkt_stat_group, 301 301 - &pkt_wq_group, 302 302 - NULL, 303 303 - }; 304 304 - 305 305 - static void pkt_sysfs_dev_new(struct pktcdvd_device *pd) 306 306 - { 307 307 - if (class_is_registered(&class_pktcdvd)) { 308 308 - pd->dev = device_create_with_groups(&class_pktcdvd, NULL, 309 309 - MKDEV(0, 0), pd, pkt_groups, 310 310 - "%s", pd->disk->disk_name); 311 311 - if (IS_ERR(pd->dev)) 312 312 - pd->dev = NULL; 313 313 - } 314 314 - } 315 315 - 316 316 - static void pkt_sysfs_dev_remove(struct pktcdvd_device *pd) 317 317 - { 318 318 - if (class_is_registered(&class_pktcdvd)) 319 319 - device_unregister(pd->dev); 320 320 - } 321 321 - 322 322 - 323 323 - /******************************************************************** 324 324 - /sys/class/pktcdvd/ 325 325 - add map block device 326 326 - remove unmap packet dev 327 327 - device_map show mappings 328 328 - *******************************************************************/ 329 329 - 330 330 - static ssize_t device_map_show(const struct class *c, const struct class_attribute *attr, 331 331 - char *data) 332 332 - { 333 333 - int n = 0; 334 334 - int idx; 335 335 - mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING); 336 336 - for (idx = 0; idx < MAX_WRITERS; idx++) { 337 337 - struct pktcdvd_device *pd = pkt_devs[idx]; 338 338 - if (!pd) 339 339 - continue; 340 340 - n += sysfs_emit_at(data, n, "%s %u:%u %u:%u\n", 341 341 - pd->disk->disk_name, 342 342 - MAJOR(pd->pkt_dev), MINOR(pd->pkt_dev), 343 343 - MAJOR(file_bdev(pd->bdev_file)->bd_dev), 344 344 - MINOR(file_bdev(pd->bdev_file)->bd_dev)); 345 345 - } 346 346 - mutex_unlock(&ctl_mutex); 347 347 - return n; 348 348 - } 349 349 - static CLASS_ATTR_RO(device_map); 350 350 - 351 351 - static ssize_t add_store(const struct class *c, const struct class_attribute *attr, 352 352 - const char *buf, size_t count) 353 353 - { 354 354 - unsigned int major, minor; 355 355 - 356 356 - if (sscanf(buf, "%u:%u", &major, &minor) == 2) { 357 357 - /* pkt_setup_dev() expects caller to hold reference to self */ 358 358 - if (!try_module_get(THIS_MODULE)) 359 359 - return -ENODEV; 360 360 - 361 361 - pkt_setup_dev(MKDEV(major, minor), NULL); 362 362 - 363 363 - module_put(THIS_MODULE); 364 364 - 365 365 - return count; 366 366 - } 367 367 - 368 368 - return -EINVAL; 369 369 - } 370 370 - static CLASS_ATTR_WO(add); 371 371 - 372 372 - static ssize_t remove_store(const struct class *c, const struct class_attribute *attr, 373 373 - const char *buf, size_t count) 374 374 - { 375 375 - unsigned int major, minor; 376 376 - if (sscanf(buf, "%u:%u", &major, &minor) == 2) { 377 377 - pkt_remove_dev(MKDEV(major, minor)); 378 378 - return count; 379 379 - } 380 380 - return -EINVAL; 381 381 - } 382 382 - static CLASS_ATTR_WO(remove); 383 383 - 384 384 - static struct attribute *class_pktcdvd_attrs[] = { 385 385 - &class_attr_add.attr, 386 386 - &class_attr_remove.attr, 387 387 - &class_attr_device_map.attr, 388 388 - NULL, 389 389 - }; 390 390 - ATTRIBUTE_GROUPS(class_pktcdvd); 391 391 - 392 392 - static struct class class_pktcdvd = { 393 393 - .name = DRIVER_NAME, 394 394 - .class_groups = class_pktcdvd_groups, 395 395 - }; 396 396 - 397 397 - static int pkt_sysfs_init(void) 398 398 - { 399 399 - /* 400 400 - * create control files in sysfs 401 401 - * /sys/class/pktcdvd/... 402 402 - */ 403 403 - return class_register(&class_pktcdvd); 404 404 - } 405 405 - 406 406 - static void pkt_sysfs_cleanup(void) 407 407 - { 408 408 - class_unregister(&class_pktcdvd); 409 409 - } 410 410 - 411 411 - /******************************************************************** 412 412 - entries in debugfs 413 413 - 414 414 - /sys/kernel/debug/pktcdvd[0-7]/ 415 415 - info 416 416 - 417 417 - *******************************************************************/ 418 418 - 419 419 - static void pkt_count_states(struct pktcdvd_device *pd, int *states) 420 420 - { 421 421 - struct packet_data *pkt; 422 422 - int i; 423 423 - 424 424 - for (i = 0; i < PACKET_NUM_STATES; i++) 425 425 - states[i] = 0; 426 426 - 427 427 - spin_lock(&pd->cdrw.active_list_lock); 428 428 - list_for_each_entry(pkt, &pd->cdrw.pkt_active_list, list) { 429 429 - states[pkt->state]++; 430 430 - } 431 431 - spin_unlock(&pd->cdrw.active_list_lock); 432 432 - } 433 433 - 434 434 - static int pkt_seq_show(struct seq_file *m, void *p) 435 435 - { 436 436 - struct pktcdvd_device *pd = m->private; 437 437 - char *msg; 438 438 - int states[PACKET_NUM_STATES]; 439 439 - 440 440 - seq_printf(m, "Writer %s mapped to %pg:\n", pd->disk->disk_name, 441 441 - file_bdev(pd->bdev_file)); 442 442 - 443 443 - seq_printf(m, "\nSettings:\n"); 444 444 - seq_printf(m, "\tpacket size:\t\t%dkB\n", pd->settings.size / 2); 445 445 - 446 446 - if (pd->settings.write_type == 0) 447 447 - msg = "Packet"; 448 448 - else 449 449 - msg = "Unknown"; 450 450 - seq_printf(m, "\twrite type:\t\t%s\n", msg); 451 451 - 452 452 - seq_printf(m, "\tpacket type:\t\t%s\n", pd->settings.fp ? "Fixed" : "Variable"); 453 453 - seq_printf(m, "\tlink loss:\t\t%d\n", pd->settings.link_loss); 454 454 - 455 455 - seq_printf(m, "\ttrack mode:\t\t%d\n", pd->settings.track_mode); 456 456 - 457 457 - if (pd->settings.block_mode == PACKET_BLOCK_MODE1) 458 458 - msg = "Mode 1"; 459 459 - else if (pd->settings.block_mode == PACKET_BLOCK_MODE2) 460 460 - msg = "Mode 2"; 461 461 - else 462 462 - msg = "Unknown"; 463 463 - seq_printf(m, "\tblock mode:\t\t%s\n", msg); 464 464 - 465 465 - seq_printf(m, "\nStatistics:\n"); 466 466 - seq_printf(m, "\tpackets started:\t%lu\n", pd->stats.pkt_started); 467 467 - seq_printf(m, "\tpackets ended:\t\t%lu\n", pd->stats.pkt_ended); 468 468 - seq_printf(m, "\twritten:\t\t%lukB\n", pd->stats.secs_w >> 1); 469 469 - seq_printf(m, "\tread gather:\t\t%lukB\n", pd->stats.secs_rg >> 1); 470 470 - seq_printf(m, "\tread:\t\t\t%lukB\n", pd->stats.secs_r >> 1); 471 471 - 472 472 - seq_printf(m, "\nMisc:\n"); 473 473 - seq_printf(m, "\treference count:\t%d\n", pd->refcnt); 474 474 - seq_printf(m, "\tflags:\t\t\t0x%lx\n", pd->flags); 475 475 - seq_printf(m, "\tread speed:\t\t%ukB/s\n", pd->read_speed); 476 476 - seq_printf(m, "\twrite speed:\t\t%ukB/s\n", pd->write_speed); 477 477 - seq_printf(m, "\tstart offset:\t\t%lu\n", pd->offset); 478 478 - seq_printf(m, "\tmode page offset:\t%u\n", pd->mode_offset); 479 479 - 480 480 - seq_printf(m, "\nQueue state:\n"); 481 481 - seq_printf(m, "\tbios queued:\t\t%d\n", pd->bio_queue_size); 482 482 - seq_printf(m, "\tbios pending:\t\t%d\n", atomic_read(&pd->cdrw.pending_bios)); 483 483 - seq_printf(m, "\tcurrent sector:\t\t0x%llx\n", pd->current_sector); 484 484 - 485 485 - pkt_count_states(pd, states); 486 486 - seq_printf(m, "\tstate:\t\t\ti:%d ow:%d rw:%d ww:%d rec:%d fin:%d\n", 487 487 - states[0], states[1], states[2], states[3], states[4], states[5]); 488 488 - 489 489 - seq_printf(m, "\twrite congestion marks:\toff=%d on=%d\n", 490 490 - pd->write_congestion_off, 491 491 - pd->write_congestion_on); 492 492 - return 0; 493 493 - } 494 494 - DEFINE_SHOW_ATTRIBUTE(pkt_seq); 495 495 - 496 496 - static void pkt_debugfs_dev_new(struct pktcdvd_device *pd) 497 497 - { 498 498 - if (!pkt_debugfs_root) 499 499 - return; 500 500 - pd->dfs_d_root = debugfs_create_dir(pd->disk->disk_name, pkt_debugfs_root); 501 501 - 502 502 - pd->dfs_f_info = debugfs_create_file("info", 0444, pd->dfs_d_root, 503 503 - pd, &pkt_seq_fops); 504 504 - } 505 505 - 506 506 - static void pkt_debugfs_dev_remove(struct pktcdvd_device *pd) 507 507 - { 508 508 - if (!pkt_debugfs_root) 509 509 - return; 510 510 - debugfs_remove(pd->dfs_f_info); 511 511 - debugfs_remove(pd->dfs_d_root); 512 512 - pd->dfs_f_info = NULL; 513 513 - pd->dfs_d_root = NULL; 514 514 - } 515 515 - 516 516 - static void pkt_debugfs_init(void) 517 517 - { 518 518 - pkt_debugfs_root = debugfs_create_dir(DRIVER_NAME, NULL); 519 519 - } 520 520 - 521 521 - static void pkt_debugfs_cleanup(void) 522 522 - { 523 523 - debugfs_remove(pkt_debugfs_root); 524 524 - pkt_debugfs_root = NULL; 525 525 - } 526 526 - 527 527 - /* ----------------------------------------------------------*/ 528 528 - 529 529 - 530 530 - static void pkt_bio_finished(struct pktcdvd_device *pd) 531 531 - { 532 532 - struct device *ddev = disk_to_dev(pd->disk); 533 533 - 534 534 - BUG_ON(atomic_read(&pd->cdrw.pending_bios) <= 0); 535 535 - if (atomic_dec_and_test(&pd->cdrw.pending_bios)) { 536 536 - dev_dbg(ddev, "queue empty\n"); 537 537 - atomic_set(&pd->iosched.attention, 1); 538 538 - wake_up(&pd->wqueue); 539 539 - } 540 540 - } 541 541 - 542 542 - /* 543 543 - * Allocate a packet_data struct 544 544 - */ 545 545 - static struct packet_data *pkt_alloc_packet_data(int frames) 546 546 - { 547 547 - int i; 548 548 - struct packet_data *pkt; 549 549 - 550 550 - pkt = kzalloc(sizeof(struct packet_data), GFP_KERNEL); 551 551 - if (!pkt) 552 552 - goto no_pkt; 553 553 - 554 554 - pkt->frames = frames; 555 555 - pkt->w_bio = bio_kmalloc(frames, GFP_KERNEL); 556 556 - if (!pkt->w_bio) 557 557 - goto no_bio; 558 558 - 559 559 - for (i = 0; i < frames / FRAMES_PER_PAGE; i++) { 560 560 - pkt->pages[i] = alloc_page(GFP_KERNEL|__GFP_ZERO); 561 561 - if (!pkt->pages[i]) 562 562 - goto no_page; 563 563 - } 564 564 - 565 565 - spin_lock_init(&pkt->lock); 566 566 - bio_list_init(&pkt->orig_bios); 567 567 - 568 568 - for (i = 0; i < frames; i++) { 569 569 - pkt->r_bios[i] = bio_kmalloc(1, GFP_KERNEL); 570 570 - if (!pkt->r_bios[i]) 571 571 - goto no_rd_bio; 572 572 - } 573 573 - 574 574 - return pkt; 575 575 - 576 576 - no_rd_bio: 577 577 - for (i = 0; i < frames; i++) 578 578 - kfree(pkt->r_bios[i]); 579 579 - no_page: 580 580 - for (i = 0; i < frames / FRAMES_PER_PAGE; i++) 581 581 - if (pkt->pages[i]) 582 582 - __free_page(pkt->pages[i]); 583 583 - kfree(pkt->w_bio); 584 584 - no_bio: 585 585 - kfree(pkt); 586 586 - no_pkt: 587 587 - return NULL; 588 588 - } 589 589 - 590 590 - /* 591 591 - * Free a packet_data struct 592 592 - */ 593 593 - static void pkt_free_packet_data(struct packet_data *pkt) 594 594 - { 595 595 - int i; 596 596 - 597 597 - for (i = 0; i < pkt->frames; i++) 598 598 - kfree(pkt->r_bios[i]); 599 599 - for (i = 0; i < pkt->frames / FRAMES_PER_PAGE; i++) 600 600 - __free_page(pkt->pages[i]); 601 601 - kfree(pkt->w_bio); 602 602 - kfree(pkt); 603 603 - } 604 604 - 605 605 - static void pkt_shrink_pktlist(struct pktcdvd_device *pd) 606 606 - { 607 607 - struct packet_data *pkt, *next; 608 608 - 609 609 - BUG_ON(!list_empty(&pd->cdrw.pkt_active_list)); 610 610 - 611 611 - list_for_each_entry_safe(pkt, next, &pd->cdrw.pkt_free_list, list) { 612 612 - pkt_free_packet_data(pkt); 613 613 - } 614 614 - INIT_LIST_HEAD(&pd->cdrw.pkt_free_list); 615 615 - } 616 616 - 617 617 - static int pkt_grow_pktlist(struct pktcdvd_device *pd, int nr_packets) 618 618 - { 619 619 - struct packet_data *pkt; 620 620 - 621 621 - BUG_ON(!list_empty(&pd->cdrw.pkt_free_list)); 622 622 - 623 623 - while (nr_packets > 0) { 624 624 - pkt = pkt_alloc_packet_data(pd->settings.size >> 2); 625 625 - if (!pkt) { 626 626 - pkt_shrink_pktlist(pd); 627 627 - return 0; 628 628 - } 629 629 - pkt->id = nr_packets; 630 630 - pkt->pd = pd; 631 631 - list_add(&pkt->list, &pd->cdrw.pkt_free_list); 632 632 - nr_packets--; 633 633 - } 634 634 - return 1; 635 635 - } 636 636 - 637 637 - static inline struct pkt_rb_node *pkt_rbtree_next(struct pkt_rb_node *node) 638 638 - { 639 639 - struct rb_node *n = rb_next(&node->rb_node); 640 640 - if (!n) 641 641 - return NULL; 642 642 - return rb_entry(n, struct pkt_rb_node, rb_node); 643 643 - } 644 644 - 645 645 - static void pkt_rbtree_erase(struct pktcdvd_device *pd, struct pkt_rb_node *node) 646 646 - { 647 647 - rb_erase(&node->rb_node, &pd->bio_queue); 648 648 - mempool_free(node, &pd->rb_pool); 649 649 - pd->bio_queue_size--; 650 650 - BUG_ON(pd->bio_queue_size < 0); 651 651 - } 652 652 - 653 653 - /* 654 654 - * Find the first node in the pd->bio_queue rb tree with a starting sector >= s. 655 655 - */ 656 656 - static struct pkt_rb_node *pkt_rbtree_find(struct pktcdvd_device *pd, sector_t s) 657 657 - { 658 658 - struct rb_node *n = pd->bio_queue.rb_node; 659 659 - struct rb_node *next; 660 660 - struct pkt_rb_node *tmp; 661 661 - 662 662 - if (!n) { 663 663 - BUG_ON(pd->bio_queue_size > 0); 664 664 - return NULL; 665 665 - } 666 666 - 667 667 - for (;;) { 668 668 - tmp = rb_entry(n, struct pkt_rb_node, rb_node); 669 669 - if (s <= tmp->bio->bi_iter.bi_sector) 670 670 - next = n->rb_left; 671 671 - else 672 672 - next = n->rb_right; 673 673 - if (!next) 674 674 - break; 675 675 - n = next; 676 676 - } 677 677 - 678 678 - if (s > tmp->bio->bi_iter.bi_sector) { 679 679 - tmp = pkt_rbtree_next(tmp); 680 680 - if (!tmp) 681 681 - return NULL; 682 682 - } 683 683 - BUG_ON(s > tmp->bio->bi_iter.bi_sector); 684 684 - return tmp; 685 685 - } 686 686 - 687 687 - /* 688 688 - * Insert a node into the pd->bio_queue rb tree. 689 689 - */ 690 690 - static void pkt_rbtree_insert(struct pktcdvd_device *pd, struct pkt_rb_node *node) 691 691 - { 692 692 - struct rb_node **p = &pd->bio_queue.rb_node; 693 693 - struct rb_node *parent = NULL; 694 694 - sector_t s = node->bio->bi_iter.bi_sector; 695 695 - struct pkt_rb_node *tmp; 696 696 - 697 697 - while (*p) { 698 698 - parent = *p; 699 699 - tmp = rb_entry(parent, struct pkt_rb_node, rb_node); 700 700 - if (s < tmp->bio->bi_iter.bi_sector) 701 701 - p = &(*p)->rb_left; 702 702 - else 703 703 - p = &(*p)->rb_right; 704 704 - } 705 705 - rb_link_node(&node->rb_node, parent, p); 706 706 - rb_insert_color(&node->rb_node, &pd->bio_queue); 707 707 - pd->bio_queue_size++; 708 708 - } 709 709 - 710 710 - /* 711 711 - * Send a packet_command to the underlying block device and 712 712 - * wait for completion. 713 713 - */ 714 714 - static int pkt_generic_packet(struct pktcdvd_device *pd, struct packet_command *cgc) 715 715 - { 716 716 - struct request_queue *q = bdev_get_queue(file_bdev(pd->bdev_file)); 717 717 - struct scsi_cmnd *scmd; 718 718 - struct request *rq; 719 719 - int ret = 0; 720 720 - 721 721 - rq = scsi_alloc_request(q, (cgc->data_direction == CGC_DATA_WRITE) ? 722 722 - REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0); 723 723 - if (IS_ERR(rq)) 724 724 - return PTR_ERR(rq); 725 725 - scmd = blk_mq_rq_to_pdu(rq); 726 726 - 727 727 - if (cgc->buflen) { 728 728 - ret = blk_rq_map_kern(rq, cgc->buffer, cgc->buflen, 729 729 - GFP_NOIO); 730 730 - if (ret) 731 731 - goto out; 732 732 - } 733 733 - 734 734 - scmd->cmd_len = COMMAND_SIZE(cgc->cmd[0]); 735 735 - memcpy(scmd->cmnd, cgc->cmd, CDROM_PACKET_SIZE); 736 736 - 737 737 - rq->timeout = 60*HZ; 738 738 - if (cgc->quiet) 739 739 - rq->rq_flags |= RQF_QUIET; 740 740 - 741 741 - blk_execute_rq(rq, false); 742 742 - if (scmd->result) 743 743 - ret = -EIO; 744 744 - out: 745 745 - blk_mq_free_request(rq); 746 746 - return ret; 747 747 - } 748 748 - 749 749 - static const char *sense_key_string(__u8 index) 750 750 - { 751 751 - static const char * const info[] = { 752 752 - "No sense", "Recovered error", "Not ready", 753 753 - "Medium error", "Hardware error", "Illegal request", 754 754 - "Unit attention", "Data protect", "Blank check", 755 755 - }; 756 756 - 757 757 - return index < ARRAY_SIZE(info) ? info[index] : "INVALID"; 758 758 - } 759 759 - 760 760 - /* 761 761 - * A generic sense dump / resolve mechanism should be implemented across 762 762 - * all ATAPI + SCSI devices. 763 763 - */ 764 764 - static void pkt_dump_sense(struct pktcdvd_device *pd, 765 765 - struct packet_command *cgc) 766 766 - { 767 767 - struct device *ddev = disk_to_dev(pd->disk); 768 768 - struct scsi_sense_hdr *sshdr = cgc->sshdr; 769 769 - 770 770 - if (sshdr) 771 771 - dev_err(ddev, "%*ph - sense %02x.%02x.%02x (%s)\n", 772 772 - CDROM_PACKET_SIZE, cgc->cmd, 773 773 - sshdr->sense_key, sshdr->asc, sshdr->ascq, 774 774 - sense_key_string(sshdr->sense_key)); 775 775 - else 776 776 - dev_err(ddev, "%*ph - no sense\n", CDROM_PACKET_SIZE, cgc->cmd); 777 777 - } 778 778 - 779 779 - /* 780 780 - * flush the drive cache to media 781 781 - */ 782 782 - static int pkt_flush_cache(struct pktcdvd_device *pd) 783 783 - { 784 784 - struct packet_command cgc; 785 785 - 786 786 - init_cdrom_command(&cgc, NULL, 0, CGC_DATA_NONE); 787 787 - cgc.cmd[0] = GPCMD_FLUSH_CACHE; 788 788 - cgc.quiet = 1; 789 789 - 790 790 - /* 791 791 - * the IMMED bit -- we default to not setting it, although that 792 792 - * would allow a much faster close, this is safer 793 793 - */ 794 794 - #if 0 795 795 - cgc.cmd[1] = 1 << 1; 796 796 - #endif 797 797 - return pkt_generic_packet(pd, &cgc); 798 798 - } 799 799 - 800 800 - /* 801 801 - * speed is given as the normal factor, e.g. 4 for 4x 802 802 - */ 803 803 - static noinline_for_stack int pkt_set_speed(struct pktcdvd_device *pd, 804 804 - unsigned write_speed, unsigned read_speed) 805 805 - { 806 806 - struct packet_command cgc; 807 807 - struct scsi_sense_hdr sshdr; 808 808 - int ret; 809 809 - 810 810 - init_cdrom_command(&cgc, NULL, 0, CGC_DATA_NONE); 811 811 - cgc.sshdr = &sshdr; 812 812 - cgc.cmd[0] = GPCMD_SET_SPEED; 813 813 - put_unaligned_be16(read_speed, &cgc.cmd[2]); 814 814 - put_unaligned_be16(write_speed, &cgc.cmd[4]); 815 815 - 816 816 - ret = pkt_generic_packet(pd, &cgc); 817 817 - if (ret) 818 818 - pkt_dump_sense(pd, &cgc); 819 819 - 820 820 - return ret; 821 821 - } 822 822 - 823 823 - /* 824 824 - * Queue a bio for processing by the low-level CD device. Must be called 825 825 - * from process context. 826 826 - */ 827 827 - static void pkt_queue_bio(struct pktcdvd_device *pd, struct bio *bio) 828 828 - { 829 829 - /* 830 830 - * Some CDRW drives can not handle writes larger than one packet, 831 831 - * even if the size is a multiple of the packet size. 832 832 - */ 833 833 - bio->bi_opf |= REQ_NOMERGE; 834 834 - 835 835 - spin_lock(&pd->iosched.lock); 836 836 - if (bio_data_dir(bio) == READ) 837 837 - bio_list_add(&pd->iosched.read_queue, bio); 838 838 - else 839 839 - bio_list_add(&pd->iosched.write_queue, bio); 840 840 - spin_unlock(&pd->iosched.lock); 841 841 - 842 842 - atomic_set(&pd->iosched.attention, 1); 843 843 - wake_up(&pd->wqueue); 844 844 - } 845 845 - 846 846 - /* 847 847 - * Process the queued read/write requests. This function handles special 848 848 - * requirements for CDRW drives: 849 849 - * - A cache flush command must be inserted before a read request if the 850 850 - * previous request was a write. 851 851 - * - Switching between reading and writing is slow, so don't do it more often 852 852 - * than necessary. 853 853 - * - Optimize for throughput at the expense of latency. This means that streaming 854 854 - * writes will never be interrupted by a read, but if the drive has to seek 855 855 - * before the next write, switch to reading instead if there are any pending 856 856 - * read requests. 857 857 - * - Set the read speed according to current usage pattern. When only reading 858 858 - * from the device, it's best to use the highest possible read speed, but 859 859 - * when switching often between reading and writing, it's better to have the 860 860 - * same read and write speeds. 861 861 - */ 862 862 - static void pkt_iosched_process_queue(struct pktcdvd_device *pd) 863 863 - { 864 864 - struct device *ddev = disk_to_dev(pd->disk); 865 865 - 866 866 - if (atomic_read(&pd->iosched.attention) == 0) 867 867 - return; 868 868 - atomic_set(&pd->iosched.attention, 0); 869 869 - 870 870 - for (;;) { 871 871 - struct bio *bio; 872 872 - int reads_queued, writes_queued; 873 873 - 874 874 - spin_lock(&pd->iosched.lock); 875 875 - reads_queued = !bio_list_empty(&pd->iosched.read_queue); 876 876 - writes_queued = !bio_list_empty(&pd->iosched.write_queue); 877 877 - spin_unlock(&pd->iosched.lock); 878 878 - 879 879 - if (!reads_queued && !writes_queued) 880 880 - break; 881 881 - 882 882 - if (pd->iosched.writing) { 883 883 - int need_write_seek = 1; 884 884 - spin_lock(&pd->iosched.lock); 885 885 - bio = bio_list_peek(&pd->iosched.write_queue); 886 886 - spin_unlock(&pd->iosched.lock); 887 887 - if (bio && (bio->bi_iter.bi_sector == 888 888 - pd->iosched.last_write)) 889 889 - need_write_seek = 0; 890 890 - if (need_write_seek && reads_queued) { 891 891 - if (atomic_read(&pd->cdrw.pending_bios) > 0) { 892 892 - dev_dbg(ddev, "write, waiting\n"); 893 893 - break; 894 894 - } 895 895 - pkt_flush_cache(pd); 896 896 - pd->iosched.writing = 0; 897 897 - } 898 898 - } else { 899 899 - if (!reads_queued && writes_queued) { 900 900 - if (atomic_read(&pd->cdrw.pending_bios) > 0) { 901 901 - dev_dbg(ddev, "read, waiting\n"); 902 902 - break; 903 903 - } 904 904 - pd->iosched.writing = 1; 905 905 - } 906 906 - } 907 907 - 908 908 - spin_lock(&pd->iosched.lock); 909 909 - if (pd->iosched.writing) 910 910 - bio = bio_list_pop(&pd->iosched.write_queue); 911 911 - else 912 912 - bio = bio_list_pop(&pd->iosched.read_queue); 913 913 - spin_unlock(&pd->iosched.lock); 914 914 - 915 915 - if (!bio) 916 916 - continue; 917 917 - 918 918 - if (bio_data_dir(bio) == READ) 919 919 - pd->iosched.successive_reads += 920 920 - bio->bi_iter.bi_size >> 10; 921 921 - else { 922 922 - pd->iosched.successive_reads = 0; 923 923 - pd->iosched.last_write = bio_end_sector(bio); 924 924 - } 925 925 - if (pd->iosched.successive_reads >= HI_SPEED_SWITCH) { 926 926 - if (pd->read_speed == pd->write_speed) { 927 927 - pd->read_speed = MAX_SPEED; 928 928 - pkt_set_speed(pd, pd->write_speed, pd->read_speed); 929 929 - } 930 930 - } else { 931 931 - if (pd->read_speed != pd->write_speed) { 932 932 - pd->read_speed = pd->write_speed; 933 933 - pkt_set_speed(pd, pd->write_speed, pd->read_speed); 934 934 - } 935 935 - } 936 936 - 937 937 - atomic_inc(&pd->cdrw.pending_bios); 938 938 - submit_bio_noacct(bio); 939 939 - } 940 940 - } 941 941 - 942 942 - /* 943 943 - * Special care is needed if the underlying block device has a small 944 944 - * max_phys_segments value. 945 945 - */ 946 946 - static int pkt_set_segment_merging(struct pktcdvd_device *pd, struct request_queue *q) 947 947 - { 948 948 - struct device *ddev = disk_to_dev(pd->disk); 949 949 - 950 950 - if ((pd->settings.size << 9) / CD_FRAMESIZE <= queue_max_segments(q)) { 951 951 - /* 952 952 - * The cdrom device can handle one segment/frame 953 953 - */ 954 954 - clear_bit(PACKET_MERGE_SEGS, &pd->flags); 955 955 - return 0; 956 956 - } 957 957 - 958 958 - if ((pd->settings.size << 9) / PAGE_SIZE <= queue_max_segments(q)) { 959 959 - /* 960 960 - * We can handle this case at the expense of some extra memory 961 961 - * copies during write operations 962 962 - */ 963 963 - set_bit(PACKET_MERGE_SEGS, &pd->flags); 964 964 - return 0; 965 965 - } 966 966 - 967 967 - dev_err(ddev, "cdrom max_phys_segments too small\n"); 968 968 - return -EIO; 969 969 - } 970 970 - 971 971 - static void pkt_end_io_read(struct bio *bio) 972 972 - { 973 973 - struct packet_data *pkt = bio->bi_private; 974 974 - struct pktcdvd_device *pd = pkt->pd; 975 975 - BUG_ON(!pd); 976 976 - 977 977 - dev_dbg(disk_to_dev(pd->disk), "bio=%p sec0=%llx sec=%llx err=%d\n", 978 978 - bio, pkt->sector, bio->bi_iter.bi_sector, bio->bi_status); 979 979 - 980 980 - if (bio->bi_status) 981 981 - atomic_inc(&pkt->io_errors); 982 982 - bio_uninit(bio); 983 983 - if (atomic_dec_and_test(&pkt->io_wait)) { 984 984 - atomic_inc(&pkt->run_sm); 985 985 - wake_up(&pd->wqueue); 986 986 - } 987 987 - pkt_bio_finished(pd); 988 988 - } 989 989 - 990 990 - static void pkt_end_io_packet_write(struct bio *bio) 991 991 - { 992 992 - struct packet_data *pkt = bio->bi_private; 993 993 - struct pktcdvd_device *pd = pkt->pd; 994 994 - BUG_ON(!pd); 995 995 - 996 996 - dev_dbg(disk_to_dev(pd->disk), "id=%d, err=%d\n", pkt->id, bio->bi_status); 997 997 - 998 998 - pd->stats.pkt_ended++; 999 999 - 1000 1000 - bio_uninit(bio); 1001 1001 - pkt_bio_finished(pd); 1002 1002 - atomic_dec(&pkt->io_wait); 1003 1003 - atomic_inc(&pkt->run_sm); 1004 1004 - wake_up(&pd->wqueue); 1005 1005 - } 1006 1006 - 1007 1007 - /* 1008 1008 - * Schedule reads for the holes in a packet 1009 1009 - */ 1010 1010 - static void pkt_gather_data(struct pktcdvd_device *pd, struct packet_data *pkt) 1011 1011 - { 1012 1012 - struct device *ddev = disk_to_dev(pd->disk); 1013 1013 - int frames_read = 0; 1014 1014 - struct bio *bio; 1015 1015 - int f; 1016 1016 - char written[PACKET_MAX_SIZE]; 1017 1017 - 1018 1018 - BUG_ON(bio_list_empty(&pkt->orig_bios)); 1019 1019 - 1020 1020 - atomic_set(&pkt->io_wait, 0); 1021 1021 - atomic_set(&pkt->io_errors, 0); 1022 1022 - 1023 1023 - /* 1024 1024 - * Figure out which frames we need to read before we can write. 1025 1025 - */ 1026 1026 - memset(written, 0, sizeof(written)); 1027 1027 - spin_lock(&pkt->lock); 1028 1028 - bio_list_for_each(bio, &pkt->orig_bios) { 1029 1029 - int first_frame = (bio->bi_iter.bi_sector - pkt->sector) / 1030 1030 - (CD_FRAMESIZE >> 9); 1031 1031 - int num_frames = bio->bi_iter.bi_size / CD_FRAMESIZE; 1032 1032 - pd->stats.secs_w += num_frames * (CD_FRAMESIZE >> 9); 1033 1033 - BUG_ON(first_frame < 0); 1034 1034 - BUG_ON(first_frame + num_frames > pkt->frames); 1035 1035 - for (f = first_frame; f < first_frame + num_frames; f++) 1036 1036 - written[f] = 1; 1037 1037 - } 1038 1038 - spin_unlock(&pkt->lock); 1039 1039 - 1040 1040 - if (pkt->cache_valid) { 1041 1041 - dev_dbg(ddev, "zone %llx cached\n", pkt->sector); 1042 1042 - goto out_account; 1043 1043 - } 1044 1044 - 1045 1045 - /* 1046 1046 - * Schedule reads for missing parts of the packet. 1047 1047 - */ 1048 1048 - for (f = 0; f < pkt->frames; f++) { 1049 1049 - int p, offset; 1050 1050 - 1051 1051 - if (written[f]) 1052 1052 - continue; 1053 1053 - 1054 1054 - bio = pkt->r_bios[f]; 1055 1055 - bio_init(bio, file_bdev(pd->bdev_file), bio->bi_inline_vecs, 1, 1056 1056 - REQ_OP_READ); 1057 1057 - bio->bi_iter.bi_sector = pkt->sector + f * (CD_FRAMESIZE >> 9); 1058 1058 - bio->bi_end_io = pkt_end_io_read; 1059 1059 - bio->bi_private = pkt; 1060 1060 - 1061 1061 - p = (f * CD_FRAMESIZE) / PAGE_SIZE; 1062 1062 - offset = (f * CD_FRAMESIZE) % PAGE_SIZE; 1063 1063 - dev_dbg(ddev, "Adding frame %d, page:%p offs:%d\n", f, 1064 1064 - pkt->pages[p], offset); 1065 1065 - if (!bio_add_page(bio, pkt->pages[p], CD_FRAMESIZE, offset)) 1066 1066 - BUG(); 1067 1067 - 1068 1068 - atomic_inc(&pkt->io_wait); 1069 1069 - pkt_queue_bio(pd, bio); 1070 1070 - frames_read++; 1071 1071 - } 1072 1072 - 1073 1073 - out_account: 1074 1074 - dev_dbg(ddev, "need %d frames for zone %llx\n", frames_read, pkt->sector); 1075 1075 - pd->stats.pkt_started++; 1076 1076 - pd->stats.secs_rg += frames_read * (CD_FRAMESIZE >> 9); 1077 1077 - } 1078 1078 - 1079 1079 - /* 1080 1080 - * Find a packet matching zone, or the least recently used packet if 1081 1081 - * there is no match. 1082 1082 - */ 1083 1083 - static struct packet_data *pkt_get_packet_data(struct pktcdvd_device *pd, int zone) 1084 1084 - { 1085 1085 - struct packet_data *pkt; 1086 1086 - 1087 1087 - list_for_each_entry(pkt, &pd->cdrw.pkt_free_list, list) { 1088 1088 - if (pkt->sector == zone || pkt->list.next == &pd->cdrw.pkt_free_list) { 1089 1089 - list_del_init(&pkt->list); 1090 1090 - if (pkt->sector != zone) 1091 1091 - pkt->cache_valid = 0; 1092 1092 - return pkt; 1093 1093 - } 1094 1094 - } 1095 1095 - BUG(); 1096 1096 - return NULL; 1097 1097 - } 1098 1098 - 1099 1099 - static void pkt_put_packet_data(struct pktcdvd_device *pd, struct packet_data *pkt) 1100 1100 - { 1101 1101 - if (pkt->cache_valid) { 1102 1102 - list_add(&pkt->list, &pd->cdrw.pkt_free_list); 1103 1103 - } else { 1104 1104 - list_add_tail(&pkt->list, &pd->cdrw.pkt_free_list); 1105 1105 - } 1106 1106 - } 1107 1107 - 1108 1108 - static inline void pkt_set_state(struct device *ddev, struct packet_data *pkt, 1109 1109 - enum packet_data_state state) 1110 1110 - { 1111 1111 - static const char *state_name[] = { 1112 1112 - "IDLE", "WAITING", "READ_WAIT", "WRITE_WAIT", "RECOVERY", "FINISHED" 1113 1113 - }; 1114 1114 - enum packet_data_state old_state = pkt->state; 1115 1115 - 1116 1116 - dev_dbg(ddev, "pkt %2d : s=%6llx %s -> %s\n", 1117 1117 - pkt->id, pkt->sector, state_name[old_state], state_name[state]); 1118 1118 - 1119 1119 - pkt->state = state; 1120 1120 - } 1121 1121 - 1122 1122 - /* 1123 1123 - * Scan the work queue to see if we can start a new packet. 1124 1124 - * returns non-zero if any work was done. 1125 1125 - */ 1126 1126 - static int pkt_handle_queue(struct pktcdvd_device *pd) 1127 1127 - { 1128 1128 - struct device *ddev = disk_to_dev(pd->disk); 1129 1129 - struct packet_data *pkt, *p; 1130 1130 - struct bio *bio = NULL; 1131 1131 - sector_t zone = 0; /* Suppress gcc warning */ 1132 1132 - struct pkt_rb_node *node, *first_node; 1133 1133 - struct rb_node *n; 1134 1134 - 1135 1135 - atomic_set(&pd->scan_queue, 0); 1136 1136 - 1137 1137 - if (list_empty(&pd->cdrw.pkt_free_list)) { 1138 1138 - dev_dbg(ddev, "no pkt\n"); 1139 1139 - return 0; 1140 1140 - } 1141 1141 - 1142 1142 - /* 1143 1143 - * Try to find a zone we are not already working on. 1144 1144 - */ 1145 1145 - spin_lock(&pd->lock); 1146 1146 - first_node = pkt_rbtree_find(pd, pd->current_sector); 1147 1147 - if (!first_node) { 1148 1148 - n = rb_first(&pd->bio_queue); 1149 1149 - if (n) 1150 1150 - first_node = rb_entry(n, struct pkt_rb_node, rb_node); 1151 1151 - } 1152 1152 - node = first_node; 1153 1153 - while (node) { 1154 1154 - bio = node->bio; 1155 1155 - zone = get_zone(bio->bi_iter.bi_sector, pd); 1156 1156 - list_for_each_entry(p, &pd->cdrw.pkt_active_list, list) { 1157 1157 - if (p->sector == zone) { 1158 1158 - bio = NULL; 1159 1159 - goto try_next_bio; 1160 1160 - } 1161 1161 - } 1162 1162 - break; 1163 1163 - try_next_bio: 1164 1164 - node = pkt_rbtree_next(node); 1165 1165 - if (!node) { 1166 1166 - n = rb_first(&pd->bio_queue); 1167 1167 - if (n) 1168 1168 - node = rb_entry(n, struct pkt_rb_node, rb_node); 1169 1169 - } 1170 1170 - if (node == first_node) 1171 1171 - node = NULL; 1172 1172 - } 1173 1173 - spin_unlock(&pd->lock); 1174 1174 - if (!bio) { 1175 1175 - dev_dbg(ddev, "no bio\n"); 1176 1176 - return 0; 1177 1177 - } 1178 1178 - 1179 1179 - pkt = pkt_get_packet_data(pd, zone); 1180 1180 - 1181 1181 - pd->current_sector = zone + pd->settings.size; 1182 1182 - pkt->sector = zone; 1183 1183 - BUG_ON(pkt->frames != pd->settings.size >> 2); 1184 1184 - pkt->write_size = 0; 1185 1185 - 1186 1186 - /* 1187 1187 - * Scan work queue for bios in the same zone and link them 1188 1188 - * to this packet. 1189 1189 - */ 1190 1190 - spin_lock(&pd->lock); 1191 1191 - dev_dbg(ddev, "looking for zone %llx\n", zone); 1192 1192 - while ((node = pkt_rbtree_find(pd, zone)) != NULL) { 1193 1193 - sector_t tmp = get_zone(node->bio->bi_iter.bi_sector, pd); 1194 1194 - 1195 1195 - bio = node->bio; 1196 1196 - dev_dbg(ddev, "found zone=%llx\n", tmp); 1197 1197 - if (tmp != zone) 1198 1198 - break; 1199 1199 - pkt_rbtree_erase(pd, node); 1200 1200 - spin_lock(&pkt->lock); 1201 1201 - bio_list_add(&pkt->orig_bios, bio); 1202 1202 - pkt->write_size += bio->bi_iter.bi_size / CD_FRAMESIZE; 1203 1203 - spin_unlock(&pkt->lock); 1204 1204 - } 1205 1205 - /* check write congestion marks, and if bio_queue_size is 1206 1206 - * below, wake up any waiters 1207 1207 - */ 1208 1208 - if (pd->congested && 1209 1209 - pd->bio_queue_size <= pd->write_congestion_off) { 1210 1210 - pd->congested = false; 1211 1211 - wake_up_var(&pd->congested); 1212 1212 - } 1213 1213 - spin_unlock(&pd->lock); 1214 1214 - 1215 1215 - pkt->sleep_time = max(PACKET_WAIT_TIME, 1); 1216 1216 - pkt_set_state(ddev, pkt, PACKET_WAITING_STATE); 1217 1217 - atomic_set(&pkt->run_sm, 1); 1218 1218 - 1219 1219 - spin_lock(&pd->cdrw.active_list_lock); 1220 1220 - list_add(&pkt->list, &pd->cdrw.pkt_active_list); 1221 1221 - spin_unlock(&pd->cdrw.active_list_lock); 1222 1222 - 1223 1223 - return 1; 1224 1224 - } 1225 1225 - 1226 1226 - /** 1227 1227 - * bio_list_copy_data - copy contents of data buffers from one chain of bios to 1228 1228 - * another 1229 1229 - * @src: source bio list 1230 1230 - * @dst: destination bio list 1231 1231 - * 1232 1232 - * Stops when it reaches the end of either the @src list or @dst list - that is, 1233 1233 - * copies min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of 1234 1234 - * bios). 1235 1235 - */ 1236 1236 - static void bio_list_copy_data(struct bio *dst, struct bio *src) 1237 1237 - { 1238 1238 - struct bvec_iter src_iter = src->bi_iter; 1239 1239 - struct bvec_iter dst_iter = dst->bi_iter; 1240 1240 - 1241 1241 - while (1) { 1242 1242 - if (!src_iter.bi_size) { 1243 1243 - src = src->bi_next; 1244 1244 - if (!src) 1245 1245 - break; 1246 1246 - 1247 1247 - src_iter = src->bi_iter; 1248 1248 - } 1249 1249 - 1250 1250 - if (!dst_iter.bi_size) { 1251 1251 - dst = dst->bi_next; 1252 1252 - if (!dst) 1253 1253 - break; 1254 1254 - 1255 1255 - dst_iter = dst->bi_iter; 1256 1256 - } 1257 1257 - 1258 1258 - bio_copy_data_iter(dst, &dst_iter, src, &src_iter); 1259 1259 - } 1260 1260 - } 1261 1261 - 1262 1262 - /* 1263 1263 - * Assemble a bio to write one packet and queue the bio for processing 1264 1264 - * by the underlying block device. 1265 1265 - */ 1266 1266 - static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt) 1267 1267 - { 1268 1268 - struct device *ddev = disk_to_dev(pd->disk); 1269 1269 - int f; 1270 1270 - 1271 1271 - bio_init(pkt->w_bio, file_bdev(pd->bdev_file), pkt->w_bio->bi_inline_vecs, 1272 1272 - pkt->frames, REQ_OP_WRITE); 1273 1273 - pkt->w_bio->bi_iter.bi_sector = pkt->sector; 1274 1274 - pkt->w_bio->bi_end_io = pkt_end_io_packet_write; 1275 1275 - pkt->w_bio->bi_private = pkt; 1276 1276 - 1277 1277 - /* XXX: locking? */ 1278 1278 - for (f = 0; f < pkt->frames; f++) { 1279 1279 - struct page *page = pkt->pages[(f * CD_FRAMESIZE) / PAGE_SIZE]; 1280 1280 - unsigned offset = (f * CD_FRAMESIZE) % PAGE_SIZE; 1281 1281 - 1282 1282 - if (!bio_add_page(pkt->w_bio, page, CD_FRAMESIZE, offset)) 1283 1283 - BUG(); 1284 1284 - } 1285 1285 - dev_dbg(ddev, "vcnt=%d\n", pkt->w_bio->bi_vcnt); 1286 1286 - 1287 1287 - /* 1288 1288 - * Fill-in bvec with data from orig_bios. 1289 1289 - */ 1290 1290 - spin_lock(&pkt->lock); 1291 1291 - bio_list_copy_data(pkt->w_bio, pkt->orig_bios.head); 1292 1292 - 1293 1293 - pkt_set_state(ddev, pkt, PACKET_WRITE_WAIT_STATE); 1294 1294 - spin_unlock(&pkt->lock); 1295 1295 - 1296 1296 - dev_dbg(ddev, "Writing %d frames for zone %llx\n", pkt->write_size, pkt->sector); 1297 1297 - 1298 1298 - if (test_bit(PACKET_MERGE_SEGS, &pd->flags) || (pkt->write_size < pkt->frames)) 1299 1299 - pkt->cache_valid = 1; 1300 1300 - else 1301 1301 - pkt->cache_valid = 0; 1302 1302 - 1303 1303 - /* Start the write request */ 1304 1304 - atomic_set(&pkt->io_wait, 1); 1305 1305 - pkt_queue_bio(pd, pkt->w_bio); 1306 1306 - } 1307 1307 - 1308 1308 - static void pkt_finish_packet(struct packet_data *pkt, blk_status_t status) 1309 1309 - { 1310 1310 - struct bio *bio; 1311 1311 - 1312 1312 - if (status) 1313 1313 - pkt->cache_valid = 0; 1314 1314 - 1315 1315 - /* Finish all bios corresponding to this packet */ 1316 1316 - while ((bio = bio_list_pop(&pkt->orig_bios))) { 1317 1317 - bio->bi_status = status; 1318 1318 - bio_endio(bio); 1319 1319 - } 1320 1320 - } 1321 1321 - 1322 1322 - static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data *pkt) 1323 1323 - { 1324 1324 - struct device *ddev = disk_to_dev(pd->disk); 1325 1325 - 1326 1326 - dev_dbg(ddev, "pkt %d\n", pkt->id); 1327 1327 - 1328 1328 - for (;;) { 1329 1329 - switch (pkt->state) { 1330 1330 - case PACKET_WAITING_STATE: 1331 1331 - if ((pkt->write_size < pkt->frames) && (pkt->sleep_time > 0)) 1332 1332 - return; 1333 1333 - 1334 1334 - pkt->sleep_time = 0; 1335 1335 - pkt_gather_data(pd, pkt); 1336 1336 - pkt_set_state(ddev, pkt, PACKET_READ_WAIT_STATE); 1337 1337 - break; 1338 1338 - 1339 1339 - case PACKET_READ_WAIT_STATE: 1340 1340 - if (atomic_read(&pkt->io_wait) > 0) 1341 1341 - return; 1342 1342 - 1343 1343 - if (atomic_read(&pkt->io_errors) > 0) { 1344 1344 - pkt_set_state(ddev, pkt, PACKET_RECOVERY_STATE); 1345 1345 - } else { 1346 1346 - pkt_start_write(pd, pkt); 1347 1347 - } 1348 1348 - break; 1349 1349 - 1350 1350 - case PACKET_WRITE_WAIT_STATE: 1351 1351 - if (atomic_read(&pkt->io_wait) > 0) 1352 1352 - return; 1353 1353 - 1354 1354 - if (!pkt->w_bio->bi_status) { 1355 1355 - pkt_set_state(ddev, pkt, PACKET_FINISHED_STATE); 1356 1356 - } else { 1357 1357 - pkt_set_state(ddev, pkt, PACKET_RECOVERY_STATE); 1358 1358 - } 1359 1359 - break; 1360 1360 - 1361 1361 - case PACKET_RECOVERY_STATE: 1362 1362 - dev_dbg(ddev, "No recovery possible\n"); 1363 1363 - pkt_set_state(ddev, pkt, PACKET_FINISHED_STATE); 1364 1364 - break; 1365 1365 - 1366 1366 - case PACKET_FINISHED_STATE: 1367 1367 - pkt_finish_packet(pkt, pkt->w_bio->bi_status); 1368 1368 - return; 1369 1369 - 1370 1370 - default: 1371 1371 - BUG(); 1372 1372 - break; 1373 1373 - } 1374 1374 - } 1375 1375 - } 1376 1376 - 1377 1377 - static void pkt_handle_packets(struct pktcdvd_device *pd) 1378 1378 - { 1379 1379 - struct device *ddev = disk_to_dev(pd->disk); 1380 1380 - struct packet_data *pkt, *next; 1381 1381 - 1382 1382 - /* 1383 1383 - * Run state machine for active packets 1384 1384 - */ 1385 1385 - list_for_each_entry(pkt, &pd->cdrw.pkt_active_list, list) { 1386 1386 - if (atomic_read(&pkt->run_sm) > 0) { 1387 1387 - atomic_set(&pkt->run_sm, 0); 1388 1388 - pkt_run_state_machine(pd, pkt); 1389 1389 - } 1390 1390 - } 1391 1391 - 1392 1392 - /* 1393 1393 - * Move no longer active packets to the free list 1394 1394 - */ 1395 1395 - spin_lock(&pd->cdrw.active_list_lock); 1396 1396 - list_for_each_entry_safe(pkt, next, &pd->cdrw.pkt_active_list, list) { 1397 1397 - if (pkt->state == PACKET_FINISHED_STATE) { 1398 1398 - list_del(&pkt->list); 1399 1399 - pkt_put_packet_data(pd, pkt); 1400 1400 - pkt_set_state(ddev, pkt, PACKET_IDLE_STATE); 1401 1401 - atomic_set(&pd->scan_queue, 1); 1402 1402 - } 1403 1403 - } 1404 1404 - spin_unlock(&pd->cdrw.active_list_lock); 1405 1405 - } 1406 1406 - 1407 1407 - /* 1408 1408 - * kcdrwd is woken up when writes have been queued for one of our 1409 1409 - * registered devices 1410 1410 - */ 1411 1411 - static int kcdrwd(void *foobar) 1412 1412 - { 1413 1413 - struct pktcdvd_device *pd = foobar; 1414 1414 - struct device *ddev = disk_to_dev(pd->disk); 1415 1415 - struct packet_data *pkt; 1416 1416 - int states[PACKET_NUM_STATES]; 1417 1417 - long min_sleep_time, residue; 1418 1418 - 1419 1419 - set_user_nice(current, MIN_NICE); 1420 1420 - set_freezable(); 1421 1421 - 1422 1422 - for (;;) { 1423 1423 - DECLARE_WAITQUEUE(wait, current); 1424 1424 - 1425 1425 - /* 1426 1426 - * Wait until there is something to do 1427 1427 - */ 1428 1428 - add_wait_queue(&pd->wqueue, &wait); 1429 1429 - for (;;) { 1430 1430 - set_current_state(TASK_INTERRUPTIBLE); 1431 1431 - 1432 1432 - /* Check if we need to run pkt_handle_queue */ 1433 1433 - if (atomic_read(&pd->scan_queue) > 0) 1434 1434 - goto work_to_do; 1435 1435 - 1436 1436 - /* Check if we need to run the state machine for some packet */ 1437 1437 - list_for_each_entry(pkt, &pd->cdrw.pkt_active_list, list) { 1438 1438 - if (atomic_read(&pkt->run_sm) > 0) 1439 1439 - goto work_to_do; 1440 1440 - } 1441 1441 - 1442 1442 - /* Check if we need to process the iosched queues */ 1443 1443 - if (atomic_read(&pd->iosched.attention) != 0) 1444 1444 - goto work_to_do; 1445 1445 - 1446 1446 - /* Otherwise, go to sleep */ 1447 1447 - pkt_count_states(pd, states); 1448 1448 - dev_dbg(ddev, "i:%d ow:%d rw:%d ww:%d rec:%d fin:%d\n", 1449 1449 - states[0], states[1], states[2], states[3], states[4], states[5]); 1450 1450 - 1451 1451 - min_sleep_time = MAX_SCHEDULE_TIMEOUT; 1452 1452 - list_for_each_entry(pkt, &pd->cdrw.pkt_active_list, list) { 1453 1453 - if (pkt->sleep_time && pkt->sleep_time < min_sleep_time) 1454 1454 - min_sleep_time = pkt->sleep_time; 1455 1455 - } 1456 1456 - 1457 1457 - dev_dbg(ddev, "sleeping\n"); 1458 1458 - residue = schedule_timeout(min_sleep_time); 1459 1459 - dev_dbg(ddev, "wake up\n"); 1460 1460 - 1461 1461 - /* make swsusp happy with our thread */ 1462 1462 - try_to_freeze(); 1463 1463 - 1464 1464 - list_for_each_entry(pkt, &pd->cdrw.pkt_active_list, list) { 1465 1465 - if (!pkt->sleep_time) 1466 1466 - continue; 1467 1467 - pkt->sleep_time -= min_sleep_time - residue; 1468 1468 - if (pkt->sleep_time <= 0) { 1469 1469 - pkt->sleep_time = 0; 1470 1470 - atomic_inc(&pkt->run_sm); 1471 1471 - } 1472 1472 - } 1473 1473 - 1474 1474 - if (kthread_should_stop()) 1475 1475 - break; 1476 1476 - } 1477 1477 - work_to_do: 1478 1478 - set_current_state(TASK_RUNNING); 1479 1479 - remove_wait_queue(&pd->wqueue, &wait); 1480 1480 - 1481 1481 - if (kthread_should_stop()) 1482 1482 - break; 1483 1483 - 1484 1484 - /* 1485 1485 - * if pkt_handle_queue returns true, we can queue 1486 1486 - * another request. 1487 1487 - */ 1488 1488 - while (pkt_handle_queue(pd)) 1489 1489 - ; 1490 1490 - 1491 1491 - /* 1492 1492 - * Handle packet state machine 1493 1493 - */ 1494 1494 - pkt_handle_packets(pd); 1495 1495 - 1496 1496 - /* 1497 1497 - * Handle iosched queues 1498 1498 - */ 1499 1499 - pkt_iosched_process_queue(pd); 1500 1500 - } 1501 1501 - 1502 1502 - return 0; 1503 1503 - } 1504 1504 - 1505 1505 - static void pkt_print_settings(struct pktcdvd_device *pd) 1506 1506 - { 1507 1507 - dev_info(disk_to_dev(pd->disk), "%s packets, %u blocks, Mode-%c disc\n", 1508 1508 - pd->settings.fp ? "Fixed" : "Variable", 1509 1509 - pd->settings.size >> 2, 1510 1510 - pd->settings.block_mode == 8 ? '1' : '2'); 1511 1511 - } 1512 1512 - 1513 1513 - static int pkt_mode_sense(struct pktcdvd_device *pd, struct packet_command *cgc, int page_code, int page_control) 1514 1514 - { 1515 1515 - memset(cgc->cmd, 0, sizeof(cgc->cmd)); 1516 1516 - 1517 1517 - cgc->cmd[0] = GPCMD_MODE_SENSE_10; 1518 1518 - cgc->cmd[2] = page_code | (page_control << 6); 1519 1519 - put_unaligned_be16(cgc->buflen, &cgc->cmd[7]); 1520 1520 - cgc->data_direction = CGC_DATA_READ; 1521 1521 - return pkt_generic_packet(pd, cgc); 1522 1522 - } 1523 1523 - 1524 1524 - static int pkt_mode_select(struct pktcdvd_device *pd, struct packet_command *cgc) 1525 1525 - { 1526 1526 - memset(cgc->cmd, 0, sizeof(cgc->cmd)); 1527 1527 - memset(cgc->buffer, 0, 2); 1528 1528 - cgc->cmd[0] = GPCMD_MODE_SELECT_10; 1529 1529 - cgc->cmd[1] = 0x10; /* PF */ 1530 1530 - put_unaligned_be16(cgc->buflen, &cgc->cmd[7]); 1531 1531 - cgc->data_direction = CGC_DATA_WRITE; 1532 1532 - return pkt_generic_packet(pd, cgc); 1533 1533 - } 1534 1534 - 1535 1535 - static int pkt_get_disc_info(struct pktcdvd_device *pd, disc_information *di) 1536 1536 - { 1537 1537 - struct packet_command cgc; 1538 1538 - int ret; 1539 1539 - 1540 1540 - /* set up command and get the disc info */ 1541 1541 - init_cdrom_command(&cgc, di, sizeof(*di), CGC_DATA_READ); 1542 1542 - cgc.cmd[0] = GPCMD_READ_DISC_INFO; 1543 1543 - cgc.cmd[8] = cgc.buflen = 2; 1544 1544 - cgc.quiet = 1; 1545 1545 - 1546 1546 - ret = pkt_generic_packet(pd, &cgc); 1547 1547 - if (ret) 1548 1548 - return ret; 1549 1549 - 1550 1550 - /* not all drives have the same disc_info length, so requeue 1551 1551 - * packet with the length the drive tells us it can supply 1552 1552 - */ 1553 1553 - cgc.buflen = be16_to_cpu(di->disc_information_length) + 1554 1554 - sizeof(di->disc_information_length); 1555 1555 - 1556 1556 - if (cgc.buflen > sizeof(disc_information)) 1557 1557 - cgc.buflen = sizeof(disc_information); 1558 1558 - 1559 1559 - cgc.cmd[8] = cgc.buflen; 1560 1560 - return pkt_generic_packet(pd, &cgc); 1561 1561 - } 1562 1562 - 1563 1563 - static int pkt_get_track_info(struct pktcdvd_device *pd, __u16 track, __u8 type, track_information *ti) 1564 1564 - { 1565 1565 - struct packet_command cgc; 1566 1566 - int ret; 1567 1567 - 1568 1568 - init_cdrom_command(&cgc, ti, 8, CGC_DATA_READ); 1569 1569 - cgc.cmd[0] = GPCMD_READ_TRACK_RZONE_INFO; 1570 1570 - cgc.cmd[1] = type & 3; 1571 1571 - put_unaligned_be16(track, &cgc.cmd[4]); 1572 1572 - cgc.cmd[8] = 8; 1573 1573 - cgc.quiet = 1; 1574 1574 - 1575 1575 - ret = pkt_generic_packet(pd, &cgc); 1576 1576 - if (ret) 1577 1577 - return ret; 1578 1578 - 1579 1579 - cgc.buflen = be16_to_cpu(ti->track_information_length) + 1580 1580 - sizeof(ti->track_information_length); 1581 1581 - 1582 1582 - if (cgc.buflen > sizeof(track_information)) 1583 1583 - cgc.buflen = sizeof(track_information); 1584 1584 - 1585 1585 - cgc.cmd[8] = cgc.buflen; 1586 1586 - return pkt_generic_packet(pd, &cgc); 1587 1587 - } 1588 1588 - 1589 1589 - static noinline_for_stack int pkt_get_last_written(struct pktcdvd_device *pd, 1590 1590 - long *last_written) 1591 1591 - { 1592 1592 - disc_information di; 1593 1593 - track_information ti; 1594 1594 - __u32 last_track; 1595 1595 - int ret; 1596 1596 - 1597 1597 - ret = pkt_get_disc_info(pd, &di); 1598 1598 - if (ret) 1599 1599 - return ret; 1600 1600 - 1601 1601 - last_track = (di.last_track_msb << 8) | di.last_track_lsb; 1602 1602 - ret = pkt_get_track_info(pd, last_track, 1, &ti); 1603 1603 - if (ret) 1604 1604 - return ret; 1605 1605 - 1606 1606 - /* if this track is blank, try the previous. */ 1607 1607 - if (ti.blank) { 1608 1608 - last_track--; 1609 1609 - ret = pkt_get_track_info(pd, last_track, 1, &ti); 1610 1610 - if (ret) 1611 1611 - return ret; 1612 1612 - } 1613 1613 - 1614 1614 - /* if last recorded field is valid, return it. */ 1615 1615 - if (ti.lra_v) { 1616 1616 - *last_written = be32_to_cpu(ti.last_rec_address); 1617 1617 - } else { 1618 1618 - /* make it up instead */ 1619 1619 - *last_written = be32_to_cpu(ti.track_start) + 1620 1620 - be32_to_cpu(ti.track_size); 1621 1621 - if (ti.free_blocks) 1622 1622 - *last_written -= (be32_to_cpu(ti.free_blocks) + 7); 1623 1623 - } 1624 1624 - return 0; 1625 1625 - } 1626 1626 - 1627 1627 - /* 1628 1628 - * write mode select package based on pd->settings 1629 1629 - */ 1630 1630 - static noinline_for_stack int pkt_set_write_settings(struct pktcdvd_device *pd) 1631 1631 - { 1632 1632 - struct device *ddev = disk_to_dev(pd->disk); 1633 1633 - struct packet_command cgc; 1634 1634 - struct scsi_sense_hdr sshdr; 1635 1635 - write_param_page *wp; 1636 1636 - char buffer[128]; 1637 1637 - int ret, size; 1638 1638 - 1639 1639 - /* doesn't apply to DVD+RW or DVD-RAM */ 1640 1640 - if ((pd->mmc3_profile == 0x1a) || (pd->mmc3_profile == 0x12)) 1641 1641 - return 0; 1642 1642 - 1643 1643 - memset(buffer, 0, sizeof(buffer)); 1644 1644 - init_cdrom_command(&cgc, buffer, sizeof(*wp), CGC_DATA_READ); 1645 1645 - cgc.sshdr = &sshdr; 1646 1646 - ret = pkt_mode_sense(pd, &cgc, GPMODE_WRITE_PARMS_PAGE, 0); 1647 1647 - if (ret) { 1648 1648 - pkt_dump_sense(pd, &cgc); 1649 1649 - return ret; 1650 1650 - } 1651 1651 - 1652 1652 - size = 2 + get_unaligned_be16(&buffer[0]); 1653 1653 - pd->mode_offset = get_unaligned_be16(&buffer[6]); 1654 1654 - if (size > sizeof(buffer)) 1655 1655 - size = sizeof(buffer); 1656 1656 - 1657 1657 - /* 1658 1658 - * now get it all 1659 1659 - */ 1660 1660 - init_cdrom_command(&cgc, buffer, size, CGC_DATA_READ); 1661 1661 - cgc.sshdr = &sshdr; 1662 1662 - ret = pkt_mode_sense(pd, &cgc, GPMODE_WRITE_PARMS_PAGE, 0); 1663 1663 - if (ret) { 1664 1664 - pkt_dump_sense(pd, &cgc); 1665 1665 - return ret; 1666 1666 - } 1667 1667 - 1668 1668 - /* 1669 1669 - * write page is offset header + block descriptor length 1670 1670 - */ 1671 1671 - wp = (write_param_page *) &buffer[sizeof(struct mode_page_header) + pd->mode_offset]; 1672 1672 - 1673 1673 - wp->fp = pd->settings.fp; 1674 1674 - wp->track_mode = pd->settings.track_mode; 1675 1675 - wp->write_type = pd->settings.write_type; 1676 1676 - wp->data_block_type = pd->settings.block_mode; 1677 1677 - 1678 1678 - wp->multi_session = 0; 1679 1679 - 1680 1680 - #ifdef PACKET_USE_LS 1681 1681 - wp->link_size = 7; 1682 1682 - wp->ls_v = 1; 1683 1683 - #endif 1684 1684 - 1685 1685 - if (wp->data_block_type == PACKET_BLOCK_MODE1) { 1686 1686 - wp->session_format = 0; 1687 1687 - wp->subhdr2 = 0x20; 1688 1688 - } else if (wp->data_block_type == PACKET_BLOCK_MODE2) { 1689 1689 - wp->session_format = 0x20; 1690 1690 - wp->subhdr2 = 8; 1691 1691 - #if 0 1692 1692 - wp->mcn[0] = 0x80; 1693 1693 - memcpy(&wp->mcn[1], PACKET_MCN, sizeof(wp->mcn) - 1); 1694 1694 - #endif 1695 1695 - } else { 1696 1696 - /* 1697 1697 - * paranoia 1698 1698 - */ 1699 1699 - dev_err(ddev, "write mode wrong %d\n", wp->data_block_type); 1700 1700 - return 1; 1701 1701 - } 1702 1702 - wp->packet_size = cpu_to_be32(pd->settings.size >> 2); 1703 1703 - 1704 1704 - cgc.buflen = cgc.cmd[8] = size; 1705 1705 - ret = pkt_mode_select(pd, &cgc); 1706 1706 - if (ret) { 1707 1707 - pkt_dump_sense(pd, &cgc); 1708 1708 - return ret; 1709 1709 - } 1710 1710 - 1711 1711 - pkt_print_settings(pd); 1712 1712 - return 0; 1713 1713 - } 1714 1714 - 1715 1715 - /* 1716 1716 - * 1 -- we can write to this track, 0 -- we can't 1717 1717 - */ 1718 1718 - static int pkt_writable_track(struct pktcdvd_device *pd, track_information *ti) 1719 1719 - { 1720 1720 - struct device *ddev = disk_to_dev(pd->disk); 1721 1721 - 1722 1722 - switch (pd->mmc3_profile) { 1723 1723 - case 0x1a: /* DVD+RW */ 1724 1724 - case 0x12: /* DVD-RAM */ 1725 1725 - /* The track is always writable on DVD+RW/DVD-RAM */ 1726 1726 - return 1; 1727 1727 - default: 1728 1728 - break; 1729 1729 - } 1730 1730 - 1731 1731 - if (!ti->packet || !ti->fp) 1732 1732 - return 0; 1733 1733 - 1734 1734 - /* 1735 1735 - * "good" settings as per Mt Fuji. 1736 1736 - */ 1737 1737 - if (ti->rt == 0 && ti->blank == 0) 1738 1738 - return 1; 1739 1739 - 1740 1740 - if (ti->rt == 0 && ti->blank == 1) 1741 1741 - return 1; 1742 1742 - 1743 1743 - if (ti->rt == 1 && ti->blank == 0) 1744 1744 - return 1; 1745 1745 - 1746 1746 - dev_err(ddev, "bad state %d-%d-%d\n", ti->rt, ti->blank, ti->packet); 1747 1747 - return 0; 1748 1748 - } 1749 1749 - 1750 1750 - /* 1751 1751 - * 1 -- we can write to this disc, 0 -- we can't 1752 1752 - */ 1753 1753 - static int pkt_writable_disc(struct pktcdvd_device *pd, disc_information *di) 1754 1754 - { 1755 1755 - struct device *ddev = disk_to_dev(pd->disk); 1756 1756 - 1757 1757 - switch (pd->mmc3_profile) { 1758 1758 - case 0x0a: /* CD-RW */ 1759 1759 - case 0xffff: /* MMC3 not supported */ 1760 1760 - break; 1761 1761 - case 0x1a: /* DVD+RW */ 1762 1762 - case 0x13: /* DVD-RW */ 1763 1763 - case 0x12: /* DVD-RAM */ 1764 1764 - return 1; 1765 1765 - default: 1766 1766 - dev_dbg(ddev, "Wrong disc profile (%x)\n", pd->mmc3_profile); 1767 1767 - return 0; 1768 1768 - } 1769 1769 - 1770 1770 - /* 1771 1771 - * for disc type 0xff we should probably reserve a new track. 1772 1772 - * but i'm not sure, should we leave this to user apps? probably. 1773 1773 - */ 1774 1774 - if (di->disc_type == 0xff) { 1775 1775 - dev_notice(ddev, "unknown disc - no track?\n"); 1776 1776 - return 0; 1777 1777 - } 1778 1778 - 1779 1779 - if (di->disc_type != 0x20 && di->disc_type != 0) { 1780 1780 - dev_err(ddev, "wrong disc type (%x)\n", di->disc_type); 1781 1781 - return 0; 1782 1782 - } 1783 1783 - 1784 1784 - if (di->erasable == 0) { 1785 1785 - dev_err(ddev, "disc not erasable\n"); 1786 1786 - return 0; 1787 1787 - } 1788 1788 - 1789 1789 - if (di->border_status == PACKET_SESSION_RESERVED) { 1790 1790 - dev_err(ddev, "can't write to last track (reserved)\n"); 1791 1791 - return 0; 1792 1792 - } 1793 1793 - 1794 1794 - return 1; 1795 1795 - } 1796 1796 - 1797 1797 - static noinline_for_stack int pkt_probe_settings(struct pktcdvd_device *pd) 1798 1798 - { 1799 1799 - struct device *ddev = disk_to_dev(pd->disk); 1800 1800 - struct packet_command cgc; 1801 1801 - unsigned char buf[12]; 1802 1802 - disc_information di; 1803 1803 - track_information ti; 1804 1804 - int ret, track; 1805 1805 - 1806 1806 - init_cdrom_command(&cgc, buf, sizeof(buf), CGC_DATA_READ); 1807 1807 - cgc.cmd[0] = GPCMD_GET_CONFIGURATION; 1808 1808 - cgc.cmd[8] = 8; 1809 1809 - ret = pkt_generic_packet(pd, &cgc); 1810 1810 - pd->mmc3_profile = ret ? 0xffff : get_unaligned_be16(&buf[6]); 1811 1811 - 1812 1812 - memset(&di, 0, sizeof(disc_information)); 1813 1813 - memset(&ti, 0, sizeof(track_information)); 1814 1814 - 1815 1815 - ret = pkt_get_disc_info(pd, &di); 1816 1816 - if (ret) { 1817 1817 - dev_err(ddev, "failed get_disc\n"); 1818 1818 - return ret; 1819 1819 - } 1820 1820 - 1821 1821 - if (!pkt_writable_disc(pd, &di)) 1822 1822 - return -EROFS; 1823 1823 - 1824 1824 - pd->type = di.erasable ? PACKET_CDRW : PACKET_CDR; 1825 1825 - 1826 1826 - track = 1; /* (di.last_track_msb << 8) | di.last_track_lsb; */ 1827 1827 - ret = pkt_get_track_info(pd, track, 1, &ti); 1828 1828 - if (ret) { 1829 1829 - dev_err(ddev, "failed get_track\n"); 1830 1830 - return ret; 1831 1831 - } 1832 1832 - 1833 1833 - if (!pkt_writable_track(pd, &ti)) { 1834 1834 - dev_err(ddev, "can't write to this track\n"); 1835 1835 - return -EROFS; 1836 1836 - } 1837 1837 - 1838 1838 - /* 1839 1839 - * we keep packet size in 512 byte units, makes it easier to 1840 1840 - * deal with request calculations. 1841 1841 - */ 1842 1842 - pd->settings.size = be32_to_cpu(ti.fixed_packet_size) << 2; 1843 1843 - if (pd->settings.size == 0) { 1844 1844 - dev_notice(ddev, "detected zero packet size!\n"); 1845 1845 - return -ENXIO; 1846 1846 - } 1847 1847 - if (pd->settings.size > PACKET_MAX_SECTORS) { 1848 1848 - dev_err(ddev, "packet size is too big\n"); 1849 1849 - return -EROFS; 1850 1850 - } 1851 1851 - pd->settings.fp = ti.fp; 1852 1852 - pd->offset = (be32_to_cpu(ti.track_start) << 2) & (pd->settings.size - 1); 1853 1853 - 1854 1854 - if (ti.nwa_v) { 1855 1855 - pd->nwa = be32_to_cpu(ti.next_writable); 1856 1856 - set_bit(PACKET_NWA_VALID, &pd->flags); 1857 1857 - } 1858 1858 - 1859 1859 - /* 1860 1860 - * in theory we could use lra on -RW media as well and just zero 1861 1861 - * blocks that haven't been written yet, but in practice that 1862 1862 - * is just a no-go. we'll use that for -R, naturally. 1863 1863 - */ 1864 1864 - if (ti.lra_v) { 1865 1865 - pd->lra = be32_to_cpu(ti.last_rec_address); 1866 1866 - set_bit(PACKET_LRA_VALID, &pd->flags); 1867 1867 - } else { 1868 1868 - pd->lra = 0xffffffff; 1869 1869 - set_bit(PACKET_LRA_VALID, &pd->flags); 1870 1870 - } 1871 1871 - 1872 1872 - /* 1873 1873 - * fine for now 1874 1874 - */ 1875 1875 - pd->settings.link_loss = 7; 1876 1876 - pd->settings.write_type = 0; /* packet */ 1877 1877 - pd->settings.track_mode = ti.track_mode; 1878 1878 - 1879 1879 - /* 1880 1880 - * mode1 or mode2 disc 1881 1881 - */ 1882 1882 - switch (ti.data_mode) { 1883 1883 - case PACKET_MODE1: 1884 1884 - pd->settings.block_mode = PACKET_BLOCK_MODE1; 1885 1885 - break; 1886 1886 - case PACKET_MODE2: 1887 1887 - pd->settings.block_mode = PACKET_BLOCK_MODE2; 1888 1888 - break; 1889 1889 - default: 1890 1890 - dev_err(ddev, "unknown data mode\n"); 1891 1891 - return -EROFS; 1892 1892 - } 1893 1893 - return 0; 1894 1894 - } 1895 1895 - 1896 1896 - /* 1897 1897 - * enable/disable write caching on drive 1898 1898 - */ 1899 1899 - static noinline_for_stack int pkt_write_caching(struct pktcdvd_device *pd) 1900 1900 - { 1901 1901 - struct device *ddev = disk_to_dev(pd->disk); 1902 1902 - struct packet_command cgc; 1903 1903 - struct scsi_sense_hdr sshdr; 1904 1904 - unsigned char buf[64]; 1905 1905 - bool set = IS_ENABLED(CONFIG_CDROM_PKTCDVD_WCACHE); 1906 1906 - int ret; 1907 1907 - 1908 1908 - init_cdrom_command(&cgc, buf, sizeof(buf), CGC_DATA_READ); 1909 1909 - cgc.sshdr = &sshdr; 1910 1910 - cgc.buflen = pd->mode_offset + 12; 1911 1911 - 1912 1912 - /* 1913 1913 - * caching mode page might not be there, so quiet this command 1914 1914 - */ 1915 1915 - cgc.quiet = 1; 1916 1916 - 1917 1917 - ret = pkt_mode_sense(pd, &cgc, GPMODE_WCACHING_PAGE, 0); 1918 1918 - if (ret) 1919 1919 - return ret; 1920 1920 - 1921 1921 - /* 1922 1922 - * use drive write caching -- we need deferred error handling to be 1923 1923 - * able to successfully recover with this option (drive will return good 1924 1924 - * status as soon as the cdb is validated). 1925 1925 - */ 1926 1926 - buf[pd->mode_offset + 10] |= (set << 2); 1927 1927 - 1928 1928 - cgc.buflen = cgc.cmd[8] = 2 + get_unaligned_be16(&buf[0]); 1929 1929 - ret = pkt_mode_select(pd, &cgc); 1930 1930 - if (ret) { 1931 1931 - dev_err(ddev, "write caching control failed\n"); 1932 1932 - pkt_dump_sense(pd, &cgc); 1933 1933 - } else if (!ret && set) 1934 1934 - dev_notice(ddev, "enabled write caching\n"); 1935 1935 - return ret; 1936 1936 - } 1937 1937 - 1938 1938 - static int pkt_lock_door(struct pktcdvd_device *pd, int lockflag) 1939 1939 - { 1940 1940 - struct packet_command cgc; 1941 1941 - 1942 1942 - init_cdrom_command(&cgc, NULL, 0, CGC_DATA_NONE); 1943 1943 - cgc.cmd[0] = GPCMD_PREVENT_ALLOW_MEDIUM_REMOVAL; 1944 1944 - cgc.cmd[4] = lockflag ? 1 : 0; 1945 1945 - return pkt_generic_packet(pd, &cgc); 1946 1946 - } 1947 1947 - 1948 1948 - /* 1949 1949 - * Returns drive maximum write speed 1950 1950 - */ 1951 1951 - static noinline_for_stack int pkt_get_max_speed(struct pktcdvd_device *pd, 1952 1952 - unsigned *write_speed) 1953 1953 - { 1954 1954 - struct packet_command cgc; 1955 1955 - struct scsi_sense_hdr sshdr; 1956 1956 - unsigned char buf[256+18]; 1957 1957 - unsigned char *cap_buf; 1958 1958 - int ret, offset; 1959 1959 - 1960 1960 - cap_buf = &buf[sizeof(struct mode_page_header) + pd->mode_offset]; 1961 1961 - init_cdrom_command(&cgc, buf, sizeof(buf), CGC_DATA_UNKNOWN); 1962 1962 - cgc.sshdr = &sshdr; 1963 1963 - 1964 1964 - ret = pkt_mode_sense(pd, &cgc, GPMODE_CAPABILITIES_PAGE, 0); 1965 1965 - if (ret) { 1966 1966 - cgc.buflen = pd->mode_offset + cap_buf[1] + 2 + 1967 1967 - sizeof(struct mode_page_header); 1968 1968 - ret = pkt_mode_sense(pd, &cgc, GPMODE_CAPABILITIES_PAGE, 0); 1969 1969 - if (ret) { 1970 1970 - pkt_dump_sense(pd, &cgc); 1971 1971 - return ret; 1972 1972 - } 1973 1973 - } 1974 1974 - 1975 1975 - offset = 20; /* Obsoleted field, used by older drives */ 1976 1976 - if (cap_buf[1] >= 28) 1977 1977 - offset = 28; /* Current write speed selected */ 1978 1978 - if (cap_buf[1] >= 30) { 1979 1979 - /* If the drive reports at least one "Logical Unit Write 1980 1980 - * Speed Performance Descriptor Block", use the information 1981 1981 - * in the first block. (contains the highest speed) 1982 1982 - */ 1983 1983 - int num_spdb = get_unaligned_be16(&cap_buf[30]); 1984 1984 - if (num_spdb > 0) 1985 1985 - offset = 34; 1986 1986 - } 1987 1987 - 1988 1988 - *write_speed = get_unaligned_be16(&cap_buf[offset]); 1989 1989 - return 0; 1990 1990 - } 1991 1991 - 1992 1992 - /* These tables from cdrecord - I don't have orange book */ 1993 1993 - /* standard speed CD-RW (1-4x) */ 1994 1994 - static char clv_to_speed[16] = { 1995 1995 - /* 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 */ 1996 1996 - 0, 2, 4, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 1997 1997 - }; 1998 1998 - /* high speed CD-RW (-10x) */ 1999 1999 - static char hs_clv_to_speed[16] = { 2000 2000 - /* 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 */ 2001 2001 - 0, 2, 4, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 2002 2002 - }; 2003 2003 - /* ultra high speed CD-RW */ 2004 2004 - static char us_clv_to_speed[16] = { 2005 2005 - /* 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 */ 2006 2006 - 0, 2, 4, 8, 0, 0,16, 0,24,32,40,48, 0, 0, 0, 0 2007 2007 - }; 2008 2008 - 2009 2009 - /* 2010 2010 - * reads the maximum media speed from ATIP 2011 2011 - */ 2012 2012 - static noinline_for_stack int pkt_media_speed(struct pktcdvd_device *pd, 2013 2013 - unsigned *speed) 2014 2014 - { 2015 2015 - struct device *ddev = disk_to_dev(pd->disk); 2016 2016 - struct packet_command cgc; 2017 2017 - struct scsi_sense_hdr sshdr; 2018 2018 - unsigned char buf[64]; 2019 2019 - unsigned int size, st, sp; 2020 2020 - int ret; 2021 2021 - 2022 2022 - init_cdrom_command(&cgc, buf, 2, CGC_DATA_READ); 2023 2023 - cgc.sshdr = &sshdr; 2024 2024 - cgc.cmd[0] = GPCMD_READ_TOC_PMA_ATIP; 2025 2025 - cgc.cmd[1] = 2; 2026 2026 - cgc.cmd[2] = 4; /* READ ATIP */ 2027 2027 - cgc.cmd[8] = 2; 2028 2028 - ret = pkt_generic_packet(pd, &cgc); 2029 2029 - if (ret) { 2030 2030 - pkt_dump_sense(pd, &cgc); 2031 2031 - return ret; 2032 2032 - } 2033 2033 - size = 2 + get_unaligned_be16(&buf[0]); 2034 2034 - if (size > sizeof(buf)) 2035 2035 - size = sizeof(buf); 2036 2036 - 2037 2037 - init_cdrom_command(&cgc, buf, size, CGC_DATA_READ); 2038 2038 - cgc.sshdr = &sshdr; 2039 2039 - cgc.cmd[0] = GPCMD_READ_TOC_PMA_ATIP; 2040 2040 - cgc.cmd[1] = 2; 2041 2041 - cgc.cmd[2] = 4; 2042 2042 - cgc.cmd[8] = size; 2043 2043 - ret = pkt_generic_packet(pd, &cgc); 2044 2044 - if (ret) { 2045 2045 - pkt_dump_sense(pd, &cgc); 2046 2046 - return ret; 2047 2047 - } 2048 2048 - 2049 2049 - if (!(buf[6] & 0x40)) { 2050 2050 - dev_notice(ddev, "disc type is not CD-RW\n"); 2051 2051 - return 1; 2052 2052 - } 2053 2053 - if (!(buf[6] & 0x4)) { 2054 2054 - dev_notice(ddev, "A1 values on media are not valid, maybe not CDRW?\n"); 2055 2055 - return 1; 2056 2056 - } 2057 2057 - 2058 2058 - st = (buf[6] >> 3) & 0x7; /* disc sub-type */ 2059 2059 - 2060 2060 - sp = buf[16] & 0xf; /* max speed from ATIP A1 field */ 2061 2061 - 2062 2062 - /* Info from cdrecord */ 2063 2063 - switch (st) { 2064 2064 - case 0: /* standard speed */ 2065 2065 - *speed = clv_to_speed[sp]; 2066 2066 - break; 2067 2067 - case 1: /* high speed */ 2068 2068 - *speed = hs_clv_to_speed[sp]; 2069 2069 - break; 2070 2070 - case 2: /* ultra high speed */ 2071 2071 - *speed = us_clv_to_speed[sp]; 2072 2072 - break; 2073 2073 - default: 2074 2074 - dev_notice(ddev, "unknown disc sub-type %d\n", st); 2075 2075 - return 1; 2076 2076 - } 2077 2077 - if (*speed) { 2078 2078 - dev_info(ddev, "maximum media speed: %d\n", *speed); 2079 2079 - return 0; 2080 2080 - } else { 2081 2081 - dev_notice(ddev, "unknown speed %d for sub-type %d\n", sp, st); 2082 2082 - return 1; 2083 2083 - } 2084 2084 - } 2085 2085 - 2086 2086 - static noinline_for_stack int pkt_perform_opc(struct pktcdvd_device *pd) 2087 2087 - { 2088 2088 - struct device *ddev = disk_to_dev(pd->disk); 2089 2089 - struct packet_command cgc; 2090 2090 - struct scsi_sense_hdr sshdr; 2091 2091 - int ret; 2092 2092 - 2093 2093 - dev_dbg(ddev, "Performing OPC\n"); 2094 2094 - 2095 2095 - init_cdrom_command(&cgc, NULL, 0, CGC_DATA_NONE); 2096 2096 - cgc.sshdr = &sshdr; 2097 2097 - cgc.timeout = 60*HZ; 2098 2098 - cgc.cmd[0] = GPCMD_SEND_OPC; 2099 2099 - cgc.cmd[1] = 1; 2100 2100 - ret = pkt_generic_packet(pd, &cgc); 2101 2101 - if (ret) 2102 2102 - pkt_dump_sense(pd, &cgc); 2103 2103 - return ret; 2104 2104 - } 2105 2105 - 2106 2106 - static int pkt_open_write(struct pktcdvd_device *pd) 2107 2107 - { 2108 2108 - struct device *ddev = disk_to_dev(pd->disk); 2109 2109 - int ret; 2110 2110 - unsigned int write_speed, media_write_speed, read_speed; 2111 2111 - 2112 2112 - ret = pkt_probe_settings(pd); 2113 2113 - if (ret) { 2114 2114 - dev_dbg(ddev, "failed probe\n"); 2115 2115 - return ret; 2116 2116 - } 2117 2117 - 2118 2118 - ret = pkt_set_write_settings(pd); 2119 2119 - if (ret) { 2120 2120 - dev_notice(ddev, "failed saving write settings\n"); 2121 2121 - return -EIO; 2122 2122 - } 2123 2123 - 2124 2124 - pkt_write_caching(pd); 2125 2125 - 2126 2126 - ret = pkt_get_max_speed(pd, &write_speed); 2127 2127 - if (ret) 2128 2128 - write_speed = 16 * 177; 2129 2129 - switch (pd->mmc3_profile) { 2130 2130 - case 0x13: /* DVD-RW */ 2131 2131 - case 0x1a: /* DVD+RW */ 2132 2132 - case 0x12: /* DVD-RAM */ 2133 2133 - dev_notice(ddev, "write speed %ukB/s\n", write_speed); 2134 2134 - break; 2135 2135 - default: 2136 2136 - ret = pkt_media_speed(pd, &media_write_speed); 2137 2137 - if (ret) 2138 2138 - media_write_speed = 16; 2139 2139 - write_speed = min(write_speed, media_write_speed * 177); 2140 2140 - dev_notice(ddev, "write speed %ux\n", write_speed / 176); 2141 2141 - break; 2142 2142 - } 2143 2143 - read_speed = write_speed; 2144 2144 - 2145 2145 - ret = pkt_set_speed(pd, write_speed, read_speed); 2146 2146 - if (ret) { 2147 2147 - dev_notice(ddev, "couldn't set write speed\n"); 2148 2148 - return -EIO; 2149 2149 - } 2150 2150 - pd->write_speed = write_speed; 2151 2151 - pd->read_speed = read_speed; 2152 2152 - 2153 2153 - ret = pkt_perform_opc(pd); 2154 2154 - if (ret) 2155 2155 - dev_notice(ddev, "Optimum Power Calibration failed\n"); 2156 2156 - 2157 2157 - return 0; 2158 2158 - } 2159 2159 - 2160 2160 - /* 2161 2161 - * called at open time. 2162 2162 - */ 2163 2163 - static int pkt_open_dev(struct pktcdvd_device *pd, bool write) 2164 2164 - { 2165 2165 - struct device *ddev = disk_to_dev(pd->disk); 2166 2166 - int ret; 2167 2167 - long lba; 2168 2168 - struct request_queue *q; 2169 2169 - struct file *bdev_file; 2170 2170 - 2171 2171 - /* 2172 2172 - * We need to re-open the cdrom device without O_NONBLOCK to be able 2173 2173 - * to read/write from/to it. It is already opened in O_NONBLOCK mode 2174 2174 - * so open should not fail. 2175 2175 - */ 2176 2176 - bdev_file = bdev_file_open_by_dev(file_bdev(pd->bdev_file)->bd_dev, 2177 2177 - BLK_OPEN_READ, pd, NULL); 2178 2178 - if (IS_ERR(bdev_file)) { 2179 2179 - ret = PTR_ERR(bdev_file); 2180 2180 - goto out; 2181 2181 - } 2182 2182 - pd->f_open_bdev = bdev_file; 2183 2183 - 2184 2184 - ret = pkt_get_last_written(pd, &lba); 2185 2185 - if (ret) { 2186 2186 - dev_err(ddev, "pkt_get_last_written failed\n"); 2187 2187 - goto out_putdev; 2188 2188 - } 2189 2189 - 2190 2190 - set_capacity(pd->disk, lba << 2); 2191 2191 - set_capacity_and_notify(file_bdev(pd->bdev_file)->bd_disk, lba << 2); 2192 2192 - 2193 2193 - q = bdev_get_queue(file_bdev(pd->bdev_file)); 2194 2194 - if (write) { 2195 2195 - ret = pkt_open_write(pd); 2196 2196 - if (ret) 2197 2197 - goto out_putdev; 2198 2198 - set_bit(PACKET_WRITABLE, &pd->flags); 2199 2199 - } else { 2200 2200 - pkt_set_speed(pd, MAX_SPEED, MAX_SPEED); 2201 2201 - clear_bit(PACKET_WRITABLE, &pd->flags); 2202 2202 - } 2203 2203 - 2204 2204 - ret = pkt_set_segment_merging(pd, q); 2205 2205 - if (ret) 2206 2206 - goto out_putdev; 2207 2207 - 2208 2208 - if (write) { 2209 2209 - if (!pkt_grow_pktlist(pd, CONFIG_CDROM_PKTCDVD_BUFFERS)) { 2210 2210 - dev_err(ddev, "not enough memory for buffers\n"); 2211 2211 - ret = -ENOMEM; 2212 2212 - goto out_putdev; 2213 2213 - } 2214 2214 - dev_info(ddev, "%lukB available on disc\n", lba << 1); 2215 2215 - } 2216 2216 - set_blocksize(bdev_file, CD_FRAMESIZE); 2217 2217 - 2218 2218 - return 0; 2219 2219 - 2220 2220 - out_putdev: 2221 2221 - fput(bdev_file); 2222 2222 - out: 2223 2223 - return ret; 2224 2224 - } 2225 2225 - 2226 2226 - /* 2227 2227 - * called when the device is closed. makes sure that the device flushes 2228 2228 - * the internal cache before we close. 2229 2229 - */ 2230 2230 - static void pkt_release_dev(struct pktcdvd_device *pd, int flush) 2231 2231 - { 2232 2232 - struct device *ddev = disk_to_dev(pd->disk); 2233 2233 - 2234 2234 - if (flush && pkt_flush_cache(pd)) 2235 2235 - dev_notice(ddev, "not flushing cache\n"); 2236 2236 - 2237 2237 - pkt_lock_door(pd, 0); 2238 2238 - 2239 2239 - pkt_set_speed(pd, MAX_SPEED, MAX_SPEED); 2240 2240 - fput(pd->f_open_bdev); 2241 2241 - pd->f_open_bdev = NULL; 2242 2242 - 2243 2243 - pkt_shrink_pktlist(pd); 2244 2244 - } 2245 2245 - 2246 2246 - static struct pktcdvd_device *pkt_find_dev_from_minor(unsigned int dev_minor) 2247 2247 - { 2248 2248 - if (dev_minor >= MAX_WRITERS) 2249 2249 - return NULL; 2250 2250 - 2251 2251 - dev_minor = array_index_nospec(dev_minor, MAX_WRITERS); 2252 2252 - return pkt_devs[dev_minor]; 2253 2253 - } 2254 2254 - 2255 2255 - static int pkt_open(struct gendisk *disk, blk_mode_t mode) 2256 2256 - { 2257 2257 - struct pktcdvd_device *pd = NULL; 2258 2258 - int ret; 2259 2259 - 2260 2260 - mutex_lock(&pktcdvd_mutex); 2261 2261 - mutex_lock(&ctl_mutex); 2262 2262 - pd = pkt_find_dev_from_minor(disk->first_minor); 2263 2263 - if (!pd) { 2264 2264 - ret = -ENODEV; 2265 2265 - goto out; 2266 2266 - } 2267 2267 - BUG_ON(pd->refcnt < 0); 2268 2268 - 2269 2269 - pd->refcnt++; 2270 2270 - if (pd->refcnt > 1) { 2271 2271 - if ((mode & BLK_OPEN_WRITE) && 2272 2272 - !test_bit(PACKET_WRITABLE, &pd->flags)) { 2273 2273 - ret = -EBUSY; 2274 2274 - goto out_dec; 2275 2275 - } 2276 2276 - } else { 2277 2277 - ret = pkt_open_dev(pd, mode & BLK_OPEN_WRITE); 2278 2278 - if (ret) 2279 2279 - goto out_dec; 2280 2280 - } 2281 2281 - mutex_unlock(&ctl_mutex); 2282 2282 - mutex_unlock(&pktcdvd_mutex); 2283 2283 - return 0; 2284 2284 - 2285 2285 - out_dec: 2286 2286 - pd->refcnt--; 2287 2287 - out: 2288 2288 - mutex_unlock(&ctl_mutex); 2289 2289 - mutex_unlock(&pktcdvd_mutex); 2290 2290 - return ret; 2291 2291 - } 2292 2292 - 2293 2293 - static void pkt_release(struct gendisk *disk) 2294 2294 - { 2295 2295 - struct pktcdvd_device *pd = disk->private_data; 2296 2296 - 2297 2297 - mutex_lock(&pktcdvd_mutex); 2298 2298 - mutex_lock(&ctl_mutex); 2299 2299 - pd->refcnt--; 2300 2300 - BUG_ON(pd->refcnt < 0); 2301 2301 - if (pd->refcnt == 0) { 2302 2302 - int flush = test_bit(PACKET_WRITABLE, &pd->flags); 2303 2303 - pkt_release_dev(pd, flush); 2304 2304 - } 2305 2305 - mutex_unlock(&ctl_mutex); 2306 2306 - mutex_unlock(&pktcdvd_mutex); 2307 2307 - } 2308 2308 - 2309 2309 - 2310 2310 - static void pkt_end_io_read_cloned(struct bio *bio) 2311 2311 - { 2312 2312 - struct packet_stacked_data *psd = bio->bi_private; 2313 2313 - struct pktcdvd_device *pd = psd->pd; 2314 2314 - 2315 2315 - psd->bio->bi_status = bio->bi_status; 2316 2316 - bio_put(bio); 2317 2317 - bio_endio(psd->bio); 2318 2318 - mempool_free(psd, &psd_pool); 2319 2319 - pkt_bio_finished(pd); 2320 2320 - } 2321 2321 - 2322 2322 - static void pkt_make_request_read(struct pktcdvd_device *pd, struct bio *bio) 2323 2323 - { 2324 2324 - struct bio *cloned_bio = bio_alloc_clone(file_bdev(pd->bdev_file), bio, 2325 2325 - GFP_NOIO, &pkt_bio_set); 2326 2326 - struct packet_stacked_data *psd = mempool_alloc(&psd_pool, GFP_NOIO); 2327 2327 - 2328 2328 - psd->pd = pd; 2329 2329 - psd->bio = bio; 2330 2330 - cloned_bio->bi_private = psd; 2331 2331 - cloned_bio->bi_end_io = pkt_end_io_read_cloned; 2332 2332 - pd->stats.secs_r += bio_sectors(bio); 2333 2333 - pkt_queue_bio(pd, cloned_bio); 2334 2334 - } 2335 2335 - 2336 2336 - static void pkt_make_request_write(struct bio *bio) 2337 2337 - { 2338 2338 - struct pktcdvd_device *pd = bio->bi_bdev->bd_disk->private_data; 2339 2339 - sector_t zone; 2340 2340 - struct packet_data *pkt; 2341 2341 - int was_empty, blocked_bio; 2342 2342 - struct pkt_rb_node *node; 2343 2343 - 2344 2344 - zone = get_zone(bio->bi_iter.bi_sector, pd); 2345 2345 - 2346 2346 - /* 2347 2347 - * If we find a matching packet in state WAITING or READ_WAIT, we can 2348 2348 - * just append this bio to that packet. 2349 2349 - */ 2350 2350 - spin_lock(&pd->cdrw.active_list_lock); 2351 2351 - blocked_bio = 0; 2352 2352 - list_for_each_entry(pkt, &pd->cdrw.pkt_active_list, list) { 2353 2353 - if (pkt->sector == zone) { 2354 2354 - spin_lock(&pkt->lock); 2355 2355 - if ((pkt->state == PACKET_WAITING_STATE) || 2356 2356 - (pkt->state == PACKET_READ_WAIT_STATE)) { 2357 2357 - bio_list_add(&pkt->orig_bios, bio); 2358 2358 - pkt->write_size += 2359 2359 - bio->bi_iter.bi_size / CD_FRAMESIZE; 2360 2360 - if ((pkt->write_size >= pkt->frames) && 2361 2361 - (pkt->state == PACKET_WAITING_STATE)) { 2362 2362 - atomic_inc(&pkt->run_sm); 2363 2363 - wake_up(&pd->wqueue); 2364 2364 - } 2365 2365 - spin_unlock(&pkt->lock); 2366 2366 - spin_unlock(&pd->cdrw.active_list_lock); 2367 2367 - return; 2368 2368 - } else { 2369 2369 - blocked_bio = 1; 2370 2370 - } 2371 2371 - spin_unlock(&pkt->lock); 2372 2372 - } 2373 2373 - } 2374 2374 - spin_unlock(&pd->cdrw.active_list_lock); 2375 2375 - 2376 2376 - /* 2377 2377 - * Test if there is enough room left in the bio work queue 2378 2378 - * (queue size >= congestion on mark). 2379 2379 - * If not, wait till the work queue size is below the congestion off mark. 2380 2380 - */ 2381 2381 - spin_lock(&pd->lock); 2382 2382 - if (pd->write_congestion_on > 0 2383 2383 - && pd->bio_queue_size >= pd->write_congestion_on) { 2384 2384 - struct wait_bit_queue_entry wqe; 2385 2385 - 2386 2386 - init_wait_var_entry(&wqe, &pd->congested, 0); 2387 2387 - for (;;) { 2388 2388 - prepare_to_wait_event(__var_waitqueue(&pd->congested), 2389 2389 - &wqe.wq_entry, 2390 2390 - TASK_UNINTERRUPTIBLE); 2391 2391 - if (pd->bio_queue_size <= pd->write_congestion_off) 2392 2392 - break; 2393 2393 - pd->congested = true; 2394 2394 - spin_unlock(&pd->lock); 2395 2395 - schedule(); 2396 2396 - spin_lock(&pd->lock); 2397 2397 - } 2398 2398 - } 2399 2399 - spin_unlock(&pd->lock); 2400 2400 - 2401 2401 - /* 2402 2402 - * No matching packet found. Store the bio in the work queue. 2403 2403 - */ 2404 2404 - node = mempool_alloc(&pd->rb_pool, GFP_NOIO); 2405 2405 - node->bio = bio; 2406 2406 - spin_lock(&pd->lock); 2407 2407 - BUG_ON(pd->bio_queue_size < 0); 2408 2408 - was_empty = (pd->bio_queue_size == 0); 2409 2409 - pkt_rbtree_insert(pd, node); 2410 2410 - spin_unlock(&pd->lock); 2411 2411 - 2412 2412 - /* 2413 2413 - * Wake up the worker thread. 2414 2414 - */ 2415 2415 - atomic_set(&pd->scan_queue, 1); 2416 2416 - if (was_empty) { 2417 2417 - /* This wake_up is required for correct operation */ 2418 2418 - wake_up(&pd->wqueue); 2419 2419 - } else if (!list_empty(&pd->cdrw.pkt_free_list) && !blocked_bio) { 2420 2420 - /* 2421 2421 - * This wake up is not required for correct operation, 2422 2422 - * but improves performance in some cases. 2423 2423 - */ 2424 2424 - wake_up(&pd->wqueue); 2425 2425 - } 2426 2426 - } 2427 2427 - 2428 2428 - static void pkt_submit_bio(struct bio *bio) 2429 2429 - { 2430 2430 - struct pktcdvd_device *pd = bio->bi_bdev->bd_disk->private_data; 2431 2431 - struct device *ddev = disk_to_dev(pd->disk); 2432 2432 - struct bio *split; 2433 2433 - 2434 2434 - bio = bio_split_to_limits(bio); 2435 2435 - if (!bio) 2436 2436 - return; 2437 2437 - 2438 2438 - dev_dbg(ddev, "start = %6llx stop = %6llx\n", 2439 2439 - bio->bi_iter.bi_sector, bio_end_sector(bio)); 2440 2440 - 2441 2441 - /* 2442 2442 - * Clone READ bios so we can have our own bi_end_io callback. 2443 2443 - */ 2444 2444 - if (bio_data_dir(bio) == READ) { 2445 2445 - pkt_make_request_read(pd, bio); 2446 2446 - return; 2447 2447 - } 2448 2448 - 2449 2449 - if (!test_bit(PACKET_WRITABLE, &pd->flags)) { 2450 2450 - dev_notice(ddev, "WRITE for ro device (%llu)\n", bio->bi_iter.bi_sector); 2451 2451 - goto end_io; 2452 2452 - } 2453 2453 - 2454 2454 - if (!bio->bi_iter.bi_size || (bio->bi_iter.bi_size % CD_FRAMESIZE)) { 2455 2455 - dev_err(ddev, "wrong bio size\n"); 2456 2456 - goto end_io; 2457 2457 - } 2458 2458 - 2459 2459 - do { 2460 2460 - sector_t zone = get_zone(bio->bi_iter.bi_sector, pd); 2461 2461 - sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd); 2462 2462 - 2463 2463 - if (last_zone != zone) { 2464 2464 - BUG_ON(last_zone != zone + pd->settings.size); 2465 2465 - 2466 2466 - split = bio_split(bio, last_zone - 2467 2467 - bio->bi_iter.bi_sector, 2468 2468 - GFP_NOIO, &pkt_bio_set); 2469 2469 - bio_chain(split, bio); 2470 2470 - } else { 2471 2471 - split = bio; 2472 2472 - } 2473 2473 - 2474 2474 - pkt_make_request_write(split); 2475 2475 - } while (split != bio); 2476 2476 - 2477 2477 - return; 2478 2478 - end_io: 2479 2479 - bio_io_error(bio); 2480 2480 - } 2481 2481 - 2482 2482 - static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev) 2483 2483 - { 2484 2484 - struct device *ddev = disk_to_dev(pd->disk); 2485 2485 - int i; 2486 2486 - struct file *bdev_file; 2487 2487 - struct scsi_device *sdev; 2488 2488 - 2489 2489 - if (pd->pkt_dev == dev) { 2490 2490 - dev_err(ddev, "recursive setup not allowed\n"); 2491 2491 - return -EBUSY; 2492 2492 - } 2493 2493 - for (i = 0; i < MAX_WRITERS; i++) { 2494 2494 - struct pktcdvd_device *pd2 = pkt_devs[i]; 2495 2495 - if (!pd2) 2496 2496 - continue; 2497 2497 - if (file_bdev(pd2->bdev_file)->bd_dev == dev) { 2498 2498 - dev_err(ddev, "%pg already setup\n", 2499 2499 - file_bdev(pd2->bdev_file)); 2500 2500 - return -EBUSY; 2501 2501 - } 2502 2502 - if (pd2->pkt_dev == dev) { 2503 2503 - dev_err(ddev, "can't chain pktcdvd devices\n"); 2504 2504 - return -EBUSY; 2505 2505 - } 2506 2506 - } 2507 2507 - 2508 2508 - bdev_file = bdev_file_open_by_dev(dev, BLK_OPEN_READ | BLK_OPEN_NDELAY, 2509 2509 - NULL, NULL); 2510 2510 - if (IS_ERR(bdev_file)) 2511 2511 - return PTR_ERR(bdev_file); 2512 2512 - sdev = scsi_device_from_queue(file_bdev(bdev_file)->bd_disk->queue); 2513 2513 - if (!sdev) { 2514 2514 - fput(bdev_file); 2515 2515 - return -EINVAL; 2516 2516 - } 2517 2517 - put_device(&sdev->sdev_gendev); 2518 2518 - 2519 2519 - /* This is safe, since we have a reference from open(). */ 2520 2520 - __module_get(THIS_MODULE); 2521 2521 - 2522 2522 - pd->bdev_file = bdev_file; 2523 2523 - 2524 2524 - atomic_set(&pd->cdrw.pending_bios, 0); 2525 2525 - pd->cdrw.thread = kthread_run(kcdrwd, pd, "%s", pd->disk->disk_name); 2526 2526 - if (IS_ERR(pd->cdrw.thread)) { 2527 2527 - dev_err(ddev, "can't start kernel thread\n"); 2528 2528 - goto out_mem; 2529 2529 - } 2530 2530 - 2531 2531 - proc_create_single_data(pd->disk->disk_name, 0, pkt_proc, pkt_seq_show, pd); 2532 2532 - dev_notice(ddev, "writer mapped to %pg\n", file_bdev(bdev_file)); 2533 2533 - return 0; 2534 2534 - 2535 2535 - out_mem: 2536 2536 - fput(bdev_file); 2537 2537 - /* This is safe: open() is still holding a reference. */ 2538 2538 - module_put(THIS_MODULE); 2539 2539 - return -ENOMEM; 2540 2540 - } 2541 2541 - 2542 2542 - static int pkt_ioctl(struct block_device *bdev, blk_mode_t mode, 2543 2543 - unsigned int cmd, unsigned long arg) 2544 2544 - { 2545 2545 - struct pktcdvd_device *pd = bdev->bd_disk->private_data; 2546 2546 - struct device *ddev = disk_to_dev(pd->disk); 2547 2547 - int ret; 2548 2548 - 2549 2549 - dev_dbg(ddev, "cmd %x, dev %d:%d\n", cmd, MAJOR(bdev->bd_dev), MINOR(bdev->bd_dev)); 2550 2550 - 2551 2551 - mutex_lock(&pktcdvd_mutex); 2552 2552 - switch (cmd) { 2553 2553 - case CDROMEJECT: 2554 2554 - /* 2555 2555 - * The door gets locked when the device is opened, so we 2556 2556 - * have to unlock it or else the eject command fails. 2557 2557 - */ 2558 2558 - if (pd->refcnt == 1) 2559 2559 - pkt_lock_door(pd, 0); 2560 2560 - fallthrough; 2561 2561 - /* 2562 2562 - * forward selected CDROM ioctls to CD-ROM, for UDF 2563 2563 - */ 2564 2564 - case CDROMMULTISESSION: 2565 2565 - case CDROMREADTOCENTRY: 2566 2566 - case CDROM_LAST_WRITTEN: 2567 2567 - case CDROM_SEND_PACKET: 2568 2568 - case SCSI_IOCTL_SEND_COMMAND: 2569 2569 - if (!bdev->bd_disk->fops->ioctl) 2570 2570 - ret = -ENOTTY; 2571 2571 - else 2572 2572 - ret = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg); 2573 2573 - break; 2574 2574 - default: 2575 2575 - dev_dbg(ddev, "Unknown ioctl (%x)\n", cmd); 2576 2576 - ret = -ENOTTY; 2577 2577 - } 2578 2578 - mutex_unlock(&pktcdvd_mutex); 2579 2579 - 2580 2580 - return ret; 2581 2581 - } 2582 2582 - 2583 2583 - static unsigned int pkt_check_events(struct gendisk *disk, 2584 2584 - unsigned int clearing) 2585 2585 - { 2586 2586 - struct pktcdvd_device *pd = disk->private_data; 2587 2587 - struct gendisk *attached_disk; 2588 2588 - 2589 2589 - if (!pd) 2590 2590 - return 0; 2591 2591 - if (!pd->bdev_file) 2592 2592 - return 0; 2593 2593 - attached_disk = file_bdev(pd->bdev_file)->bd_disk; 2594 2594 - if (!attached_disk || !attached_disk->fops->check_events) 2595 2595 - return 0; 2596 2596 - return attached_disk->fops->check_events(attached_disk, clearing); 2597 2597 - } 2598 2598 - 2599 2599 - static char *pkt_devnode(struct gendisk *disk, umode_t *mode) 2600 2600 - { 2601 2601 - return kasprintf(GFP_KERNEL, "pktcdvd/%s", disk->disk_name); 2602 2602 - } 2603 2603 - 2604 2604 - static const struct block_device_operations pktcdvd_ops = { 2605 2605 - .owner = THIS_MODULE, 2606 2606 - .submit_bio = pkt_submit_bio, 2607 2607 - .open = pkt_open, 2608 2608 - .release = pkt_release, 2609 2609 - .ioctl = pkt_ioctl, 2610 2610 - .compat_ioctl = blkdev_compat_ptr_ioctl, 2611 2611 - .check_events = pkt_check_events, 2612 2612 - .devnode = pkt_devnode, 2613 2613 - }; 2614 2614 - 2615 2615 - /* 2616 2616 - * Set up mapping from pktcdvd device to CD-ROM device. 2617 2617 - */ 2618 2618 - static int pkt_setup_dev(dev_t dev, dev_t* pkt_dev) 2619 2619 - { 2620 2620 - struct queue_limits lim = { 2621 2621 - .max_hw_sectors = PACKET_MAX_SECTORS, 2622 2622 - .logical_block_size = CD_FRAMESIZE, 2623 2623 - .features = BLK_FEAT_ROTATIONAL, 2624 2624 - }; 2625 2625 - int idx; 2626 2626 - int ret = -ENOMEM; 2627 2627 - struct pktcdvd_device *pd; 2628 2628 - struct gendisk *disk; 2629 2629 - 2630 2630 - mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING); 2631 2631 - 2632 2632 - for (idx = 0; idx < MAX_WRITERS; idx++) 2633 2633 - if (!pkt_devs[idx]) 2634 2634 - break; 2635 2635 - if (idx == MAX_WRITERS) { 2636 2636 - pr_err("max %d writers supported\n", MAX_WRITERS); 2637 2637 - ret = -EBUSY; 2638 2638 - goto out_mutex; 2639 2639 - } 2640 2640 - 2641 2641 - pd = kzalloc(sizeof(struct pktcdvd_device), GFP_KERNEL); 2642 2642 - if (!pd) 2643 2643 - goto out_mutex; 2644 2644 - 2645 2645 - ret = mempool_init_kmalloc_pool(&pd->rb_pool, PKT_RB_POOL_SIZE, 2646 2646 - sizeof(struct pkt_rb_node)); 2647 2647 - if (ret) 2648 2648 - goto out_mem; 2649 2649 - 2650 2650 - INIT_LIST_HEAD(&pd->cdrw.pkt_free_list); 2651 2651 - INIT_LIST_HEAD(&pd->cdrw.pkt_active_list); 2652 2652 - spin_lock_init(&pd->cdrw.active_list_lock); 2653 2653 - 2654 2654 - spin_lock_init(&pd->lock); 2655 2655 - spin_lock_init(&pd->iosched.lock); 2656 2656 - bio_list_init(&pd->iosched.read_queue); 2657 2657 - bio_list_init(&pd->iosched.write_queue); 2658 2658 - init_waitqueue_head(&pd->wqueue); 2659 2659 - pd->bio_queue = RB_ROOT; 2660 2660 - 2661 2661 - pd->write_congestion_on = write_congestion_on; 2662 2662 - pd->write_congestion_off = write_congestion_off; 2663 2663 - 2664 2664 - disk = blk_alloc_disk(&lim, NUMA_NO_NODE); 2665 2665 - if (IS_ERR(disk)) { 2666 2666 - ret = PTR_ERR(disk); 2667 2667 - goto out_mem; 2668 2668 - } 2669 2669 - pd->disk = disk; 2670 2670 - disk->major = pktdev_major; 2671 2671 - disk->first_minor = idx; 2672 2672 - disk->minors = 1; 2673 2673 - disk->fops = &pktcdvd_ops; 2674 2674 - disk->flags = GENHD_FL_REMOVABLE | GENHD_FL_NO_PART; 2675 2675 - snprintf(disk->disk_name, sizeof(disk->disk_name), DRIVER_NAME"%d", idx); 2676 2676 - disk->private_data = pd; 2677 2677 - 2678 2678 - pd->pkt_dev = MKDEV(pktdev_major, idx); 2679 2679 - ret = pkt_new_dev(pd, dev); 2680 2680 - if (ret) 2681 2681 - goto out_mem2; 2682 2682 - 2683 2683 - /* inherit events of the host device */ 2684 2684 - disk->events = file_bdev(pd->bdev_file)->bd_disk->events; 2685 2685 - 2686 2686 - ret = add_disk(disk); 2687 2687 - if (ret) 2688 2688 - goto out_mem2; 2689 2689 - 2690 2690 - pkt_sysfs_dev_new(pd); 2691 2691 - pkt_debugfs_dev_new(pd); 2692 2692 - 2693 2693 - pkt_devs[idx] = pd; 2694 2694 - if (pkt_dev) 2695 2695 - *pkt_dev = pd->pkt_dev; 2696 2696 - 2697 2697 - mutex_unlock(&ctl_mutex); 2698 2698 - return 0; 2699 2699 - 2700 2700 - out_mem2: 2701 2701 - put_disk(disk); 2702 2702 - out_mem: 2703 2703 - mempool_exit(&pd->rb_pool); 2704 2704 - kfree(pd); 2705 2705 - out_mutex: 2706 2706 - mutex_unlock(&ctl_mutex); 2707 2707 - pr_err("setup of pktcdvd device failed\n"); 2708 2708 - return ret; 2709 2709 - } 2710 2710 - 2711 2711 - /* 2712 2712 - * Tear down mapping from pktcdvd device to CD-ROM device. 2713 2713 - */ 2714 2714 - static int pkt_remove_dev(dev_t pkt_dev) 2715 2715 - { 2716 2716 - struct pktcdvd_device *pd; 2717 2717 - struct device *ddev; 2718 2718 - int idx; 2719 2719 - int ret = 0; 2720 2720 - 2721 2721 - mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING); 2722 2722 - 2723 2723 - for (idx = 0; idx < MAX_WRITERS; idx++) { 2724 2724 - pd = pkt_devs[idx]; 2725 2725 - if (pd && (pd->pkt_dev == pkt_dev)) 2726 2726 - break; 2727 2727 - } 2728 2728 - if (idx == MAX_WRITERS) { 2729 2729 - pr_debug("dev not setup\n"); 2730 2730 - ret = -ENXIO; 2731 2731 - goto out; 2732 2732 - } 2733 2733 - 2734 2734 - if (pd->refcnt > 0) { 2735 2735 - ret = -EBUSY; 2736 2736 - goto out; 2737 2737 - } 2738 2738 - 2739 2739 - ddev = disk_to_dev(pd->disk); 2740 2740 - 2741 2741 - if (!IS_ERR(pd->cdrw.thread)) 2742 2742 - kthread_stop(pd->cdrw.thread); 2743 2743 - 2744 2744 - pkt_devs[idx] = NULL; 2745 2745 - 2746 2746 - pkt_debugfs_dev_remove(pd); 2747 2747 - pkt_sysfs_dev_remove(pd); 2748 2748 - 2749 2749 - fput(pd->bdev_file); 2750 2750 - 2751 2751 - remove_proc_entry(pd->disk->disk_name, pkt_proc); 2752 2752 - dev_notice(ddev, "writer unmapped\n"); 2753 2753 - 2754 2754 - del_gendisk(pd->disk); 2755 2755 - put_disk(pd->disk); 2756 2756 - 2757 2757 - mempool_exit(&pd->rb_pool); 2758 2758 - kfree(pd); 2759 2759 - 2760 2760 - /* This is safe: open() is still holding a reference. */ 2761 2761 - module_put(THIS_MODULE); 2762 2762 - 2763 2763 - out: 2764 2764 - mutex_unlock(&ctl_mutex); 2765 2765 - return ret; 2766 2766 - } 2767 2767 - 2768 2768 - static void pkt_get_status(struct pkt_ctrl_command *ctrl_cmd) 2769 2769 - { 2770 2770 - struct pktcdvd_device *pd; 2771 2771 - 2772 2772 - mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING); 2773 2773 - 2774 2774 - pd = pkt_find_dev_from_minor(ctrl_cmd->dev_index); 2775 2775 - if (pd) { 2776 2776 - ctrl_cmd->dev = new_encode_dev(file_bdev(pd->bdev_file)->bd_dev); 2777 2777 - ctrl_cmd->pkt_dev = new_encode_dev(pd->pkt_dev); 2778 2778 - } else { 2779 2779 - ctrl_cmd->dev = 0; 2780 2780 - ctrl_cmd->pkt_dev = 0; 2781 2781 - } 2782 2782 - ctrl_cmd->num_devices = MAX_WRITERS; 2783 2783 - 2784 2784 - mutex_unlock(&ctl_mutex); 2785 2785 - } 2786 2786 - 2787 2787 - static long pkt_ctl_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 2788 2788 - { 2789 2789 - void __user *argp = (void __user *)arg; 2790 2790 - struct pkt_ctrl_command ctrl_cmd; 2791 2791 - int ret = 0; 2792 2792 - dev_t pkt_dev = 0; 2793 2793 - 2794 2794 - if (cmd != PACKET_CTRL_CMD) 2795 2795 - return -ENOTTY; 2796 2796 - 2797 2797 - if (copy_from_user(&ctrl_cmd, argp, sizeof(struct pkt_ctrl_command))) 2798 2798 - return -EFAULT; 2799 2799 - 2800 2800 - switch (ctrl_cmd.command) { 2801 2801 - case PKT_CTRL_CMD_SETUP: 2802 2802 - if (!capable(CAP_SYS_ADMIN)) 2803 2803 - return -EPERM; 2804 2804 - ret = pkt_setup_dev(new_decode_dev(ctrl_cmd.dev), &pkt_dev); 2805 2805 - ctrl_cmd.pkt_dev = new_encode_dev(pkt_dev); 2806 2806 - break; 2807 2807 - case PKT_CTRL_CMD_TEARDOWN: 2808 2808 - if (!capable(CAP_SYS_ADMIN)) 2809 2809 - return -EPERM; 2810 2810 - ret = pkt_remove_dev(new_decode_dev(ctrl_cmd.pkt_dev)); 2811 2811 - break; 2812 2812 - case PKT_CTRL_CMD_STATUS: 2813 2813 - pkt_get_status(&ctrl_cmd); 2814 2814 - break; 2815 2815 - default: 2816 2816 - return -ENOTTY; 2817 2817 - } 2818 2818 - 2819 2819 - if (copy_to_user(argp, &ctrl_cmd, sizeof(struct pkt_ctrl_command))) 2820 2820 - return -EFAULT; 2821 2821 - return ret; 2822 2822 - } 2823 2823 - 2824 2824 - #ifdef CONFIG_COMPAT 2825 2825 - static long pkt_ctl_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 2826 2826 - { 2827 2827 - return pkt_ctl_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); 2828 2828 - } 2829 2829 - #endif 2830 2830 - 2831 2831 - static const struct file_operations pkt_ctl_fops = { 2832 2832 - .open = nonseekable_open, 2833 2833 - .unlocked_ioctl = pkt_ctl_ioctl, 2834 2834 - #ifdef CONFIG_COMPAT 2835 2835 - .compat_ioctl = pkt_ctl_compat_ioctl, 2836 2836 - #endif 2837 2837 - .owner = THIS_MODULE, 2838 2838 - }; 2839 2839 - 2840 2840 - static struct miscdevice pkt_misc = { 2841 2841 - .minor = MISC_DYNAMIC_MINOR, 2842 2842 - .name = DRIVER_NAME, 2843 2843 - .nodename = "pktcdvd/control", 2844 2844 - .fops = &pkt_ctl_fops 2845 2845 - }; 2846 2846 - 2847 2847 - static int __init pkt_init(void) 2848 2848 - { 2849 2849 - int ret; 2850 2850 - 2851 2851 - mutex_init(&ctl_mutex); 2852 2852 - 2853 2853 - ret = mempool_init_kmalloc_pool(&psd_pool, PSD_POOL_SIZE, 2854 2854 - sizeof(struct packet_stacked_data)); 2855 2855 - if (ret) 2856 2856 - return ret; 2857 2857 - ret = bioset_init(&pkt_bio_set, BIO_POOL_SIZE, 0, 0); 2858 2858 - if (ret) { 2859 2859 - mempool_exit(&psd_pool); 2860 2860 - return ret; 2861 2861 - } 2862 2862 - 2863 2863 - ret = register_blkdev(pktdev_major, DRIVER_NAME); 2864 2864 - if (ret < 0) { 2865 2865 - pr_err("unable to register block device\n"); 2866 2866 - goto out2; 2867 2867 - } 2868 2868 - if (!pktdev_major) 2869 2869 - pktdev_major = ret; 2870 2870 - 2871 2871 - ret = pkt_sysfs_init(); 2872 2872 - if (ret) 2873 2873 - goto out; 2874 2874 - 2875 2875 - pkt_debugfs_init(); 2876 2876 - 2877 2877 - ret = misc_register(&pkt_misc); 2878 2878 - if (ret) { 2879 2879 - pr_err("unable to register misc device\n"); 2880 2880 - goto out_misc; 2881 2881 - } 2882 2882 - 2883 2883 - pkt_proc = proc_mkdir("driver/"DRIVER_NAME, NULL); 2884 2884 - 2885 2885 - return 0; 2886 2886 - 2887 2887 - out_misc: 2888 2888 - pkt_debugfs_cleanup(); 2889 2889 - pkt_sysfs_cleanup(); 2890 2890 - out: 2891 2891 - unregister_blkdev(pktdev_major, DRIVER_NAME); 2892 2892 - out2: 2893 2893 - mempool_exit(&psd_pool); 2894 2894 - bioset_exit(&pkt_bio_set); 2895 2895 - return ret; 2896 2896 - } 2897 2897 - 2898 2898 - static void __exit pkt_exit(void) 2899 2899 - { 2900 2900 - remove_proc_entry("driver/"DRIVER_NAME, NULL); 2901 2901 - misc_deregister(&pkt_misc); 2902 2902 - 2903 2903 - pkt_debugfs_cleanup(); 2904 2904 - pkt_sysfs_cleanup(); 2905 2905 - 2906 2906 - unregister_blkdev(pktdev_major, DRIVER_NAME); 2907 2907 - mempool_exit(&psd_pool); 2908 2908 - bioset_exit(&pkt_bio_set); 2909 2909 - } 2910 2910 - 2911 2911 - MODULE_DESCRIPTION("Packet writing layer for CD/DVD drives"); 2912 2912 - MODULE_AUTHOR("Jens Axboe <axboe@suse.de>"); 2913 2913 - MODULE_LICENSE("GPL"); 2914 2914 - 2915 2915 - module_init(pkt_init); 2916 2916 - module_exit(pkt_exit);

+3 -1

drivers/block/sunvdc.c

reviewed

··· 957 957 dev = device_find_child(vdev->dev.parent, &port_data, 958 958 vdc_device_probed); 959 959 960 960 - if (dev) 960 960 + if (dev) { 961 961 + put_device(dev); 961 962 return true; 963 963 + } 962 964 963 965 return false; 964 966 }

+358 -224

drivers/block/ublk_drv.c

reviewed

··· 48 48 49 49 #define UBLK_MINORS (1U << MINORBITS) 50 50 51 51 + #define UBLK_INVALID_BUF_IDX ((u16)-1) 52 52 + 51 53 /* private ioctl command mirror */ 52 54 #define UBLK_CMD_DEL_DEV_ASYNC _IOC_NR(UBLK_U_CMD_DEL_DEV_ASYNC) 53 55 #define UBLK_CMD_UPDATE_SIZE _IOC_NR(UBLK_U_CMD_UPDATE_SIZE) ··· 72 70 | UBLK_F_UPDATE_SIZE \ 73 71 | UBLK_F_AUTO_BUF_REG \ 74 72 | UBLK_F_QUIESCE \ 75 75 - | UBLK_F_PER_IO_DAEMON) 73 73 + | UBLK_F_PER_IO_DAEMON \ 74 74 + | UBLK_F_BUF_REG_OFF_DAEMON) 76 75 77 76 #define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \ 78 77 | UBLK_F_USER_RECOVERY_REISSUE \ ··· 84 81 (UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DISCARD | \ 85 82 UBLK_PARAM_TYPE_DEVT | UBLK_PARAM_TYPE_ZONED | \ 86 83 UBLK_PARAM_TYPE_DMA_ALIGN | UBLK_PARAM_TYPE_SEGMENT) 87 87 - 88 88 - struct ublk_rq_data { 89 89 - refcount_t ref; 90 90 - 91 91 - /* for auto-unregister buffer in case of UBLK_F_AUTO_BUF_REG */ 92 92 - u16 buf_index; 93 93 - void *buf_ctx_handle; 94 94 - }; 95 84 96 85 struct ublk_uring_cmd_pdu { 97 86 /* ··· 104 109 * setup in ublk uring_cmd handler 105 110 */ 106 111 struct ublk_queue *ubq; 107 107 - 108 108 - struct ublk_auto_buf_reg buf; 109 112 110 113 u16 tag; 111 114 }; ··· 148 155 /* atomic RW with ubq->cancel_lock */ 149 156 #define UBLK_IO_FLAG_CANCELED 0x80000000 150 157 158 158 + /* 159 159 + * Initialize refcount to a large number to include any registered buffers. 160 160 + * UBLK_IO_COMMIT_AND_FETCH_REQ will release these references minus those for 161 161 + * any buffers registered on the io daemon task. 162 162 + */ 163 163 + #define UBLK_REFCOUNT_INIT (REFCOUNT_MAX / 2) 164 164 + 151 165 struct ublk_io { 152 166 /* userspace buffer address from io cmd */ 153 153 - __u64 addr; 167 167 + union { 168 168 + __u64 addr; 169 169 + struct ublk_auto_buf_reg buf; 170 170 + }; 154 171 unsigned int flags; 155 172 int res; 156 173 ··· 172 169 }; 173 170 174 171 struct task_struct *task; 175 175 - }; 172 172 + 173 173 + /* 174 174 + * The number of uses of this I/O by the ublk server 175 175 + * if user copy or zero copy are enabled: 176 176 + * - UBLK_REFCOUNT_INIT from dispatch to the server 177 177 + * until UBLK_IO_COMMIT_AND_FETCH_REQ 178 178 + * - 1 for each inflight ublk_ch_{read,write}_iter() call 179 179 + * - 1 for each io_uring registered buffer not registered on task 180 180 + * The I/O can only be completed once all references are dropped. 181 181 + * User copy and buffer registration operations are only permitted 182 182 + * if the reference count is nonzero. 183 183 + */ 184 184 + refcount_t ref; 185 185 + /* Count of buffers registered on task and not yet unregistered */ 186 186 + unsigned task_registered_buffers; 187 187 + 188 188 + void *buf_ctx_handle; 189 189 + } ____cacheline_aligned_in_smp; 176 190 177 191 struct ublk_queue { 178 192 int q_id; ··· 236 216 struct completion completion; 237 217 unsigned int nr_queues_ready; 238 218 unsigned int nr_privileged_daemon; 219 219 + struct mutex cancel_mutex; 220 220 + bool canceling; 221 221 + pid_t ublksrv_tgid; 239 222 }; 240 223 241 224 /* header of ublk_params */ ··· 251 228 static void ublk_stop_dev_unlocked(struct ublk_device *ub); 252 229 static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq); 253 230 static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub, 254 254 - const struct ublk_queue *ubq, int tag, size_t offset); 231 231 + const struct ublk_queue *ubq, struct ublk_io *io, 232 232 + size_t offset); 255 233 static inline unsigned int ublk_req_build_flags(struct request *req); 256 234 257 235 static inline struct ublksrv_io_desc * ··· 697 673 } 698 674 699 675 static inline void ublk_init_req_ref(const struct ublk_queue *ubq, 700 700 - struct request *req) 676 676 + struct ublk_io *io) 701 677 { 702 702 - if (ublk_need_req_ref(ubq)) { 703 703 - struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); 704 704 - 705 705 - refcount_set(&data->ref, 1); 706 706 - } 678 678 + if (ublk_need_req_ref(ubq)) 679 679 + refcount_set(&io->ref, UBLK_REFCOUNT_INIT); 707 680 } 708 681 709 709 - static inline bool ublk_get_req_ref(const struct ublk_queue *ubq, 710 710 - struct request *req) 682 682 + static inline bool ublk_get_req_ref(struct ublk_io *io) 711 683 { 712 712 - if (ublk_need_req_ref(ubq)) { 713 713 - struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); 714 714 - 715 715 - return refcount_inc_not_zero(&data->ref); 716 716 - } 717 717 - 718 718 - return true; 684 684 + return refcount_inc_not_zero(&io->ref); 719 685 } 720 686 721 721 - static inline void ublk_put_req_ref(const struct ublk_queue *ubq, 722 722 - struct request *req) 687 687 + static inline void ublk_put_req_ref(struct ublk_io *io, struct request *req) 723 688 { 724 724 - if (ublk_need_req_ref(ubq)) { 725 725 - struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); 726 726 - 727 727 - if (refcount_dec_and_test(&data->ref)) 728 728 - __ublk_complete_rq(req); 729 729 - } else { 689 689 + if (refcount_dec_and_test(&io->ref)) 730 690 __ublk_complete_rq(req); 731 731 - } 691 691 + } 692 692 + 693 693 + static inline bool ublk_sub_req_ref(struct ublk_io *io) 694 694 + { 695 695 + unsigned sub_refs = UBLK_REFCOUNT_INIT - io->task_registered_buffers; 696 696 + 697 697 + io->task_registered_buffers = 0; 698 698 + return refcount_sub_and_test(sub_refs, &io->ref); 732 699 } 733 700 734 701 static inline bool ublk_need_get_data(const struct ublk_queue *ubq) ··· 996 981 } 997 982 998 983 static int ublk_map_io(const struct ublk_queue *ubq, const struct request *req, 999 999 - struct ublk_io *io) 984 984 + const struct ublk_io *io) 1000 985 { 1001 986 const unsigned int rq_bytes = blk_rq_bytes(req); 1002 987 ··· 1020 1005 1021 1006 static int ublk_unmap_io(const struct ublk_queue *ubq, 1022 1007 const struct request *req, 1023 1023 - struct ublk_io *io) 1008 1008 + const struct ublk_io *io) 1024 1009 { 1025 1010 const unsigned int rq_bytes = blk_rq_bytes(req); 1026 1011 ··· 1155 1140 1156 1141 if (blk_update_request(req, BLK_STS_OK, io->res)) 1157 1142 blk_mq_requeue_request(req, true); 1158 1158 - else 1143 1143 + else if (likely(!blk_should_fake_timeout(req->q))) 1159 1144 __blk_mq_end_request(req, BLK_STS_OK); 1160 1145 1161 1146 return; ··· 1203 1188 blk_mq_end_request(rq, BLK_STS_IOERR); 1204 1189 } 1205 1190 1206 1206 - static void ublk_auto_buf_reg_fallback(struct request *req) 1191 1191 + static void 1192 1192 + ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, struct ublk_io *io) 1207 1193 { 1208 1208 - const struct ublk_queue *ubq = req->mq_hctx->driver_data; 1209 1209 - struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag); 1210 1210 - struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); 1194 1194 + unsigned tag = io - ubq->ios; 1195 1195 + struct ublksrv_io_desc *iod = ublk_get_iod(ubq, tag); 1211 1196 1212 1197 iod->op_flags |= UBLK_IO_F_NEED_REG_BUF; 1213 1213 - refcount_set(&data->ref, 1); 1214 1198 } 1215 1199 1216 1216 - static bool ublk_auto_buf_reg(struct request *req, struct ublk_io *io, 1217 1217 - unsigned int issue_flags) 1200 1200 + static bool ublk_auto_buf_reg(const struct ublk_queue *ubq, struct request *req, 1201 1201 + struct ublk_io *io, unsigned int issue_flags) 1218 1202 { 1219 1219 - struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(io->cmd); 1220 1220 - struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); 1221 1203 int ret; 1222 1204 1223 1205 ret = io_buffer_register_bvec(io->cmd, req, ublk_io_release, 1224 1224 - pdu->buf.index, issue_flags); 1206 1206 + io->buf.index, issue_flags); 1225 1207 if (ret) { 1226 1226 - if (pdu->buf.flags & UBLK_AUTO_BUF_REG_FALLBACK) { 1227 1227 - ublk_auto_buf_reg_fallback(req); 1208 1208 + if (io->buf.flags & UBLK_AUTO_BUF_REG_FALLBACK) { 1209 1209 + ublk_auto_buf_reg_fallback(ubq, io); 1228 1210 return true; 1229 1211 } 1230 1212 blk_mq_end_request(req, BLK_STS_IOERR); 1231 1213 return false; 1232 1214 } 1233 1233 - /* one extra reference is dropped by ublk_io_release */ 1234 1234 - refcount_set(&data->ref, 2); 1235 1215 1236 1236 - data->buf_ctx_handle = io_uring_cmd_ctx_handle(io->cmd); 1237 1237 - /* store buffer index in request payload */ 1238 1238 - data->buf_index = pdu->buf.index; 1216 1216 + io->task_registered_buffers = 1; 1217 1217 + io->buf_ctx_handle = io_uring_cmd_ctx_handle(io->cmd); 1239 1218 io->flags |= UBLK_IO_FLAG_AUTO_BUF_REG; 1240 1219 return true; 1241 1220 } ··· 1238 1229 struct request *req, struct ublk_io *io, 1239 1230 unsigned int issue_flags) 1240 1231 { 1232 1232 + ublk_init_req_ref(ubq, io); 1241 1233 if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req)) 1242 1242 - return ublk_auto_buf_reg(req, io, issue_flags); 1234 1234 + return ublk_auto_buf_reg(ubq, req, io, issue_flags); 1243 1235 1244 1244 - ublk_init_req_ref(ubq, req); 1245 1236 return true; 1246 1237 } 1247 1238 ··· 1365 1356 static enum blk_eh_timer_return ublk_timeout(struct request *rq) 1366 1357 { 1367 1358 struct ublk_queue *ubq = rq->mq_hctx->driver_data; 1368 1368 - struct ublk_io *io = &ubq->ios[rq->tag]; 1359 1359 + pid_t tgid = ubq->dev->ublksrv_tgid; 1360 1360 + struct task_struct *p; 1361 1361 + struct pid *pid; 1369 1362 1370 1370 - if (ubq->flags & UBLK_F_UNPRIVILEGED_DEV) { 1371 1371 - send_sig(SIGKILL, io->task, 0); 1372 1372 - return BLK_EH_DONE; 1373 1373 - } 1363 1363 + if (!(ubq->flags & UBLK_F_UNPRIVILEGED_DEV)) 1364 1364 + return BLK_EH_RESET_TIMER; 1374 1365 1375 1375 - return BLK_EH_RESET_TIMER; 1366 1366 + if (unlikely(!tgid)) 1367 1367 + return BLK_EH_RESET_TIMER; 1368 1368 + 1369 1369 + rcu_read_lock(); 1370 1370 + pid = find_vpid(tgid); 1371 1371 + p = pid_task(pid, PIDTYPE_PID); 1372 1372 + if (p) 1373 1373 + send_sig(SIGKILL, p, 0); 1374 1374 + rcu_read_unlock(); 1375 1375 + return BLK_EH_DONE; 1376 1376 } 1377 1377 1378 1378 static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq, ··· 1522 1504 put_task_struct(io->task); 1523 1505 io->task = NULL; 1524 1506 } 1507 1507 + 1508 1508 + WARN_ON_ONCE(refcount_read(&io->ref)); 1509 1509 + WARN_ON_ONCE(io->task_registered_buffers); 1525 1510 } 1526 1511 } 1527 1512 ··· 1536 1515 if (test_and_set_bit(UB_STATE_OPEN, &ub->state)) 1537 1516 return -EBUSY; 1538 1517 filp->private_data = ub; 1518 1518 + ub->ublksrv_tgid = current->tgid; 1539 1519 return 0; 1540 1520 } 1541 1521 ··· 1551 1529 ub->mm = NULL; 1552 1530 ub->nr_queues_ready = 0; 1553 1531 ub->nr_privileged_daemon = 0; 1532 1532 + ub->ublksrv_tgid = -1; 1554 1533 } 1555 1534 1556 1535 static struct gendisk *ublk_get_disk(struct ublk_device *ub) ··· 1571 1548 { 1572 1549 if (disk) 1573 1550 put_device(disk_to_dev(disk)); 1551 1551 + } 1552 1552 + 1553 1553 + /* 1554 1554 + * Use this function to ensure that ->canceling is consistently set for 1555 1555 + * the device and all queues. Do not set these flags directly. 1556 1556 + * 1557 1557 + * Caller must ensure that: 1558 1558 + * - cancel_mutex is held. This ensures that there is no concurrent 1559 1559 + * access to ub->canceling and no concurrent writes to ubq->canceling. 1560 1560 + * - there are no concurrent reads of ubq->canceling from the queue_rq 1561 1561 + * path. This can be done by quiescing the queue, or through other 1562 1562 + * means. 1563 1563 + */ 1564 1564 + static void ublk_set_canceling(struct ublk_device *ub, bool canceling) 1565 1565 + __must_hold(&ub->cancel_mutex) 1566 1566 + { 1567 1567 + int i; 1568 1568 + 1569 1569 + ub->canceling = canceling; 1570 1570 + for (i = 0; i < ub->dev_info.nr_hw_queues; i++) 1571 1571 + ublk_get_queue(ub, i)->canceling = canceling; 1574 1572 } 1575 1573 1576 1574 static int ublk_ch_release(struct inode *inode, struct file *filp) ··· 1622 1578 * All requests may be inflight, so ->canceling may not be set, set 1623 1579 * it now. 1624 1580 */ 1625 1625 - for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { 1626 1626 - struct ublk_queue *ubq = ublk_get_queue(ub, i); 1627 1627 - 1628 1628 - ubq->canceling = true; 1629 1629 - ublk_abort_queue(ub, ubq); 1630 1630 - } 1581 1581 + mutex_lock(&ub->cancel_mutex); 1582 1582 + ublk_set_canceling(ub, true); 1583 1583 + for (i = 0; i < ub->dev_info.nr_hw_queues; i++) 1584 1584 + ublk_abort_queue(ub, ublk_get_queue(ub, i)); 1585 1585 + mutex_unlock(&ub->cancel_mutex); 1631 1586 blk_mq_kick_requeue_list(disk->queue); 1632 1587 1633 1588 /* ··· 1749 1706 } 1750 1707 } 1751 1708 1752 1752 - /* Must be called when queue is frozen */ 1753 1753 - static void ublk_mark_queue_canceling(struct ublk_queue *ubq) 1709 1709 + static void ublk_start_cancel(struct ublk_device *ub) 1754 1710 { 1755 1755 - spin_lock(&ubq->cancel_lock); 1756 1756 - if (!ubq->canceling) 1757 1757 - ubq->canceling = true; 1758 1758 - spin_unlock(&ubq->cancel_lock); 1759 1759 - } 1760 1760 - 1761 1761 - static void ublk_start_cancel(struct ublk_queue *ubq) 1762 1762 - { 1763 1763 - struct ublk_device *ub = ubq->dev; 1764 1711 struct gendisk *disk = ublk_get_disk(ub); 1765 1712 1766 1713 /* Our disk has been dead */ 1767 1714 if (!disk) 1768 1715 return; 1716 1716 + 1717 1717 + mutex_lock(&ub->cancel_mutex); 1718 1718 + if (ub->canceling) 1719 1719 + goto out; 1769 1720 /* 1770 1721 * Now we are serialized with ublk_queue_rq() 1771 1722 * ··· 1768 1731 * touch completed uring_cmd 1769 1732 */ 1770 1733 blk_mq_quiesce_queue(disk->queue); 1771 1771 - ublk_mark_queue_canceling(ubq); 1734 1734 + ublk_set_canceling(ub, true); 1772 1735 blk_mq_unquiesce_queue(disk->queue); 1736 1736 + out: 1737 1737 + mutex_unlock(&ub->cancel_mutex); 1773 1738 ublk_put_disk(disk); 1774 1739 } 1775 1740 ··· 1844 1805 if (WARN_ON_ONCE(task && task != io->task)) 1845 1806 return; 1846 1807 1847 1847 - if (!ubq->canceling) 1848 1848 - ublk_start_cancel(ubq); 1808 1808 + ublk_start_cancel(ubq->dev); 1849 1809 1850 1810 WARN_ON_ONCE(io->cmd != cmd); 1851 1811 ublk_cancel_cmd(ubq, pdu->tag, issue_flags); ··· 1968 1930 for (j = 0; j < ubq->q_depth; j++) 1969 1931 ubq->ios[j].flags &= ~UBLK_IO_FLAG_CANCELED; 1970 1932 spin_unlock(&ubq->cancel_lock); 1971 1971 - ubq->canceling = false; 1972 1933 ubq->fail_io = false; 1973 1934 } 1935 1935 + mutex_lock(&ub->cancel_mutex); 1936 1936 + ublk_set_canceling(ub, false); 1937 1937 + mutex_unlock(&ub->cancel_mutex); 1974 1938 } 1975 1939 1976 1940 /* device can only be started after all IOs are ready */ ··· 2007 1967 return 0; 2008 1968 } 2009 1969 2010 2010 - static inline void ublk_fill_io_cmd(struct ublk_io *io, 2011 2011 - struct io_uring_cmd *cmd, unsigned long buf_addr) 1970 1970 + static inline int ublk_set_auto_buf_reg(struct ublk_io *io, struct io_uring_cmd *cmd) 2012 1971 { 1972 1972 + io->buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); 1973 1973 + 1974 1974 + if (io->buf.reserved0 || io->buf.reserved1) 1975 1975 + return -EINVAL; 1976 1976 + 1977 1977 + if (io->buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK) 1978 1978 + return -EINVAL; 1979 1979 + return 0; 1980 1980 + } 1981 1981 + 1982 1982 + static int ublk_handle_auto_buf_reg(struct ublk_io *io, 1983 1983 + struct io_uring_cmd *cmd, 1984 1984 + u16 *buf_idx) 1985 1985 + { 1986 1986 + if (io->flags & UBLK_IO_FLAG_AUTO_BUF_REG) { 1987 1987 + io->flags &= ~UBLK_IO_FLAG_AUTO_BUF_REG; 1988 1988 + 1989 1989 + /* 1990 1990 + * `UBLK_F_AUTO_BUF_REG` only works iff `UBLK_IO_FETCH_REQ` 1991 1991 + * and `UBLK_IO_COMMIT_AND_FETCH_REQ` are issued from same 1992 1992 + * `io_ring_ctx`. 1993 1993 + * 1994 1994 + * If this uring_cmd's io_ring_ctx isn't same with the 1995 1995 + * one for registering the buffer, it is ublk server's 1996 1996 + * responsibility for unregistering the buffer, otherwise 1997 1997 + * this ublk request gets stuck. 1998 1998 + */ 1999 1999 + if (io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) 2000 2000 + *buf_idx = io->buf.index; 2001 2001 + } 2002 2002 + 2003 2003 + return ublk_set_auto_buf_reg(io, cmd); 2004 2004 + } 2005 2005 + 2006 2006 + /* Once we return, `io->req` can't be used any more */ 2007 2007 + static inline struct request * 2008 2008 + ublk_fill_io_cmd(struct ublk_io *io, struct io_uring_cmd *cmd) 2009 2009 + { 2010 2010 + struct request *req = io->req; 2011 2011 + 2013 2012 io->cmd = cmd; 2014 2013 io->flags |= UBLK_IO_FLAG_ACTIVE; 2014 2014 + /* now this cmd slot is owned by ublk driver */ 2015 2015 + io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV; 2016 2016 + 2017 2017 + return req; 2018 2018 + } 2019 2019 + 2020 2020 + static inline int 2021 2021 + ublk_config_io_buf(const struct ublk_queue *ubq, struct ublk_io *io, 2022 2022 + struct io_uring_cmd *cmd, unsigned long buf_addr, 2023 2023 + u16 *buf_idx) 2024 2024 + { 2025 2025 + if (ublk_support_auto_buf_reg(ubq)) 2026 2026 + return ublk_handle_auto_buf_reg(io, cmd, buf_idx); 2027 2027 + 2015 2028 io->addr = buf_addr; 2029 2029 + return 0; 2016 2030 } 2017 2031 2018 2032 static inline void ublk_prep_cancel(struct io_uring_cmd *cmd, ··· 2084 1990 io_uring_cmd_mark_cancelable(cmd, issue_flags); 2085 1991 } 2086 1992 2087 2087 - static inline int ublk_set_auto_buf_reg(struct io_uring_cmd *cmd) 2088 2088 - { 2089 2089 - struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); 2090 2090 - 2091 2091 - pdu->buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); 2092 2092 - 2093 2093 - if (pdu->buf.reserved0 || pdu->buf.reserved1) 2094 2094 - return -EINVAL; 2095 2095 - 2096 2096 - if (pdu->buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK) 2097 2097 - return -EINVAL; 2098 2098 - return 0; 2099 2099 - } 2100 2100 - 2101 1993 static void ublk_io_release(void *priv) 2102 1994 { 2103 1995 struct request *rq = priv; 2104 1996 struct ublk_queue *ubq = rq->mq_hctx->driver_data; 1997 1997 + struct ublk_io *io = &ubq->ios[rq->tag]; 2105 1998 2106 2106 - ublk_put_req_ref(ubq, rq); 1999 1999 + /* 2000 2000 + * task_registered_buffers may be 0 if buffers were registered off task 2001 2001 + * but unregistered on task. Or after UBLK_IO_COMMIT_AND_FETCH_REQ. 2002 2002 + */ 2003 2003 + if (current == io->task && io->task_registered_buffers) 2004 2004 + io->task_registered_buffers--; 2005 2005 + else 2006 2006 + ublk_put_req_ref(io, rq); 2107 2007 } 2108 2008 2109 2009 static int ublk_register_io_buf(struct io_uring_cmd *cmd, 2110 2110 - const struct ublk_queue *ubq, unsigned int tag, 2010 2010 + const struct ublk_queue *ubq, 2011 2011 + struct ublk_io *io, 2111 2012 unsigned int index, unsigned int issue_flags) 2112 2013 { 2113 2014 struct ublk_device *ub = cmd->file->private_data; ··· 2112 2023 if (!ublk_support_zero_copy(ubq)) 2113 2024 return -EINVAL; 2114 2025 2115 2115 - req = __ublk_check_and_get_req(ub, ubq, tag, 0); 2026 2026 + req = __ublk_check_and_get_req(ub, ubq, io, 0); 2116 2027 if (!req) 2117 2028 return -EINVAL; 2118 2029 2119 2030 ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index, 2120 2031 issue_flags); 2121 2032 if (ret) { 2122 2122 - ublk_put_req_ref(ubq, req); 2033 2033 + ublk_put_req_ref(io, req); 2123 2034 return ret; 2124 2035 } 2125 2036 2126 2037 return 0; 2127 2038 } 2128 2039 2040 2040 + static int 2041 2041 + ublk_daemon_register_io_buf(struct io_uring_cmd *cmd, 2042 2042 + const struct ublk_queue *ubq, struct ublk_io *io, 2043 2043 + unsigned index, unsigned issue_flags) 2044 2044 + { 2045 2045 + unsigned new_registered_buffers; 2046 2046 + struct request *req = io->req; 2047 2047 + int ret; 2048 2048 + 2049 2049 + /* 2050 2050 + * Ensure there are still references for ublk_sub_req_ref() to release. 2051 2051 + * If not, fall back on the thread-safe buffer registration. 2052 2052 + */ 2053 2053 + new_registered_buffers = io->task_registered_buffers + 1; 2054 2054 + if (unlikely(new_registered_buffers >= UBLK_REFCOUNT_INIT)) 2055 2055 + return ublk_register_io_buf(cmd, ubq, io, index, issue_flags); 2056 2056 + 2057 2057 + if (!ublk_support_zero_copy(ubq) || !ublk_rq_has_data(req)) 2058 2058 + return -EINVAL; 2059 2059 + 2060 2060 + ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index, 2061 2061 + issue_flags); 2062 2062 + if (ret) 2063 2063 + return ret; 2064 2064 + 2065 2065 + io->task_registered_buffers = new_registered_buffers; 2066 2066 + return 0; 2067 2067 + } 2068 2068 + 2129 2069 static int ublk_unregister_io_buf(struct io_uring_cmd *cmd, 2130 2130 - const struct ublk_queue *ubq, 2070 2070 + const struct ublk_device *ub, 2131 2071 unsigned int index, unsigned int issue_flags) 2132 2072 { 2133 2133 - if (!ublk_support_zero_copy(ubq)) 2073 2073 + if (!(ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY)) 2134 2074 return -EINVAL; 2135 2075 2136 2076 return io_buffer_unregister_bvec(cmd, index, issue_flags); 2077 2077 + } 2078 2078 + 2079 2079 + static int ublk_check_fetch_buf(const struct ublk_queue *ubq, __u64 buf_addr) 2080 2080 + { 2081 2081 + if (ublk_need_map_io(ubq)) { 2082 2082 + /* 2083 2083 + * FETCH_RQ has to provide IO buffer if NEED GET 2084 2084 + * DATA is not enabled 2085 2085 + */ 2086 2086 + if (!buf_addr && !ublk_need_get_data(ubq)) 2087 2087 + return -EINVAL; 2088 2088 + } else if (buf_addr) { 2089 2089 + /* User copy requires addr to be unset */ 2090 2090 + return -EINVAL; 2091 2091 + } 2092 2092 + return 0; 2137 2093 } 2138 2094 2139 2095 static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_queue *ubq, ··· 2207 2073 2208 2074 WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV); 2209 2075 2210 2210 - if (ublk_need_map_io(ubq)) { 2211 2211 - /* 2212 2212 - * FETCH_RQ has to provide IO buffer if NEED GET 2213 2213 - * DATA is not enabled 2214 2214 - */ 2215 2215 - if (!buf_addr && !ublk_need_get_data(ubq)) 2216 2216 - goto out; 2217 2217 - } else if (buf_addr) { 2218 2218 - /* User copy requires addr to be unset */ 2219 2219 - ret = -EINVAL; 2076 2076 + ublk_fill_io_cmd(io, cmd); 2077 2077 + ret = ublk_config_io_buf(ubq, io, cmd, buf_addr, NULL); 2078 2078 + if (ret) 2220 2079 goto out; 2221 2221 - } 2222 2080 2223 2223 - if (ublk_support_auto_buf_reg(ubq)) { 2224 2224 - ret = ublk_set_auto_buf_reg(cmd); 2225 2225 - if (ret) 2226 2226 - goto out; 2227 2227 - } 2228 2228 - 2229 2229 - ublk_fill_io_cmd(io, cmd, buf_addr); 2230 2081 WRITE_ONCE(io->task, get_task_struct(current)); 2231 2082 ublk_mark_io_ready(ub, ubq); 2232 2083 out: ··· 2219 2100 return ret; 2220 2101 } 2221 2102 2222 2222 - static int ublk_commit_and_fetch(const struct ublk_queue *ubq, 2223 2223 - struct ublk_io *io, struct io_uring_cmd *cmd, 2224 2224 - const struct ublksrv_io_cmd *ub_cmd, 2225 2225 - unsigned int issue_flags) 2103 2103 + static int ublk_check_commit_and_fetch(const struct ublk_queue *ubq, 2104 2104 + struct ublk_io *io, __u64 buf_addr) 2226 2105 { 2227 2106 struct request *req = io->req; 2228 2107 ··· 2229 2112 * COMMIT_AND_FETCH_REQ has to provide IO buffer if 2230 2113 * NEED GET DATA is not enabled or it is Read IO. 2231 2114 */ 2232 2232 - if (!ub_cmd->addr && (!ublk_need_get_data(ubq) || 2115 2115 + if (!buf_addr && (!ublk_need_get_data(ubq) || 2233 2116 req_op(req) == REQ_OP_READ)) 2234 2117 return -EINVAL; 2235 2235 - } else if (req_op(req) != REQ_OP_ZONE_APPEND && ub_cmd->addr) { 2118 2118 + } else if (req_op(req) != REQ_OP_ZONE_APPEND && buf_addr) { 2236 2119 /* 2237 2120 * User copy requires addr to be unset when command is 2238 2121 * not zone append ··· 2240 2123 return -EINVAL; 2241 2124 } 2242 2125 2243 2243 - if (ublk_support_auto_buf_reg(ubq)) { 2244 2244 - int ret; 2245 2245 - 2246 2246 - /* 2247 2247 - * `UBLK_F_AUTO_BUF_REG` only works iff `UBLK_IO_FETCH_REQ` 2248 2248 - * and `UBLK_IO_COMMIT_AND_FETCH_REQ` are issued from same 2249 2249 - * `io_ring_ctx`. 2250 2250 - * 2251 2251 - * If this uring_cmd's io_ring_ctx isn't same with the 2252 2252 - * one for registering the buffer, it is ublk server's 2253 2253 - * responsibility for unregistering the buffer, otherwise 2254 2254 - * this ublk request gets stuck. 2255 2255 - */ 2256 2256 - if (io->flags & UBLK_IO_FLAG_AUTO_BUF_REG) { 2257 2257 - struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); 2258 2258 - 2259 2259 - if (data->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) 2260 2260 - io_buffer_unregister_bvec(cmd, data->buf_index, 2261 2261 - issue_flags); 2262 2262 - io->flags &= ~UBLK_IO_FLAG_AUTO_BUF_REG; 2263 2263 - } 2264 2264 - 2265 2265 - ret = ublk_set_auto_buf_reg(cmd); 2266 2266 - if (ret) 2267 2267 - return ret; 2268 2268 - } 2269 2269 - 2270 2270 - ublk_fill_io_cmd(io, cmd, ub_cmd->addr); 2271 2271 - 2272 2272 - /* now this cmd slot is owned by ublk driver */ 2273 2273 - io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV; 2274 2274 - io->res = ub_cmd->result; 2275 2275 - 2276 2276 - if (req_op(req) == REQ_OP_ZONE_APPEND) 2277 2277 - req->__sector = ub_cmd->zone_append_lba; 2278 2278 - 2279 2279 - if (likely(!blk_should_fake_timeout(req->q))) 2280 2280 - ublk_put_req_ref(ubq, req); 2281 2281 - 2282 2126 return 0; 2127 2127 + } 2128 2128 + 2129 2129 + static bool ublk_need_complete_req(const struct ublk_queue *ubq, 2130 2130 + struct ublk_io *io) 2131 2131 + { 2132 2132 + if (ublk_need_req_ref(ubq)) 2133 2133 + return ublk_sub_req_ref(io); 2134 2134 + return true; 2283 2135 } 2284 2136 2285 2137 static bool ublk_get_data(const struct ublk_queue *ubq, struct ublk_io *io, ··· 2273 2187 unsigned int issue_flags, 2274 2188 const struct ublksrv_io_cmd *ub_cmd) 2275 2189 { 2190 2190 + u16 buf_idx = UBLK_INVALID_BUF_IDX; 2276 2191 struct ublk_device *ub = cmd->file->private_data; 2277 2277 - struct task_struct *task; 2278 2192 struct ublk_queue *ubq; 2279 2193 struct ublk_io *io; 2280 2194 u32 cmd_op = cmd->cmd_op; 2281 2195 unsigned tag = ub_cmd->tag; 2282 2282 - int ret = -EINVAL; 2283 2196 struct request *req; 2197 2197 + int ret; 2198 2198 + bool compl; 2284 2199 2285 2200 pr_devel("%s: received: cmd op %d queue %d tag %d result %d\n", 2286 2201 __func__, cmd->cmd_op, ub_cmd->q_id, tag, 2287 2202 ub_cmd->result); 2288 2203 2204 2204 + ret = ublk_check_cmd_op(cmd_op); 2205 2205 + if (ret) 2206 2206 + goto out; 2207 2207 + 2208 2208 + /* 2209 2209 + * io_buffer_unregister_bvec() doesn't access the ubq or io, 2210 2210 + * so no need to validate the q_id, tag, or task 2211 2211 + */ 2212 2212 + if (_IOC_NR(cmd_op) == UBLK_IO_UNREGISTER_IO_BUF) 2213 2213 + return ublk_unregister_io_buf(cmd, ub, ub_cmd->addr, 2214 2214 + issue_flags); 2215 2215 + 2216 2216 + ret = -EINVAL; 2289 2217 if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues) 2290 2218 goto out; 2291 2219 ··· 2309 2209 goto out; 2310 2210 2311 2211 io = &ubq->ios[tag]; 2312 2312 - task = READ_ONCE(io->task); 2313 2313 - if (task && task != current) 2314 2314 - goto out; 2212 2212 + /* UBLK_IO_FETCH_REQ can be handled on any task, which sets io->task */ 2213 2213 + if (unlikely(_IOC_NR(cmd_op) == UBLK_IO_FETCH_REQ)) { 2214 2214 + ret = ublk_check_fetch_buf(ubq, ub_cmd->addr); 2215 2215 + if (ret) 2216 2216 + goto out; 2217 2217 + ret = ublk_fetch(cmd, ubq, io, ub_cmd->addr); 2218 2218 + if (ret) 2219 2219 + goto out; 2315 2220 2316 2316 - /* there is pending io cmd, something must be wrong */ 2317 2317 - if (io->flags & UBLK_IO_FLAG_ACTIVE) { 2318 2318 - ret = -EBUSY; 2221 2221 + ublk_prep_cancel(cmd, issue_flags, ubq, tag); 2222 2222 + return -EIOCBQUEUED; 2223 2223 + } 2224 2224 + 2225 2225 + if (READ_ONCE(io->task) != current) { 2226 2226 + /* 2227 2227 + * ublk_register_io_buf() accesses only the io's refcount, 2228 2228 + * so can be handled on any task 2229 2229 + */ 2230 2230 + if (_IOC_NR(cmd_op) == UBLK_IO_REGISTER_IO_BUF) 2231 2231 + return ublk_register_io_buf(cmd, ubq, io, ub_cmd->addr, 2232 2232 + issue_flags); 2233 2233 + 2319 2234 goto out; 2320 2235 } 2321 2236 2322 2322 - /* only UBLK_IO_FETCH_REQ is allowed if io is not OWNED_BY_SRV */ 2323 2323 - if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV) && 2324 2324 - _IOC_NR(cmd_op) != UBLK_IO_FETCH_REQ) 2237 2237 + /* there is pending io cmd, something must be wrong */ 2238 2238 + if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) { 2239 2239 + ret = -EBUSY; 2325 2240 goto out; 2241 2241 + } 2326 2242 2327 2243 /* 2328 2244 * ensure that the user issues UBLK_IO_NEED_GET_DATA ··· 2348 2232 ^ (_IOC_NR(cmd_op) == UBLK_IO_NEED_GET_DATA)) 2349 2233 goto out; 2350 2234 2351 2351 - ret = ublk_check_cmd_op(cmd_op); 2352 2352 - if (ret) 2353 2353 - goto out; 2354 2354 - 2355 2355 - ret = -EINVAL; 2356 2235 switch (_IOC_NR(cmd_op)) { 2357 2236 case UBLK_IO_REGISTER_IO_BUF: 2358 2358 - return ublk_register_io_buf(cmd, ubq, tag, ub_cmd->addr, issue_flags); 2359 2359 - case UBLK_IO_UNREGISTER_IO_BUF: 2360 2360 - return ublk_unregister_io_buf(cmd, ubq, ub_cmd->addr, issue_flags); 2361 2361 - case UBLK_IO_FETCH_REQ: 2362 2362 - ret = ublk_fetch(cmd, ubq, io, ub_cmd->addr); 2237 2237 + return ublk_daemon_register_io_buf(cmd, ubq, io, ub_cmd->addr, 2238 2238 + issue_flags); 2239 2239 + case UBLK_IO_COMMIT_AND_FETCH_REQ: 2240 2240 + ret = ublk_check_commit_and_fetch(ubq, io, ub_cmd->addr); 2363 2241 if (ret) 2364 2242 goto out; 2365 2365 - break; 2366 2366 - case UBLK_IO_COMMIT_AND_FETCH_REQ: 2367 2367 - ret = ublk_commit_and_fetch(ubq, io, cmd, ub_cmd, issue_flags); 2243 2243 + io->res = ub_cmd->result; 2244 2244 + req = ublk_fill_io_cmd(io, cmd); 2245 2245 + ret = ublk_config_io_buf(ubq, io, cmd, ub_cmd->addr, &buf_idx); 2246 2246 + compl = ublk_need_complete_req(ubq, io); 2247 2247 + 2248 2248 + /* can't touch 'ublk_io' any more */ 2249 2249 + if (buf_idx != UBLK_INVALID_BUF_IDX) 2250 2250 + io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); 2251 2251 + if (req_op(req) == REQ_OP_ZONE_APPEND) 2252 2252 + req->__sector = ub_cmd->zone_append_lba; 2253 2253 + if (compl) 2254 2254 + __ublk_complete_rq(req); 2255 2255 + 2368 2256 if (ret) 2369 2257 goto out; 2370 2258 break; ··· 2378 2258 * uring_cmd active first and prepare for handling new requeued 2379 2259 * request 2380 2260 */ 2381 2381 - req = io->req; 2382 2382 - ublk_fill_io_cmd(io, cmd, ub_cmd->addr); 2383 2383 - io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV; 2261 2261 + req = ublk_fill_io_cmd(io, cmd); 2262 2262 + ret = ublk_config_io_buf(ubq, io, cmd, ub_cmd->addr, NULL); 2263 2263 + WARN_ON_ONCE(ret); 2384 2264 if (likely(ublk_get_data(ubq, io, req))) { 2385 2265 __ublk_prep_compl_io_cmd(io, req); 2386 2266 return UBLK_IO_RES_OK; ··· 2399 2279 } 2400 2280 2401 2281 static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub, 2402 2402 - const struct ublk_queue *ubq, int tag, size_t offset) 2282 2282 + const struct ublk_queue *ubq, struct ublk_io *io, size_t offset) 2403 2283 { 2284 2284 + unsigned tag = io - ubq->ios; 2404 2285 struct request *req; 2405 2286 2287 2287 + /* 2288 2288 + * can't use io->req in case of concurrent UBLK_IO_COMMIT_AND_FETCH_REQ, 2289 2289 + * which would overwrite it with io->cmd 2290 2290 + */ 2406 2291 req = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], tag); 2407 2292 if (!req) 2408 2293 return NULL; 2409 2294 2410 2410 - if (!ublk_get_req_ref(ubq, req)) 2295 2295 + if (!ublk_get_req_ref(io)) 2411 2296 return NULL; 2412 2297 2413 2298 if (unlikely(!blk_mq_request_started(req) || req->tag != tag)) ··· 2426 2301 2427 2302 return req; 2428 2303 fail_put: 2429 2429 - ublk_put_req_ref(ubq, req); 2304 2304 + ublk_put_req_ref(io, req); 2430 2305 return NULL; 2431 2306 } 2432 2307 ··· 2493 2368 } 2494 2369 2495 2370 static struct request *ublk_check_and_get_req(struct kiocb *iocb, 2496 2496 - struct iov_iter *iter, size_t *off, int dir) 2371 2371 + struct iov_iter *iter, size_t *off, int dir, 2372 2372 + struct ublk_io **io) 2497 2373 { 2498 2374 struct ublk_device *ub = iocb->ki_filp->private_data; 2499 2375 struct ublk_queue *ubq; ··· 2528 2402 if (tag >= ubq->q_depth) 2529 2403 return ERR_PTR(-EINVAL); 2530 2404 2531 2531 - req = __ublk_check_and_get_req(ub, ubq, tag, buf_off); 2405 2405 + *io = &ubq->ios[tag]; 2406 2406 + req = __ublk_check_and_get_req(ub, ubq, *io, buf_off); 2532 2407 if (!req) 2533 2408 return ERR_PTR(-EINVAL); 2534 2409 ··· 2542 2415 *off = buf_off; 2543 2416 return req; 2544 2417 fail: 2545 2545 - ublk_put_req_ref(ubq, req); 2418 2418 + ublk_put_req_ref(*io, req); 2546 2419 return ERR_PTR(-EACCES); 2547 2420 } 2548 2421 2549 2422 static ssize_t ublk_ch_read_iter(struct kiocb *iocb, struct iov_iter *to) 2550 2423 { 2551 2551 - struct ublk_queue *ubq; 2552 2424 struct request *req; 2425 2425 + struct ublk_io *io; 2553 2426 size_t buf_off; 2554 2427 size_t ret; 2555 2428 2556 2556 - req = ublk_check_and_get_req(iocb, to, &buf_off, ITER_DEST); 2429 2429 + req = ublk_check_and_get_req(iocb, to, &buf_off, ITER_DEST, &io); 2557 2430 if (IS_ERR(req)) 2558 2431 return PTR_ERR(req); 2559 2432 2560 2433 ret = ublk_copy_user_pages(req, buf_off, to, ITER_DEST); 2561 2561 - ubq = req->mq_hctx->driver_data; 2562 2562 - ublk_put_req_ref(ubq, req); 2434 2434 + ublk_put_req_ref(io, req); 2563 2435 2564 2436 return ret; 2565 2437 } 2566 2438 2567 2439 static ssize_t ublk_ch_write_iter(struct kiocb *iocb, struct iov_iter *from) 2568 2440 { 2569 2569 - struct ublk_queue *ubq; 2570 2441 struct request *req; 2442 2442 + struct ublk_io *io; 2571 2443 size_t buf_off; 2572 2444 size_t ret; 2573 2445 2574 2574 - req = ublk_check_and_get_req(iocb, from, &buf_off, ITER_SOURCE); 2446 2446 + req = ublk_check_and_get_req(iocb, from, &buf_off, ITER_SOURCE, &io); 2575 2447 if (IS_ERR(req)) 2576 2448 return PTR_ERR(req); 2577 2449 2578 2450 ret = ublk_copy_user_pages(req, buf_off, from, ITER_SOURCE); 2579 2579 - ubq = req->mq_hctx->driver_data; 2580 2580 - ublk_put_req_ref(ubq, req); 2451 2451 + ublk_put_req_ref(io, req); 2581 2452 2582 2453 return ret; 2583 2454 } ··· 2600 2475 struct ublk_io *io = &ubq->ios[i]; 2601 2476 if (io->task) 2602 2477 put_task_struct(io->task); 2478 2478 + WARN_ON_ONCE(refcount_read(&io->ref)); 2479 2479 + WARN_ON_ONCE(io->task_registered_buffers); 2603 2480 } 2604 2481 2605 2482 if (ubq->io_cmd_buf) ··· 2640 2513 2641 2514 for (i = 0; i < nr_queues; i++) 2642 2515 ublk_deinit_queue(ub, i); 2643 2643 - kfree(ub->__queues); 2516 2516 + kvfree(ub->__queues); 2644 2517 } 2645 2518 2646 2519 static int ublk_init_queues(struct ublk_device *ub) ··· 2651 2524 int i, ret = -ENOMEM; 2652 2525 2653 2526 ub->queue_size = ubq_size; 2654 2654 - ub->__queues = kcalloc(nr_queues, ubq_size, GFP_KERNEL); 2527 2527 + ub->__queues = kvcalloc(nr_queues, ubq_size, GFP_KERNEL); 2655 2528 if (!ub->__queues) 2656 2529 return ret; 2657 2530 ··· 2707 2580 ublk_deinit_queues(ub); 2708 2581 ublk_free_dev_number(ub); 2709 2582 mutex_destroy(&ub->mutex); 2583 2583 + mutex_destroy(&ub->cancel_mutex); 2710 2584 kfree(ub); 2711 2585 } 2712 2586 ··· 2755 2627 ub->tag_set.nr_hw_queues = ub->dev_info.nr_hw_queues; 2756 2628 ub->tag_set.queue_depth = ub->dev_info.queue_depth; 2757 2629 ub->tag_set.numa_node = NUMA_NO_NODE; 2758 2758 - ub->tag_set.cmd_size = sizeof(struct ublk_rq_data); 2759 2630 ub->tag_set.driver_data = ub; 2760 2631 return blk_mq_alloc_tag_set(&ub->tag_set); 2761 2632 } ··· 2855 2728 2856 2729 if (wait_for_completion_interruptible(&ub->completion) != 0) 2857 2730 return -EINTR; 2731 2731 + 2732 2732 + if (ub->ublksrv_tgid != ublksrv_pid) 2733 2733 + return -EINVAL; 2858 2734 2859 2735 mutex_lock(&ub->mutex); 2860 2736 if (ub->dev_info.state == UBLK_S_DEV_LIVE || ··· 3063 2933 goto out_unlock; 3064 2934 mutex_init(&ub->mutex); 3065 2935 spin_lock_init(&ub->lock); 2936 2936 + mutex_init(&ub->cancel_mutex); 3066 2937 3067 2938 ret = ublk_alloc_dev_number(ub, header->dev_id); 3068 2939 if (ret < 0) ··· 3084 2953 3085 2954 ub->dev_info.flags |= UBLK_F_CMD_IOCTL_ENCODE | 3086 2955 UBLK_F_URING_CMD_COMP_IN_TASK | 3087 3087 - UBLK_F_PER_IO_DAEMON; 2956 2956 + UBLK_F_PER_IO_DAEMON | 2957 2957 + UBLK_F_BUF_REG_OFF_DAEMON; 3088 2958 3089 2959 /* GET_DATA isn't needed any more with USER_COPY or ZERO COPY */ 3090 2960 if (ub->dev_info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY | ··· 3135 3003 ublk_free_dev_number(ub); 3136 3004 out_free_ub: 3137 3005 mutex_destroy(&ub->mutex); 3006 3006 + mutex_destroy(&ub->cancel_mutex); 3138 3007 kfree(ub); 3139 3008 out_unlock: 3140 3009 mutex_unlock(&ublk_ctl_mutex); ··· 3360 3227 pr_devel("%s: All FETCH_REQs received, dev id %d\n", __func__, 3361 3228 header->dev_id); 3362 3229 3230 3230 + if (ub->ublksrv_tgid != ublksrv_pid) 3231 3231 + return -EINVAL; 3232 3232 + 3363 3233 mutex_lock(&ub->mutex); 3364 3234 if (ublk_nosrv_should_stop_dev(ub)) 3365 3235 goto out_unlock; ··· 3476 3340 /* zero means wait forever */ 3477 3341 u64 timeout_ms = header->data[0]; 3478 3342 struct gendisk *disk; 3479 3479 - int i, ret = -ENODEV; 3343 3343 + int ret = -ENODEV; 3480 3344 3481 3345 if (!(ub->dev_info.flags & UBLK_F_QUIESCE)) 3482 3346 return -EOPNOTSUPP; ··· 3493 3357 if (ub->dev_info.state != UBLK_S_DEV_LIVE) 3494 3358 goto put_disk; 3495 3359 3496 3496 - /* Mark all queues as canceling */ 3360 3360 + /* Mark the device as canceling */ 3361 3361 + mutex_lock(&ub->cancel_mutex); 3497 3362 blk_mq_quiesce_queue(disk->queue); 3498 3498 - for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { 3499 3499 - struct ublk_queue *ubq = ublk_get_queue(ub, i); 3500 3500 - 3501 3501 - ubq->canceling = true; 3502 3502 - } 3363 3363 + ublk_set_canceling(ub, true); 3503 3364 blk_mq_unquiesce_queue(disk->queue); 3365 3365 + mutex_unlock(&ub->cancel_mutex); 3504 3366 3505 3367 if (!timeout_ms) 3506 3368 timeout_ms = UINT_MAX;

+2 -3

drivers/block/virtio_blk.c

reviewed

··· 976 976 return -EINVAL; 977 977 } 978 978 979 979 - num_vqs = min_t(unsigned int, 980 980 - min_not_zero(num_request_queues, nr_cpu_ids), 981 981 - num_vqs); 979 979 + num_vqs = blk_mq_num_possible_queues( 980 980 + min_not_zero(num_request_queues, num_vqs)); 982 981 983 982 num_poll_vqs = min_t(unsigned int, poll_queues, num_vqs - 1); 984 983

+7 -8

drivers/block/zram/zcomp.c

reviewed

··· 8 8 #include <linux/sched.h> 9 9 #include <linux/cpuhotplug.h> 10 10 #include <linux/vmalloc.h> 11 11 + #include <linux/sysfs.h> 11 12 12 13 #include "zcomp.h" 13 14 ··· 90 89 } 91 90 92 91 /* show available compressors */ 93 93 - ssize_t zcomp_available_show(const char *comp, char *buf) 92 92 + ssize_t zcomp_available_show(const char *comp, char *buf, ssize_t at) 94 93 { 95 95 - ssize_t sz = 0; 96 94 int i; 97 95 98 96 for (i = 0; i < ARRAY_SIZE(backends) - 1; i++) { 99 97 if (!strcmp(comp, backends[i]->name)) { 100 100 - sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, 101 101 - "[%s] ", backends[i]->name); 98 98 + at += sysfs_emit_at(buf, at, "[%s] ", 99 99 + backends[i]->name); 102 100 } else { 103 103 - sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, 104 104 - "%s ", backends[i]->name); 101 101 + at += sysfs_emit_at(buf, at, "%s ", backends[i]->name); 105 102 } 106 103 } 107 104 108 108 - sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); 109 109 - return sz; 105 105 + at += sysfs_emit_at(buf, at, "\n"); 106 106 + return at; 110 107 } 111 108 112 109 struct zcomp_strm *zcomp_stream_get(struct zcomp *comp)

+1 -1

drivers/block/zram/zcomp.h

reviewed

··· 79 79 80 80 int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node); 81 81 int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node); 82 82 - ssize_t zcomp_available_show(const char *comp, char *buf); 82 82 + ssize_t zcomp_available_show(const char *comp, char *buf, ssize_t at); 83 83 bool zcomp_available_algorithm(const char *comp); 84 84 85 85 struct zcomp *zcomp_create(const char *alg, struct zcomp_params *params);

+16 -15

drivers/block/zram/zram_drv.c

reviewed

··· 373 373 val = init_done(zram); 374 374 up_read(&zram->init_lock); 375 375 376 376 - return scnprintf(buf, PAGE_SIZE, "%u\n", val); 376 376 + return sysfs_emit(buf, "%u\n", val); 377 377 } 378 378 379 379 static ssize_t disksize_show(struct device *dev, ··· 381 381 { 382 382 struct zram *zram = dev_to_zram(dev); 383 383 384 384 - return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize); 384 384 + return sysfs_emit(buf, "%llu\n", zram->disksize); 385 385 } 386 386 387 387 static ssize_t mem_limit_store(struct device *dev, ··· 532 532 spin_unlock(&zram->wb_limit_lock); 533 533 up_read(&zram->init_lock); 534 534 535 535 - return scnprintf(buf, PAGE_SIZE, "%d\n", val); 535 535 + return sysfs_emit(buf, "%d\n", val); 536 536 } 537 537 538 538 static ssize_t writeback_limit_store(struct device *dev, ··· 567 567 spin_unlock(&zram->wb_limit_lock); 568 568 up_read(&zram->init_lock); 569 569 570 570 - return scnprintf(buf, PAGE_SIZE, "%llu\n", val); 570 570 + return sysfs_emit(buf, "%llu\n", val); 571 571 } 572 572 573 573 static void reset_bdev(struct zram *zram) ··· 1225 1225 zram->comp_algs[prio] = alg; 1226 1226 } 1227 1227 1228 1228 - static ssize_t __comp_algorithm_show(struct zram *zram, u32 prio, char *buf) 1228 1228 + static ssize_t __comp_algorithm_show(struct zram *zram, u32 prio, 1229 1229 + char *buf, ssize_t at) 1229 1230 { 1230 1231 ssize_t sz; 1231 1232 1232 1233 down_read(&zram->init_lock); 1233 1233 - sz = zcomp_available_show(zram->comp_algs[prio], buf); 1234 1234 + sz = zcomp_available_show(zram->comp_algs[prio], buf, at); 1234 1235 up_read(&zram->init_lock); 1235 1236 1236 1237 return sz; ··· 1388 1387 { 1389 1388 struct zram *zram = dev_to_zram(dev); 1390 1389 1391 1391 - return __comp_algorithm_show(zram, ZRAM_PRIMARY_COMP, buf); 1390 1390 + return __comp_algorithm_show(zram, ZRAM_PRIMARY_COMP, buf, 0); 1392 1391 } 1393 1392 1394 1393 static ssize_t comp_algorithm_store(struct device *dev, ··· 1416 1415 if (!zram->comp_algs[prio]) 1417 1416 continue; 1418 1417 1419 1419 - sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, "#%d: ", prio); 1420 1420 - sz += __comp_algorithm_show(zram, prio, buf + sz); 1418 1418 + sz += sysfs_emit_at(buf, sz, "#%d: ", prio); 1419 1419 + sz += __comp_algorithm_show(zram, prio, buf, sz); 1421 1420 } 1422 1421 1423 1422 return sz; ··· 1489 1488 ssize_t ret; 1490 1489 1491 1490 down_read(&zram->init_lock); 1492 1492 - ret = scnprintf(buf, PAGE_SIZE, 1491 1491 + ret = sysfs_emit(buf, 1493 1492 "%8llu %8llu 0 %8llu\n", 1494 1493 (u64)atomic64_read(&zram->stats.failed_reads), 1495 1494 (u64)atomic64_read(&zram->stats.failed_writes), ··· 1519 1518 orig_size = atomic64_read(&zram->stats.pages_stored); 1520 1519 max_used = atomic_long_read(&zram->stats.max_used_pages); 1521 1520 1522 1522 - ret = scnprintf(buf, PAGE_SIZE, 1521 1521 + ret = sysfs_emit(buf, 1523 1522 "%8llu %8llu %8llu %8lu %8ld %8llu %8lu %8llu %8llu\n", 1524 1523 orig_size << PAGE_SHIFT, 1525 1524 (u64)atomic64_read(&zram->stats.compr_data_size), ··· 1544 1543 ssize_t ret; 1545 1544 1546 1545 down_read(&zram->init_lock); 1547 1547 - ret = scnprintf(buf, PAGE_SIZE, 1548 1548 - "%8llu %8llu %8llu\n", 1546 1546 + ret = sysfs_emit(buf, 1547 1547 + "%8llu %8llu %8llu\n", 1549 1548 FOUR_K((u64)atomic64_read(&zram->stats.bd_count)), 1550 1549 FOUR_K((u64)atomic64_read(&zram->stats.bd_reads)), 1551 1550 FOUR_K((u64)atomic64_read(&zram->stats.bd_writes))); ··· 1563 1562 ssize_t ret; 1564 1563 1565 1564 down_read(&zram->init_lock); 1566 1566 - ret = scnprintf(buf, PAGE_SIZE, 1565 1565 + ret = sysfs_emit(buf, 1567 1566 "version: %d\n0 %8llu\n", 1568 1567 version, 1569 1568 (u64)atomic64_read(&zram->stats.miss_free)); ··· 2811 2810 2812 2811 if (ret < 0) 2813 2812 return ret; 2814 2814 - return scnprintf(buf, PAGE_SIZE, "%d\n", ret); 2813 2813 + return sysfs_emit(buf, "%d\n", ret); 2815 2814 } 2816 2815 /* This attribute must be set to 0400, so CLASS_ATTR_RO() can not be used */ 2817 2816 static struct class_attribute class_attr_hot_add =

+2 -6

drivers/cdrom/cdrom.c

reviewed

··· 624 624 if (check_media_type == 1) 625 625 cdi->options |= (int) CDO_CHECK_TYPE; 626 626 627 627 - if (CDROM_CAN(CDC_MRW_W)) 628 628 - cdi->exit = cdrom_mrw_exit; 629 629 - 630 627 if (cdi->ops->read_cdda_bpc) 631 628 cdi->cdda_method = CDDA_BPC_FULL; 632 629 else ··· 647 650 mutex_lock(&cdrom_mutex); 648 651 list_del(&cdi->list); 649 652 mutex_unlock(&cdrom_mutex); 650 650 - 651 651 - if (cdi->exit) 652 652 - cdi->exit(cdi); 653 653 654 654 cd_dbg(CD_REG_UNREG, "drive \"/dev/%s\" unregistered\n", cdi->name); 655 655 } ··· 1258 1264 cd_dbg(CD_CLOSE, "Use count for \"/dev/%s\" now zero\n", 1259 1265 cdi->name); 1260 1266 cdrom_dvd_rw_close_write(cdi); 1267 1267 + if (CDROM_CAN(CDC_MRW_W)) 1268 1268 + cdrom_mrw_exit(cdi); 1261 1269 1262 1270 if ((cdo->capability & CDC_LOCK) && !cdi->keeplocked) { 1263 1271 cd_dbg(CD_CLOSE, "Unlocking door!\n");

+11 -11

drivers/md/bcache/super.c

reviewed

··· 168 168 { 169 169 const char *err; 170 170 struct cache_sb_disk *s; 171 171 - struct page *page; 171 171 + struct folio *folio; 172 172 unsigned int i; 173 173 174 174 - page = read_cache_page_gfp(bdev->bd_mapping, 175 175 - SB_OFFSET >> PAGE_SHIFT, GFP_KERNEL); 176 176 - if (IS_ERR(page)) 174 174 + folio = mapping_read_folio_gfp(bdev->bd_mapping, 175 175 + SB_OFFSET >> PAGE_SHIFT, GFP_KERNEL); 176 176 + if (IS_ERR(folio)) 177 177 return "IO error"; 178 178 - s = page_address(page) + offset_in_page(SB_OFFSET); 178 178 + s = folio_address(folio) + offset_in_folio(folio, SB_OFFSET); 179 179 180 180 sb->offset = le64_to_cpu(s->offset); 181 181 sb->version = le64_to_cpu(s->version); ··· 272 272 *res = s; 273 273 return NULL; 274 274 err: 275 275 - put_page(page); 275 275 + folio_put(folio); 276 276 return err; 277 277 } 278 278 ··· 1366 1366 mutex_unlock(&bch_register_lock); 1367 1367 1368 1368 if (dc->sb_disk) 1369 1369 - put_page(virt_to_page(dc->sb_disk)); 1369 1369 + folio_put(virt_to_folio(dc->sb_disk)); 1370 1370 1371 1371 if (dc->bdev_file) 1372 1372 fput(dc->bdev_file); ··· 2216 2216 free_fifo(&ca->free[i]); 2217 2217 2218 2218 if (ca->sb_disk) 2219 2219 - put_page(virt_to_page(ca->sb_disk)); 2219 2219 + folio_put(virt_to_folio(ca->sb_disk)); 2220 2220 2221 2221 if (ca->bdev_file) 2222 2222 fput(ca->bdev_file); ··· 2593 2593 if (!holder) { 2594 2594 ret = -ENOMEM; 2595 2595 err = "cannot allocate memory"; 2596 2596 - goto out_put_sb_page; 2596 2596 + goto out_put_sb_folio; 2597 2597 } 2598 2598 2599 2599 /* Now reopen in exclusive mode with proper holder */ ··· 2667 2667 2668 2668 out_free_holder: 2669 2669 kfree(holder); 2670 2670 - out_put_sb_page: 2671 2671 - put_page(virt_to_page(sb_disk)); 2670 2670 + out_put_sb_folio: 2671 2671 + folio_put(virt_to_folio(sb_disk)); 2672 2672 out_blkdev_put: 2673 2673 if (bdev_file) 2674 2674 fput(bdev_file);

+38 -9

drivers/md/dm-crypt.c

reviewed

··· 253 253 static unsigned int max_write_size = 0; 254 254 module_param(max_write_size, uint, 0644); 255 255 MODULE_PARM_DESC(max_write_size, "Maximum size of a write request"); 256 256 - static unsigned get_max_request_size(struct crypt_config *cc, bool wrt) 256 256 + 257 257 + static unsigned get_max_request_sectors(struct dm_target *ti, struct bio *bio) 257 258 { 259 259 + struct crypt_config *cc = ti->private; 258 260 unsigned val, sector_align; 259 259 - val = !wrt ? READ_ONCE(max_read_size) : READ_ONCE(max_write_size); 260 260 - if (likely(!val)) 261 261 - val = !wrt ? DM_CRYPT_DEFAULT_MAX_READ_SIZE : DM_CRYPT_DEFAULT_MAX_WRITE_SIZE; 262 262 - if (wrt || cc->used_tag_size) { 263 263 - if (unlikely(val > BIO_MAX_VECS << PAGE_SHIFT)) 264 264 - val = BIO_MAX_VECS << PAGE_SHIFT; 261 261 + bool wrt = op_is_write(bio_op(bio)); 262 262 + 263 263 + if (wrt) { 264 264 + /* 265 265 + * For zoned devices, splitting write operations creates the 266 266 + * risk of deadlocking queue freeze operations with zone write 267 267 + * plugging BIO work when the reminder of a split BIO is 268 268 + * issued. So always allow the entire BIO to proceed. 269 269 + */ 270 270 + if (ti->emulate_zone_append) 271 271 + return bio_sectors(bio); 272 272 + 273 273 + val = min_not_zero(READ_ONCE(max_write_size), 274 274 + DM_CRYPT_DEFAULT_MAX_WRITE_SIZE); 275 275 + } else { 276 276 + val = min_not_zero(READ_ONCE(max_read_size), 277 277 + DM_CRYPT_DEFAULT_MAX_READ_SIZE); 265 278 } 266 266 - sector_align = max(bdev_logical_block_size(cc->dev->bdev), (unsigned)cc->sector_size); 279 279 + 280 280 + if (wrt || cc->used_tag_size) 281 281 + val = min(val, BIO_MAX_VECS << PAGE_SHIFT); 282 282 + 283 283 + sector_align = max(bdev_logical_block_size(cc->dev->bdev), 284 284 + (unsigned)cc->sector_size); 267 285 val = round_down(val, sector_align); 268 286 if (unlikely(!val)) 269 287 val = sector_align; ··· 3514 3496 /* 3515 3497 * Check if bio is too large, split as needed. 3516 3498 */ 3517 3517 - max_sectors = get_max_request_size(cc, bio_data_dir(bio) == WRITE); 3499 3499 + max_sectors = get_max_request_sectors(ti, bio); 3518 3500 if (unlikely(bio_sectors(bio) > max_sectors)) 3519 3501 dm_accept_partial_bio(bio, max_sectors); 3520 3502 ··· 3751 3733 max_t(unsigned int, limits->physical_block_size, cc->sector_size); 3752 3734 limits->io_min = max_t(unsigned int, limits->io_min, cc->sector_size); 3753 3735 limits->dma_alignment = limits->logical_block_size - 1; 3736 3736 + 3737 3737 + /* 3738 3738 + * For zoned dm-crypt targets, there will be no internal splitting of 3739 3739 + * write BIOs to avoid exceeding BIO_MAX_VECS vectors per BIO. But 3740 3740 + * without respecting this limit, crypt_alloc_buffer() will trigger a 3741 3741 + * BUG(). Avoid this by forcing DM core to split write BIOs to this 3742 3742 + * limit. 3743 3743 + */ 3744 3744 + if (ti->emulate_zone_append) 3745 3745 + limits->max_hw_sectors = min(limits->max_hw_sectors, 3746 3746 + BIO_MAX_VECS << PAGE_SECTORS_SHIFT); 3754 3747 } 3755 3748 3756 3749 static struct target_type crypt_target = {

+1

drivers/md/dm-stripe.c

reviewed

··· 458 458 struct stripe_c *sc = ti->private; 459 459 unsigned int chunk_size = sc->chunk_size << SECTOR_SHIFT; 460 460 461 461 + limits->chunk_sectors = sc->chunk_size; 461 462 limits->io_min = chunk_size; 462 463 limits->io_opt = chunk_size * sc->stripes; 463 464 }

+38 -16

drivers/md/dm.c

reviewed

··· 1293 1293 /* 1294 1294 * A target may call dm_accept_partial_bio only from the map routine. It is 1295 1295 * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management 1296 1296 - * operations, REQ_OP_ZONE_APPEND (zone append writes) and any bio serviced by 1297 1297 - * __send_duplicate_bios(). 1296 1296 + * operations, zone append writes (native with REQ_OP_ZONE_APPEND or emulated 1297 1297 + * with write BIOs flagged with BIO_EMULATES_ZONE_APPEND) and any bio serviced 1298 1298 + * by __send_duplicate_bios(). 1298 1299 * 1299 1300 * dm_accept_partial_bio informs the dm that the target only wants to process 1300 1301 * additional n_sectors sectors of the bio and the rest of the data should be ··· 1328 1327 unsigned int bio_sectors = bio_sectors(bio); 1329 1328 1330 1329 BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO)); 1331 1331 - BUG_ON(op_is_zone_mgmt(bio_op(bio))); 1332 1332 - BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND); 1333 1330 BUG_ON(bio_sectors > *tio->len_ptr); 1334 1331 BUG_ON(n_sectors > bio_sectors); 1332 1332 + 1333 1333 + if (static_branch_unlikely(&zoned_enabled) && 1334 1334 + unlikely(bdev_is_zoned(bio->bi_bdev))) { 1335 1335 + enum req_op op = bio_op(bio); 1336 1336 + 1337 1337 + BUG_ON(op_is_zone_mgmt(op)); 1338 1338 + BUG_ON(op == REQ_OP_WRITE); 1339 1339 + BUG_ON(op == REQ_OP_WRITE_ZEROES); 1340 1340 + BUG_ON(op == REQ_OP_ZONE_APPEND); 1341 1341 + } 1335 1342 1336 1343 *tio->len_ptr -= bio_sectors - n_sectors; 1337 1344 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; ··· 1785 1776 } 1786 1777 1787 1778 #ifdef CONFIG_BLK_DEV_ZONED 1788 1788 - static inline bool dm_zone_bio_needs_split(struct mapped_device *md, 1789 1789 - struct bio *bio) 1779 1779 + static inline bool dm_zone_bio_needs_split(struct bio *bio) 1790 1780 { 1791 1781 /* 1792 1792 - * For mapped device that need zone append emulation, we must 1793 1793 - * split any large BIO that straddles zone boundaries. 1782 1782 + * Special case the zone operations that cannot or should not be split. 1794 1783 */ 1795 1795 - return dm_emulate_zone_append(md) && bio_straddles_zones(bio) && 1796 1796 - !bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING); 1784 1784 + switch (bio_op(bio)) { 1785 1785 + case REQ_OP_ZONE_APPEND: 1786 1786 + case REQ_OP_ZONE_FINISH: 1787 1787 + case REQ_OP_ZONE_RESET: 1788 1788 + case REQ_OP_ZONE_RESET_ALL: 1789 1789 + return false; 1790 1790 + default: 1791 1791 + break; 1792 1792 + } 1793 1793 + 1794 1794 + /* 1795 1795 + * When mapped devices use the block layer zone write plugging, we must 1796 1796 + * split any large BIO to the mapped device limits to not submit BIOs 1797 1797 + * that span zone boundaries and to avoid potential deadlocks with 1798 1798 + * queue freeze operations. 1799 1799 + */ 1800 1800 + return bio_needs_zone_write_plugging(bio) || bio_straddles_zones(bio); 1797 1801 } 1802 1802 + 1798 1803 static inline bool dm_zone_plug_bio(struct mapped_device *md, struct bio *bio) 1799 1804 { 1800 1800 - return dm_emulate_zone_append(md) && blk_zone_plug_bio(bio, 0); 1805 1805 + if (!bio_needs_zone_write_plugging(bio)) 1806 1806 + return false; 1807 1807 + return blk_zone_plug_bio(bio, 0); 1801 1808 } 1802 1809 1803 1810 static blk_status_t __send_zone_reset_all_emulated(struct clone_info *ci, ··· 1929 1904 } 1930 1905 1931 1906 #else 1932 1932 - static inline bool dm_zone_bio_needs_split(struct mapped_device *md, 1933 1933 - struct bio *bio) 1907 1907 + static inline bool dm_zone_bio_needs_split(struct bio *bio) 1934 1908 { 1935 1909 return false; 1936 1910 } ··· 1956 1932 1957 1933 is_abnormal = is_abnormal_io(bio); 1958 1934 if (static_branch_unlikely(&zoned_enabled)) { 1959 1959 - /* Special case REQ_OP_ZONE_RESET_ALL as it cannot be split. */ 1960 1960 - need_split = (bio_op(bio) != REQ_OP_ZONE_RESET_ALL) && 1961 1961 - (is_abnormal || dm_zone_bio_needs_split(md, bio)); 1935 1935 + need_split = is_abnormal || dm_zone_bio_needs_split(bio); 1962 1936 } else { 1963 1937 need_split = is_abnormal; 1964 1938 }

+43 -30

drivers/md/md.c

reviewed

··· 636 636 mddev->ctime || mddev->hold_active) 637 637 return; 638 638 639 639 - /* Array is not configured at all, and not held active, so destroy it */ 640 640 - set_bit(MD_DELETED, &mddev->flags); 641 641 - 642 639 /* 643 640 * Call queue_work inside the spinlock so that flush_workqueue() after 644 641 * mddev_find will succeed in waiting for the work to be done. ··· 870 873 kobject_del(&rdev->kobj); 871 874 export_rdev(rdev, mddev); 872 875 } 876 876 + 877 877 + /* Call del_gendisk after release reconfig_mutex to avoid 878 878 + * deadlock (e.g. call del_gendisk under the lock and an 879 879 + * access to sysfs files waits the lock) 880 880 + * And MD_DELETED is only used for md raid which is set in 881 881 + * do_md_stop. dm raid only uses md_stop to stop. So dm raid 882 882 + * doesn't need to check MD_DELETED when getting reconfig lock 883 883 + */ 884 884 + if (test_bit(MD_DELETED, &mddev->flags)) 885 885 + del_gendisk(mddev->gendisk); 873 886 } 874 887 EXPORT_SYMBOL_GPL(mddev_unlock); 875 888 ··· 5781 5774 struct md_sysfs_entry *entry = container_of(attr, struct md_sysfs_entry, attr); 5782 5775 struct mddev *mddev = container_of(kobj, struct mddev, kobj); 5783 5776 ssize_t rv; 5777 5777 + struct kernfs_node *kn = NULL; 5784 5778 5785 5779 if (!entry->store) 5786 5780 return -EIO; 5787 5781 if (!capable(CAP_SYS_ADMIN)) 5788 5782 return -EACCES; 5783 5783 + 5784 5784 + if (entry->store == array_state_store && cmd_match(page, "clear")) 5785 5785 + kn = sysfs_break_active_protection(kobj, attr); 5786 5786 + 5789 5787 spin_lock(&all_mddevs_lock); 5790 5788 if (!mddev_get(mddev)) { 5791 5789 spin_unlock(&all_mddevs_lock); 5790 5790 + if (kn) 5791 5791 + sysfs_unbreak_active_protection(kn); 5792 5792 return -EBUSY; 5793 5793 } 5794 5794 spin_unlock(&all_mddevs_lock); 5795 5795 rv = entry->store(mddev, page, length); 5796 5796 mddev_put(mddev); 5797 5797 + 5798 5798 + if (kn) 5799 5799 + sysfs_unbreak_active_protection(kn); 5800 5800 + 5797 5801 return rv; 5798 5802 } 5799 5803 ··· 5812 5794 { 5813 5795 struct mddev *mddev = container_of(ko, struct mddev, kobj); 5814 5796 5815 5815 - if (mddev->sysfs_state) 5816 5816 - sysfs_put(mddev->sysfs_state); 5817 5817 - if (mddev->sysfs_level) 5818 5818 - sysfs_put(mddev->sysfs_level); 5819 5819 - 5820 5820 - del_gendisk(mddev->gendisk); 5821 5797 put_disk(mddev->gendisk); 5822 5798 } 5823 5799 ··· 6425 6413 mddev->persistent = 0; 6426 6414 mddev->level = LEVEL_NONE; 6427 6415 mddev->clevel[0] = 0; 6428 6428 - /* 6429 6429 - * Don't clear MD_CLOSING, or mddev can be opened again. 6430 6430 - * 'hold_active != 0' means mddev is still in the creation 6431 6431 - * process and will be used later. 6432 6432 - */ 6433 6433 - if (mddev->hold_active) 6434 6434 - mddev->flags = 0; 6435 6435 - else 6436 6436 - mddev->flags &= BIT_ULL_MASK(MD_CLOSING); 6416 6416 + /* if UNTIL_STOP is set, it's cleared here */ 6417 6417 + mddev->hold_active = 0; 6418 6418 + /* Don't clear MD_CLOSING, or mddev can be opened again. */ 6419 6419 + mddev->flags &= BIT_ULL_MASK(MD_CLOSING); 6437 6420 mddev->sb_flags = 0; 6438 6421 mddev->ro = MD_RDWR; 6439 6422 mddev->metadata_type[0] = 0; ··· 6523 6516 if (mddev->private) 6524 6517 pers->free(mddev, mddev->private); 6525 6518 mddev->private = NULL; 6526 6526 - if (pers->sync_request && mddev->to_remove == NULL) 6527 6527 - mddev->to_remove = &md_redundancy_group; 6528 6519 put_pers(pers); 6529 6520 clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery); 6530 6521 ··· 6651 6646 mddev->bitmap_info.offset = 0; 6652 6647 6653 6648 export_array(mddev); 6654 6654 - 6655 6649 md_clean(mddev); 6656 6656 - if (mddev->hold_active == UNTIL_STOP) 6657 6657 - mddev->hold_active = 0; 6650 6650 + set_bit(MD_DELETED, &mddev->flags); 6658 6651 } 6659 6652 md_new_event(); 6660 6653 sysfs_notify_dirent_safe(mddev->sysfs_state); ··· 9459 9456 return false; 9460 9457 } 9461 9458 9462 9462 - static int remove_and_add_spares(struct mddev *mddev, 9463 9463 - struct md_rdev *this) 9459 9459 + static int remove_spares(struct mddev *mddev, struct md_rdev *this) 9464 9460 { 9465 9461 struct md_rdev *rdev; 9466 9466 - int spares = 0; 9467 9462 int removed = 0; 9468 9468 - 9469 9469 - if (this && test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 9470 9470 - /* Mustn't remove devices when resync thread is running */ 9471 9471 - return 0; 9472 9463 9473 9464 rdev_for_each(rdev, mddev) { 9474 9465 if ((this == NULL || rdev == this) && rdev_removeable(rdev) && ··· 9477 9480 if (removed && mddev->kobj.sd) 9478 9481 sysfs_notify_dirent_safe(mddev->sysfs_degraded); 9479 9482 9483 9483 + return removed; 9484 9484 + } 9485 9485 + 9486 9486 + static int remove_and_add_spares(struct mddev *mddev, 9487 9487 + struct md_rdev *this) 9488 9488 + { 9489 9489 + struct md_rdev *rdev; 9490 9490 + int spares = 0; 9491 9491 + int removed = 0; 9492 9492 + 9493 9493 + if (this && test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 9494 9494 + /* Mustn't remove devices when resync thread is running */ 9495 9495 + return 0; 9496 9496 + 9497 9497 + removed = remove_spares(mddev, this); 9480 9498 if (this && removed) 9481 9499 goto no_add; 9482 9500 ··· 9534 9522 9535 9523 /* Check if resync is in progress. */ 9536 9524 if (mddev->recovery_cp < MaxSector) { 9525 9525 + remove_spares(mddev, NULL); 9537 9526 set_bit(MD_RECOVERY_SYNC, &mddev->recovery); 9538 9527 clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery); 9539 9528 return true;

+24 -2

drivers/md/md.h

reviewed

··· 700 700 701 701 static inline int __must_check mddev_lock(struct mddev *mddev) 702 702 { 703 703 - return mutex_lock_interruptible(&mddev->reconfig_mutex); 703 703 + int ret; 704 704 + 705 705 + ret = mutex_lock_interruptible(&mddev->reconfig_mutex); 706 706 + 707 707 + /* MD_DELETED is set in do_md_stop with reconfig_mutex. 708 708 + * So check it here. 709 709 + */ 710 710 + if (!ret && test_bit(MD_DELETED, &mddev->flags)) { 711 711 + ret = -ENODEV; 712 712 + mutex_unlock(&mddev->reconfig_mutex); 713 713 + } 714 714 + 715 715 + return ret; 704 716 } 705 717 706 718 /* Sometimes we need to take the lock in a situation where 707 719 * failure due to interrupts is not acceptable. 720 720 + * It doesn't need to check MD_DELETED here, the owner which 721 721 + * holds the lock here can't be stopped. And all paths can't 722 722 + * call this function after do_md_stop. 708 723 */ 709 724 static inline void mddev_lock_nointr(struct mddev *mddev) 710 725 { ··· 728 713 729 714 static inline int mddev_trylock(struct mddev *mddev) 730 715 { 731 731 - return mutex_trylock(&mddev->reconfig_mutex); 716 716 + int ret; 717 717 + 718 718 + ret = mutex_trylock(&mddev->reconfig_mutex); 719 719 + if (!ret && test_bit(MD_DELETED, &mddev->flags)) { 720 720 + ret = -ENODEV; 721 721 + mutex_unlock(&mddev->reconfig_mutex); 722 722 + } 723 723 + return ret; 732 724 } 733 725 extern void mddev_unlock(struct mddev *mddev); 734 726

+1

drivers/md/raid0.c

reviewed

··· 384 384 lim.max_write_zeroes_sectors = mddev->chunk_sectors; 385 385 lim.io_min = mddev->chunk_sectors << 9; 386 386 lim.io_opt = lim.io_min * mddev->raid_disks; 387 387 + lim.chunk_sectors = mddev->chunk_sectors; 387 388 lim.features |= BLK_FEAT_ATOMIC_WRITES; 388 389 err = mddev_stack_rdev_limits(mddev, &lim, MDDEV_STACK_INTEGRITY); 389 390 if (err)

+1 -3

drivers/md/raid10.c

reviewed

··· 2446 2446 * that are active 2447 2447 */ 2448 2448 for (i = 0; i < conf->copies; i++) { 2449 2449 - int d; 2450 2450 - 2451 2449 tbio = r10_bio->devs[i].repl_bio; 2452 2450 if (!tbio || !tbio->bi_end_io) 2453 2451 continue; 2454 2452 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write 2455 2453 && r10_bio->devs[i].bio != fbio) 2456 2454 bio_copy_data(tbio, fbio); 2457 2457 - d = r10_bio->devs[i].devnum; 2458 2455 atomic_inc(&r10_bio->remaining); 2459 2456 submit_bio_noacct(tbio); 2460 2457 } ··· 4009 4012 md_init_stacking_limits(&lim); 4010 4013 lim.max_write_zeroes_sectors = 0; 4011 4014 lim.io_min = mddev->chunk_sectors << 9; 4015 4015 + lim.chunk_sectors = mddev->chunk_sectors; 4012 4016 lim.io_opt = lim.io_min * raid10_nr_stripes(conf); 4013 4017 lim.features |= BLK_FEAT_ATOMIC_WRITES; 4014 4018 err = mddev_stack_rdev_limits(mddev, &lim, MDDEV_STACK_INTEGRITY);

+1 -1

drivers/md/raid5.c

reviewed

··· 9040 9040 int ret; 9041 9041 9042 9042 raid5_wq = alloc_workqueue("raid5wq", 9043 9043 - WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0); 9043 9043 + WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_SYSFS, 0); 9044 9044 if (!raid5_wq) 9045 9045 return -ENOMEM; 9046 9046

+2 -2

drivers/nvme/host/apple.c

reviewed

··· 301 301 memcpy(&q->sqes[tag], cmd, sizeof(*cmd)); 302 302 303 303 /* 304 304 - * This lock here doesn't make much sense at a first glace but 305 305 - * removing it will result in occasional missed completetion 304 304 + * This lock here doesn't make much sense at a first glance but 305 305 + * removing it will result in occasional missed completion 306 306 * interrupts even though the commands still appear on the CQ. 307 307 * It's unclear why this happens but our best guess is that 308 308 * there is a bug in the firmware triggered when a new command

+2 -2

drivers/nvme/host/constants.c

reviewed

··· 133 133 [NVME_SC_NS_NOT_ATTACHED] = "Namespace Not Attached", 134 134 [NVME_SC_THIN_PROV_NOT_SUPP] = "Thin Provisioning Not Supported", 135 135 [NVME_SC_CTRL_LIST_INVALID] = "Controller List Invalid", 136 136 - [NVME_SC_SELT_TEST_IN_PROGRESS] = "Device Self-test In Progress", 136 136 + [NVME_SC_SELF_TEST_IN_PROGRESS] = "Device Self-test In Progress", 137 137 [NVME_SC_BP_WRITE_PROHIBITED] = "Boot Partition Write Prohibited", 138 138 [NVME_SC_CTRL_ID_INVALID] = "Invalid Controller Identifier", 139 139 [NVME_SC_SEC_CTRL_STATE_INVALID] = "Invalid Secondary Controller State", ··· 145 145 [NVME_SC_BAD_ATTRIBUTES] = "Conflicting Attributes", 146 146 [NVME_SC_INVALID_PI] = "Invalid Protection Information", 147 147 [NVME_SC_READ_ONLY] = "Attempted Write to Read Only Range", 148 148 - [NVME_SC_CMD_SIZE_LIM_EXCEEDED ] = "Command Size Limits Exceeded", 148 148 + [NVME_SC_CMD_SIZE_LIM_EXCEEDED] = "Command Size Limits Exceeded", 149 149 [NVME_SC_ZONE_BOUNDARY_ERROR] = "Zoned Boundary Error", 150 150 [NVME_SC_ZONE_FULL] = "Zone Is Full", 151 151 [NVME_SC_ZONE_READ_ONLY] = "Zone Is Read Only",

+1 -1

drivers/nvme/host/core.c

reviewed

··· 4300 4300 } 4301 4301 4302 4302 /* 4303 4303 - * If available try to use the Command Set Idependent Identify Namespace 4303 4303 + * If available try to use the Command Set Independent Identify Namespace 4304 4304 * data structure to find all the generic information that is needed to 4305 4305 * set up a namespace. If not fall back to the legacy version. 4306 4306 */

+5 -5

drivers/nvme/host/fc.c

reviewed

··· 899 899 * may crash. 900 900 * 901 901 * As such: 902 902 - * Wrapper all the dma routines and check the dev pointer. 902 902 + * Wrap all the dma routines and check the dev pointer. 903 903 * 904 904 * If simple mappings (return just a dma address, we'll noop them, 905 905 * returning a dma address of 0. ··· 1955 1955 } 1956 1956 1957 1957 /* 1958 1958 - * For the linux implementation, if we have an unsucceesful 1959 1959 - * status, they blk-mq layer can typically be called with the 1958 1958 + * For the linux implementation, if we have an unsuccessful 1959 1959 + * status, the blk-mq layer can typically be called with the 1960 1960 * non-zero status and the content of the cqe isn't important. 1961 1961 */ 1962 1962 if (status) ··· 2429 2429 2430 2430 /* 2431 2431 * This routine runs through all outstanding commands on the association 2432 2432 - * and aborts them. This routine is typically be called by the 2432 2432 + * and aborts them. This routine is typically called by the 2433 2433 * delete_association routine. It is also called due to an error during 2434 2434 * reconnect. In that scenario, it is most likely a command that initializes 2435 2435 * the controller, including fabric Connect commands on io queues, that ··· 2622 2622 * as part of the exchange. The CQE is the last thing for the io, 2623 2623 * which is transferred (explicitly or implicitly) with the RSP IU 2624 2624 * sent on the exchange. After the CQE is received, the FC exchange is 2625 2625 - * terminaed and the Exchange may be used on a different io. 2625 2625 + * terminated and the Exchange may be used on a different io. 2626 2626 * 2627 2627 * The transport to LLDD api has the transport making a request for a 2628 2628 * new fcp io request to the LLDD. The LLDD then allocates a FC exchange

+1 -1

drivers/nvme/host/nvme.h

reviewed

··· 69 69 NVME_QUIRK_IDENTIFY_CNS = (1 << 1), 70 70 71 71 /* 72 72 - * The controller deterministically returns O's on reads to 72 72 + * The controller deterministically returns 0's on reads to 73 73 * logical blocks that deallocate was called on. 74 74 */ 75 75 NVME_QUIRK_DEALLOCATE_ZEROES = (1 << 2),

+407 -281

drivers/nvme/host/pci.c

reviewed

··· 7 7 #include <linux/acpi.h> 8 8 #include <linux/async.h> 9 9 #include <linux/blkdev.h> 10 10 - #include <linux/blk-mq.h> 10 10 + #include <linux/blk-mq-dma.h> 11 11 #include <linux/blk-integrity.h> 12 12 #include <linux/dmi.h> 13 13 #include <linux/init.h> ··· 27 27 #include <linux/io-64-nonatomic-lo-hi.h> 28 28 #include <linux/io-64-nonatomic-hi-lo.h> 29 29 #include <linux/sed-opal.h> 30 30 - #include <linux/pci-p2pdma.h> 31 30 32 31 #include "trace.h" 33 32 #include "nvme.h" ··· 38 39 #define NVME_SMALL_POOL_SIZE 256 39 40 40 41 /* 41 41 - * These can be higher, but we need to ensure that any command doesn't 42 42 - * require an sg allocation that needs more than a page of data. 42 42 + * Arbitrary upper bound. 43 43 */ 44 44 - #define NVME_MAX_KB_SZ 8192 44 44 + #define NVME_MAX_BYTES SZ_8M 45 45 #define NVME_MAX_NR_DESCRIPTORS 5 46 46 47 47 /* 48 48 - * For data SGLs we support a single descriptors worth of SGL entries, but for 49 49 - * now we also limit it to avoid an allocation larger than PAGE_SIZE for the 50 50 - * scatterlist. 48 48 + * For data SGLs we support a single descriptors worth of SGL entries. 49 49 + * For PRPs, segments don't matter at all. 51 50 */ 52 51 #define NVME_MAX_SEGS \ 53 53 - min(NVME_CTRL_PAGE_SIZE / sizeof(struct nvme_sgl_desc), \ 54 54 - (PAGE_SIZE / sizeof(struct scatterlist))) 52 52 + (NVME_CTRL_PAGE_SIZE / sizeof(struct nvme_sgl_desc)) 55 53 56 54 /* 57 55 * For metadata SGLs, only the small descriptor is supported, and the first ··· 56 60 */ 57 61 #define NVME_MAX_META_SEGS \ 58 62 ((NVME_SMALL_POOL_SIZE / sizeof(struct nvme_sgl_desc)) - 1) 63 63 + 64 64 + /* 65 65 + * The last entry is used to link to the next descriptor. 66 66 + */ 67 67 + #define PRPS_PER_PAGE \ 68 68 + (((NVME_CTRL_PAGE_SIZE / sizeof(__le64))) - 1) 69 69 + 70 70 + /* 71 71 + * I/O could be non-aligned both at the beginning and end. 72 72 + */ 73 73 + #define MAX_PRP_RANGE \ 74 74 + (NVME_MAX_BYTES + 2 * (NVME_CTRL_PAGE_SIZE - 1)) 75 75 + 76 76 + static_assert(MAX_PRP_RANGE / NVME_CTRL_PAGE_SIZE <= 77 77 + (1 /* prp1 */ + NVME_MAX_NR_DESCRIPTORS * PRPS_PER_PAGE)); 59 78 60 79 static int use_threaded_interrupts; 61 80 module_param(use_threaded_interrupts, int, 0444); ··· 108 97 int ret; 109 98 110 99 ret = kstrtouint(val, 10, &n); 111 111 - if (ret != 0 || n > num_possible_cpus()) 100 100 + if (ret != 0 || n > blk_mq_num_possible_queues(0)) 112 101 return -EINVAL; 113 102 return param_set_uint(val, kp); 114 103 } ··· 173 162 bool hmb; 174 163 struct sg_table *hmb_sgt; 175 164 176 176 - mempool_t *iod_mempool; 165 165 + mempool_t *dmavec_mempool; 177 166 mempool_t *iod_meta_mempool; 178 167 179 168 /* shadow doorbell buffer support: */ ··· 257 246 IOD_ABORTED = 1U << 0, 258 247 259 248 /* uses the small descriptor pool */ 260 260 - IOD_SMALL_DESCRIPTOR = 1U << 1, 249 249 + IOD_SMALL_DESCRIPTOR = 1U << 1, 250 250 + 251 251 + /* single segment dma mapping */ 252 252 + IOD_SINGLE_SEGMENT = 1U << 2, 253 253 + }; 254 254 + 255 255 + struct nvme_dma_vec { 256 256 + dma_addr_t addr; 257 257 + unsigned int len; 261 258 }; 262 259 263 260 /* ··· 276 257 struct nvme_command cmd; 277 258 u8 flags; 278 259 u8 nr_descriptors; 279 279 - unsigned int dma_len; /* length of single DMA segment mapping */ 280 280 - dma_addr_t first_dma; 260 260 + 261 261 + unsigned int total_len; 262 262 + struct dma_iova_state dma_state; 263 263 + void *descriptors[NVME_MAX_NR_DESCRIPTORS]; 264 264 + struct nvme_dma_vec *dma_vecs; 265 265 + unsigned int nr_dma_vecs; 266 266 + 281 267 dma_addr_t meta_dma; 282 282 - struct sg_table sgt; 283 268 struct sg_table meta_sgt; 284 269 struct nvme_sgl_desc *meta_descriptor; 285 285 - void *descriptors[NVME_MAX_NR_DESCRIPTORS]; 286 270 }; 287 271 288 272 static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev) ··· 426 404 } 427 405 428 406 return true; 429 429 - } 430 430 - 431 431 - /* 432 432 - * Will slightly overestimate the number of pages needed. This is OK 433 433 - * as it only leads to a small amount of wasted memory for the lifetime of 434 434 - * the I/O. 435 435 - */ 436 436 - static __always_inline int nvme_pci_npages_prp(void) 437 437 - { 438 438 - unsigned max_bytes = (NVME_MAX_KB_SZ * 1024) + NVME_CTRL_PAGE_SIZE; 439 439 - unsigned nprps = DIV_ROUND_UP(max_bytes, NVME_CTRL_PAGE_SIZE); 440 440 - return DIV_ROUND_UP(8 * nprps, NVME_CTRL_PAGE_SIZE - 8); 441 407 } 442 408 443 409 static struct nvme_descriptor_pools * ··· 588 578 spin_unlock(&nvmeq->sq_lock); 589 579 } 590 580 591 591 - static inline bool nvme_pci_metadata_use_sgls(struct nvme_dev *dev, 592 592 - struct request *req) 581 581 + enum nvme_use_sgl { 582 582 + SGL_UNSUPPORTED, 583 583 + SGL_SUPPORTED, 584 584 + SGL_FORCED, 585 585 + }; 586 586 + 587 587 + static inline bool nvme_pci_metadata_use_sgls(struct request *req) 593 588 { 589 589 + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 590 590 + struct nvme_dev *dev = nvmeq->dev; 591 591 + 594 592 if (!nvme_ctrl_meta_sgl_supported(&dev->ctrl)) 595 593 return false; 596 594 return req->nr_integrity_segments > 1 || 597 595 nvme_req(req)->flags & NVME_REQ_USERCMD; 598 596 } 599 597 600 600 - static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req, 601 601 - int nseg) 598 598 + static inline enum nvme_use_sgl nvme_pci_use_sgls(struct nvme_dev *dev, 599 599 + struct request *req) 602 600 { 603 601 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 604 604 - unsigned int avg_seg_size; 605 602 606 606 - avg_seg_size = DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg); 603 603 + if (nvmeq->qid && nvme_ctrl_sgl_supported(&dev->ctrl)) { 604 604 + if (nvme_req(req)->flags & NVME_REQ_USERCMD) 605 605 + return SGL_FORCED; 606 606 + if (req->nr_integrity_segments > 1) 607 607 + return SGL_FORCED; 608 608 + return SGL_SUPPORTED; 609 609 + } 607 610 608 608 - if (!nvme_ctrl_sgl_supported(&dev->ctrl)) 609 609 - return false; 610 610 - if (!nvmeq->qid) 611 611 - return false; 612 612 - if (nvme_pci_metadata_use_sgls(dev, req)) 613 613 - return true; 614 614 - if (!sgl_threshold || avg_seg_size < sgl_threshold) 615 615 - return nvme_req(req)->flags & NVME_REQ_USERCMD; 616 616 - return true; 611 611 + return SGL_UNSUPPORTED; 612 612 + } 613 613 + 614 614 + static unsigned int nvme_pci_avg_seg_size(struct request *req) 615 615 + { 616 616 + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 617 617 + unsigned int nseg; 618 618 + 619 619 + if (blk_rq_dma_map_coalesce(&iod->dma_state)) 620 620 + nseg = 1; 621 621 + else 622 622 + nseg = blk_rq_nr_phys_segments(req); 623 623 + return DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg); 617 624 } 618 625 619 626 static inline struct dma_pool *nvme_dma_pool(struct nvme_queue *nvmeq, ··· 641 614 return nvmeq->descriptor_pools.large; 642 615 } 643 616 644 644 - static void nvme_free_descriptors(struct nvme_queue *nvmeq, struct request *req) 617 617 + static inline bool nvme_pci_cmd_use_sgl(struct nvme_command *cmd) 645 618 { 619 619 + return cmd->common.flags & 620 620 + (NVME_CMD_SGL_METABUF | NVME_CMD_SGL_METASEG); 621 621 + } 622 622 + 623 623 + static inline dma_addr_t nvme_pci_first_desc_dma_addr(struct nvme_command *cmd) 624 624 + { 625 625 + if (nvme_pci_cmd_use_sgl(cmd)) 626 626 + return le64_to_cpu(cmd->common.dptr.sgl.addr); 627 627 + return le64_to_cpu(cmd->common.dptr.prp2); 628 628 + } 629 629 + 630 630 + static void nvme_free_descriptors(struct request *req) 631 631 + { 632 632 + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 646 633 const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1; 647 634 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 648 648 - dma_addr_t dma_addr = iod->first_dma; 635 635 + dma_addr_t dma_addr = nvme_pci_first_desc_dma_addr(&iod->cmd); 649 636 int i; 650 637 651 638 if (iod->nr_descriptors == 1) { ··· 678 637 } 679 638 } 680 639 681 681 - static void nvme_unmap_data(struct nvme_dev *dev, struct nvme_queue *nvmeq, 682 682 - struct request *req) 640 640 + static void nvme_free_prps(struct request *req) 683 641 { 684 642 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 643 643 + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 644 644 + unsigned int i; 685 645 686 686 - if (iod->dma_len) { 687 687 - dma_unmap_page(dev->dev, iod->first_dma, iod->dma_len, 688 688 - rq_dma_dir(req)); 646 646 + for (i = 0; i < iod->nr_dma_vecs; i++) 647 647 + dma_unmap_page(nvmeq->dev->dev, iod->dma_vecs[i].addr, 648 648 + iod->dma_vecs[i].len, rq_dma_dir(req)); 649 649 + mempool_free(iod->dma_vecs, nvmeq->dev->dmavec_mempool); 650 650 + } 651 651 + 652 652 + static void nvme_free_sgls(struct request *req) 653 653 + { 654 654 + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 655 655 + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 656 656 + struct device *dma_dev = nvmeq->dev->dev; 657 657 + dma_addr_t sqe_dma_addr = le64_to_cpu(iod->cmd.common.dptr.sgl.addr); 658 658 + unsigned int sqe_dma_len = le32_to_cpu(iod->cmd.common.dptr.sgl.length); 659 659 + struct nvme_sgl_desc *sg_list = iod->descriptors[0]; 660 660 + enum dma_data_direction dir = rq_dma_dir(req); 661 661 + 662 662 + if (iod->nr_descriptors) { 663 663 + unsigned int nr_entries = sqe_dma_len / sizeof(*sg_list), i; 664 664 + 665 665 + for (i = 0; i < nr_entries; i++) 666 666 + dma_unmap_page(dma_dev, le64_to_cpu(sg_list[i].addr), 667 667 + le32_to_cpu(sg_list[i].length), dir); 668 668 + } else { 669 669 + dma_unmap_page(dma_dev, sqe_dma_addr, sqe_dma_len, dir); 670 670 + } 671 671 + } 672 672 + 673 673 + static void nvme_unmap_data(struct request *req) 674 674 + { 675 675 + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 676 676 + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 677 677 + struct device *dma_dev = nvmeq->dev->dev; 678 678 + 679 679 + if (iod->flags & IOD_SINGLE_SEGMENT) { 680 680 + static_assert(offsetof(union nvme_data_ptr, prp1) == 681 681 + offsetof(union nvme_data_ptr, sgl.addr)); 682 682 + dma_unmap_page(dma_dev, le64_to_cpu(iod->cmd.common.dptr.prp1), 683 683 + iod->total_len, rq_dma_dir(req)); 689 684 return; 690 685 } 691 686 692 692 - WARN_ON_ONCE(!iod->sgt.nents); 693 693 - 694 694 - dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0); 695 695 - nvme_free_descriptors(nvmeq, req); 696 696 - mempool_free(iod->sgt.sgl, dev->iod_mempool); 697 697 - } 698 698 - 699 699 - static void nvme_print_sgl(struct scatterlist *sgl, int nents) 700 700 - { 701 701 - int i; 702 702 - struct scatterlist *sg; 703 703 - 704 704 - for_each_sg(sgl, sg, nents, i) { 705 705 - dma_addr_t phys = sg_phys(sg); 706 706 - pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d " 707 707 - "dma_address:%pad dma_length:%d\n", 708 708 - i, &phys, sg->offset, sg->length, &sg_dma_address(sg), 709 709 - sg_dma_len(sg)); 687 687 + if (!blk_rq_dma_unmap(req, dma_dev, &iod->dma_state, iod->total_len)) { 688 688 + if (nvme_pci_cmd_use_sgl(&iod->cmd)) 689 689 + nvme_free_sgls(req); 690 690 + else 691 691 + nvme_free_prps(req); 710 692 } 693 693 + 694 694 + if (iod->nr_descriptors) 695 695 + nvme_free_descriptors(req); 711 696 } 712 697 713 713 - static blk_status_t nvme_pci_setup_prps(struct nvme_queue *nvmeq, 714 714 - struct request *req, struct nvme_rw_command *cmnd) 698 698 + static bool nvme_pci_prp_iter_next(struct request *req, struct device *dma_dev, 699 699 + struct blk_dma_iter *iter) 715 700 { 716 701 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 717 717 - int length = blk_rq_payload_bytes(req); 718 718 - struct scatterlist *sg = iod->sgt.sgl; 719 719 - int dma_len = sg_dma_len(sg); 720 720 - u64 dma_addr = sg_dma_address(sg); 721 721 - int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1); 722 722 - __le64 *prp_list; 723 723 - dma_addr_t prp_dma; 724 724 - int i; 725 702 726 726 - length -= (NVME_CTRL_PAGE_SIZE - offset); 727 727 - if (length <= 0) { 728 728 - iod->first_dma = 0; 703 703 + if (iter->len) 704 704 + return true; 705 705 + if (!blk_rq_dma_map_iter_next(req, dma_dev, &iod->dma_state, iter)) 706 706 + return false; 707 707 + if (!dma_use_iova(&iod->dma_state) && dma_need_unmap(dma_dev)) { 708 708 + iod->dma_vecs[iod->nr_dma_vecs].addr = iter->addr; 709 709 + iod->dma_vecs[iod->nr_dma_vecs].len = iter->len; 710 710 + iod->nr_dma_vecs++; 711 711 + } 712 712 + return true; 713 713 + } 714 714 + 715 715 + static blk_status_t nvme_pci_setup_data_prp(struct request *req, 716 716 + struct blk_dma_iter *iter) 717 717 + { 718 718 + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 719 719 + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 720 720 + unsigned int length = blk_rq_payload_bytes(req); 721 721 + dma_addr_t prp1_dma, prp2_dma = 0; 722 722 + unsigned int prp_len, i; 723 723 + __le64 *prp_list; 724 724 + 725 725 + if (!dma_use_iova(&iod->dma_state) && dma_need_unmap(nvmeq->dev->dev)) { 726 726 + iod->dma_vecs = mempool_alloc(nvmeq->dev->dmavec_mempool, 727 727 + GFP_ATOMIC); 728 728 + if (!iod->dma_vecs) 729 729 + return BLK_STS_RESOURCE; 730 730 + iod->dma_vecs[0].addr = iter->addr; 731 731 + iod->dma_vecs[0].len = iter->len; 732 732 + iod->nr_dma_vecs = 1; 733 733 + } 734 734 + 735 735 + /* 736 736 + * PRP1 always points to the start of the DMA transfers. 737 737 + * 738 738 + * This is the only PRP (except for the list entries) that could be 739 739 + * non-aligned. 740 740 + */ 741 741 + prp1_dma = iter->addr; 742 742 + prp_len = min(length, NVME_CTRL_PAGE_SIZE - 743 743 + (iter->addr & (NVME_CTRL_PAGE_SIZE - 1))); 744 744 + iod->total_len += prp_len; 745 745 + iter->addr += prp_len; 746 746 + iter->len -= prp_len; 747 747 + length -= prp_len; 748 748 + if (!length) 749 749 + goto done; 750 750 + 751 751 + if (!nvme_pci_prp_iter_next(req, nvmeq->dev->dev, iter)) { 752 752 + if (WARN_ON_ONCE(!iter->status)) 753 753 + goto bad_sgl; 729 754 goto done; 730 755 } 731 756 732 732 - dma_len -= (NVME_CTRL_PAGE_SIZE - offset); 733 733 - if (dma_len) { 734 734 - dma_addr += (NVME_CTRL_PAGE_SIZE - offset); 735 735 - } else { 736 736 - sg = sg_next(sg); 737 737 - dma_addr = sg_dma_address(sg); 738 738 - dma_len = sg_dma_len(sg); 739 739 - } 740 740 - 757 757 + /* 758 758 + * PRP2 is usually a list, but can point to data if all data to be 759 759 + * transferred fits into PRP1 + PRP2: 760 760 + */ 741 761 if (length <= NVME_CTRL_PAGE_SIZE) { 742 742 - iod->first_dma = dma_addr; 762 762 + prp2_dma = iter->addr; 763 763 + iod->total_len += length; 743 764 goto done; 744 765 } 745 766 ··· 810 707 iod->flags |= IOD_SMALL_DESCRIPTOR; 811 708 812 709 prp_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC, 813 813 - &prp_dma); 814 814 - if (!prp_list) 815 815 - return BLK_STS_RESOURCE; 710 710 + &prp2_dma); 711 711 + if (!prp_list) { 712 712 + iter->status = BLK_STS_RESOURCE; 713 713 + goto done; 714 714 + } 816 715 iod->descriptors[iod->nr_descriptors++] = prp_list; 817 817 - iod->first_dma = prp_dma; 716 716 + 818 717 i = 0; 819 718 for (;;) { 719 719 + prp_list[i++] = cpu_to_le64(iter->addr); 720 720 + prp_len = min(length, NVME_CTRL_PAGE_SIZE); 721 721 + if (WARN_ON_ONCE(iter->len < prp_len)) 722 722 + goto bad_sgl; 723 723 + 724 724 + iod->total_len += prp_len; 725 725 + iter->addr += prp_len; 726 726 + iter->len -= prp_len; 727 727 + length -= prp_len; 728 728 + if (!length) 729 729 + break; 730 730 + 731 731 + if (!nvme_pci_prp_iter_next(req, nvmeq->dev->dev, iter)) { 732 732 + if (WARN_ON_ONCE(!iter->status)) 733 733 + goto bad_sgl; 734 734 + goto done; 735 735 + } 736 736 + 737 737 + /* 738 738 + * If we've filled the entire descriptor, allocate a new that is 739 739 + * pointed to be the last entry in the previous PRP list. To 740 740 + * accommodate for that move the last actual entry to the new 741 741 + * descriptor. 742 742 + */ 820 743 if (i == NVME_CTRL_PAGE_SIZE >> 3) { 821 744 __le64 *old_prp_list = prp_list; 745 745 + dma_addr_t prp_list_dma; 822 746 823 747 prp_list = dma_pool_alloc(nvmeq->descriptor_pools.large, 824 824 - GFP_ATOMIC, &prp_dma); 825 825 - if (!prp_list) 826 826 - goto free_prps; 748 748 + GFP_ATOMIC, &prp_list_dma); 749 749 + if (!prp_list) { 750 750 + iter->status = BLK_STS_RESOURCE; 751 751 + goto done; 752 752 + } 827 753 iod->descriptors[iod->nr_descriptors++] = prp_list; 754 754 + 828 755 prp_list[0] = old_prp_list[i - 1]; 829 829 - old_prp_list[i - 1] = cpu_to_le64(prp_dma); 756 756 + old_prp_list[i - 1] = cpu_to_le64(prp_list_dma); 830 757 i = 1; 831 758 } 832 832 - prp_list[i++] = cpu_to_le64(dma_addr); 833 833 - dma_len -= NVME_CTRL_PAGE_SIZE; 834 834 - dma_addr += NVME_CTRL_PAGE_SIZE; 835 835 - length -= NVME_CTRL_PAGE_SIZE; 836 836 - if (length <= 0) 837 837 - break; 838 838 - if (dma_len > 0) 839 839 - continue; 840 840 - if (unlikely(dma_len < 0)) 841 841 - goto bad_sgl; 842 842 - sg = sg_next(sg); 843 843 - dma_addr = sg_dma_address(sg); 844 844 - dma_len = sg_dma_len(sg); 845 759 } 760 760 + 846 761 done: 847 847 - cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sgt.sgl)); 848 848 - cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma); 849 849 - return BLK_STS_OK; 850 850 - free_prps: 851 851 - nvme_free_descriptors(nvmeq, req); 852 852 - return BLK_STS_RESOURCE; 762 762 + /* 763 763 + * nvme_unmap_data uses the DPT field in the SQE to tear down the 764 764 + * mapping, so initialize it even for failures. 765 765 + */ 766 766 + iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma); 767 767 + iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma); 768 768 + if (unlikely(iter->status)) 769 769 + nvme_unmap_data(req); 770 770 + return iter->status; 771 771 + 853 772 bad_sgl: 854 854 - WARN(DO_ONCE(nvme_print_sgl, iod->sgt.sgl, iod->sgt.nents), 855 855 - "Invalid SGL for payload:%d nents:%d\n", 856 856 - blk_rq_payload_bytes(req), iod->sgt.nents); 773 773 + dev_err_once(nvmeq->dev->dev, 774 774 + "Incorrectly formed request for payload:%d nents:%d\n", 775 775 + blk_rq_payload_bytes(req), blk_rq_nr_phys_segments(req)); 857 776 return BLK_STS_IOERR; 858 777 } 859 778 860 779 static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge, 861 861 - struct scatterlist *sg) 780 780 + struct blk_dma_iter *iter) 862 781 { 863 863 - sge->addr = cpu_to_le64(sg_dma_address(sg)); 864 864 - sge->length = cpu_to_le32(sg_dma_len(sg)); 782 782 + sge->addr = cpu_to_le64(iter->addr); 783 783 + sge->length = cpu_to_le32(iter->len); 865 784 sge->type = NVME_SGL_FMT_DATA_DESC << 4; 866 785 } 867 786 ··· 895 770 sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4; 896 771 } 897 772 898 898 - static blk_status_t nvme_pci_setup_sgls(struct nvme_queue *nvmeq, 899 899 - struct request *req, struct nvme_rw_command *cmd) 773 773 + static blk_status_t nvme_pci_setup_data_sgl(struct request *req, 774 774 + struct blk_dma_iter *iter) 900 775 { 901 776 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 777 777 + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 778 778 + unsigned int entries = blk_rq_nr_phys_segments(req); 902 779 struct nvme_sgl_desc *sg_list; 903 903 - struct scatterlist *sg = iod->sgt.sgl; 904 904 - unsigned int entries = iod->sgt.nents; 905 780 dma_addr_t sgl_dma; 906 906 - int i = 0; 781 781 + unsigned int mapped = 0; 907 782 908 908 - /* setting the transfer type as SGL */ 909 909 - cmd->flags = NVME_CMD_SGL_METABUF; 783 783 + /* set the transfer type as SGL */ 784 784 + iod->cmd.common.flags = NVME_CMD_SGL_METABUF; 910 785 911 911 - if (entries == 1) { 912 912 - nvme_pci_sgl_set_data(&cmd->dptr.sgl, sg); 786 786 + if (entries == 1 || blk_rq_dma_map_coalesce(&iod->dma_state)) { 787 787 + nvme_pci_sgl_set_data(&iod->cmd.common.dptr.sgl, iter); 788 788 + iod->total_len += iter->len; 913 789 return BLK_STS_OK; 914 790 } 915 791 ··· 922 796 if (!sg_list) 923 797 return BLK_STS_RESOURCE; 924 798 iod->descriptors[iod->nr_descriptors++] = sg_list; 925 925 - iod->first_dma = sgl_dma; 926 799 927 927 - nvme_pci_sgl_set_seg(&cmd->dptr.sgl, sgl_dma, entries); 928 800 do { 929 929 - nvme_pci_sgl_set_data(&sg_list[i++], sg); 930 930 - sg = sg_next(sg); 931 931 - } while (--entries > 0); 932 932 - 933 933 - return BLK_STS_OK; 934 934 - } 935 935 - 936 936 - static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev, 937 937 - struct request *req, struct nvme_rw_command *cmnd, 938 938 - struct bio_vec *bv) 939 939 - { 940 940 - struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 941 941 - unsigned int offset = bv->bv_offset & (NVME_CTRL_PAGE_SIZE - 1); 942 942 - unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - offset; 943 943 - 944 944 - iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0); 945 945 - if (dma_mapping_error(dev->dev, iod->first_dma)) 946 946 - return BLK_STS_RESOURCE; 947 947 - iod->dma_len = bv->bv_len; 948 948 - 949 949 - cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma); 950 950 - if (bv->bv_len > first_prp_len) 951 951 - cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len); 952 952 - else 953 953 - cmnd->dptr.prp2 = 0; 954 954 - return BLK_STS_OK; 955 955 - } 956 956 - 957 957 - static blk_status_t nvme_setup_sgl_simple(struct nvme_dev *dev, 958 958 - struct request *req, struct nvme_rw_command *cmnd, 959 959 - struct bio_vec *bv) 960 960 - { 961 961 - struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 962 962 - 963 963 - iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0); 964 964 - if (dma_mapping_error(dev->dev, iod->first_dma)) 965 965 - return BLK_STS_RESOURCE; 966 966 - iod->dma_len = bv->bv_len; 967 967 - 968 968 - cmnd->flags = NVME_CMD_SGL_METABUF; 969 969 - cmnd->dptr.sgl.addr = cpu_to_le64(iod->first_dma); 970 970 - cmnd->dptr.sgl.length = cpu_to_le32(iod->dma_len); 971 971 - cmnd->dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4; 972 972 - return BLK_STS_OK; 973 973 - } 974 974 - 975 975 - static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, 976 976 - struct nvme_command *cmnd) 977 977 - { 978 978 - struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 979 979 - struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 980 980 - blk_status_t ret = BLK_STS_RESOURCE; 981 981 - int rc; 982 982 - 983 983 - if (blk_rq_nr_phys_segments(req) == 1) { 984 984 - struct bio_vec bv = req_bvec(req); 985 985 - 986 986 - if (!is_pci_p2pdma_page(bv.bv_page)) { 987 987 - if (!nvme_pci_metadata_use_sgls(dev, req) && 988 988 - (bv.bv_offset & (NVME_CTRL_PAGE_SIZE - 1)) + 989 989 - bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2) 990 990 - return nvme_setup_prp_simple(dev, req, 991 991 - &cmnd->rw, &bv); 992 992 - 993 993 - if (nvmeq->qid && sgl_threshold && 994 994 - nvme_ctrl_sgl_supported(&dev->ctrl)) 995 995 - return nvme_setup_sgl_simple(dev, req, 996 996 - &cmnd->rw, &bv); 801 801 + if (WARN_ON_ONCE(mapped == entries)) { 802 802 + iter->status = BLK_STS_IOERR; 803 803 + break; 997 804 } 998 998 - } 805 805 + nvme_pci_sgl_set_data(&sg_list[mapped++], iter); 806 806 + iod->total_len += iter->len; 807 807 + } while (blk_rq_dma_map_iter_next(req, nvmeq->dev->dev, &iod->dma_state, 808 808 + iter)); 999 809 1000 1000 - iod->dma_len = 0; 1001 1001 - iod->sgt.sgl = mempool_alloc(dev->iod_mempool, GFP_ATOMIC); 1002 1002 - if (!iod->sgt.sgl) 1003 1003 - return BLK_STS_RESOURCE; 1004 1004 - sg_init_table(iod->sgt.sgl, blk_rq_nr_phys_segments(req)); 1005 1005 - iod->sgt.orig_nents = blk_rq_map_sg(req, iod->sgt.sgl); 1006 1006 - if (!iod->sgt.orig_nents) 1007 1007 - goto out_free_sg; 1008 1008 - 1009 1009 - rc = dma_map_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 1010 1010 - DMA_ATTR_NO_WARN); 1011 1011 - if (rc) { 1012 1012 - if (rc == -EREMOTEIO) 1013 1013 - ret = BLK_STS_TARGET; 1014 1014 - goto out_free_sg; 1015 1015 - } 1016 1016 - 1017 1017 - if (nvme_pci_use_sgls(dev, req, iod->sgt.nents)) 1018 1018 - ret = nvme_pci_setup_sgls(nvmeq, req, &cmnd->rw); 1019 1019 - else 1020 1020 - ret = nvme_pci_setup_prps(nvmeq, req, &cmnd->rw); 1021 1021 - if (ret != BLK_STS_OK) 1022 1022 - goto out_unmap_sg; 1023 1023 - return BLK_STS_OK; 1024 1024 - 1025 1025 - out_unmap_sg: 1026 1026 - dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0); 1027 1027 - out_free_sg: 1028 1028 - mempool_free(iod->sgt.sgl, dev->iod_mempool); 1029 1029 - return ret; 810 810 + nvme_pci_sgl_set_seg(&iod->cmd.common.dptr.sgl, sgl_dma, mapped); 811 811 + if (unlikely(iter->status)) 812 812 + nvme_free_sgls(req); 813 813 + return iter->status; 1030 814 } 1031 815 1032 1032 - static blk_status_t nvme_pci_setup_meta_sgls(struct nvme_dev *dev, 1033 1033 - struct request *req) 816 816 + static blk_status_t nvme_pci_setup_data_simple(struct request *req, 817 817 + enum nvme_use_sgl use_sgl) 818 818 + { 819 819 + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 820 820 + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 821 821 + struct bio_vec bv = req_bvec(req); 822 822 + unsigned int prp1_offset = bv.bv_offset & (NVME_CTRL_PAGE_SIZE - 1); 823 823 + bool prp_possible = prp1_offset + bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2; 824 824 + dma_addr_t dma_addr; 825 825 + 826 826 + if (!use_sgl && !prp_possible) 827 827 + return BLK_STS_AGAIN; 828 828 + if (is_pci_p2pdma_page(bv.bv_page)) 829 829 + return BLK_STS_AGAIN; 830 830 + 831 831 + dma_addr = dma_map_bvec(nvmeq->dev->dev, &bv, rq_dma_dir(req), 0); 832 832 + if (dma_mapping_error(nvmeq->dev->dev, dma_addr)) 833 833 + return BLK_STS_RESOURCE; 834 834 + iod->total_len = bv.bv_len; 835 835 + iod->flags |= IOD_SINGLE_SEGMENT; 836 836 + 837 837 + if (use_sgl == SGL_FORCED || !prp_possible) { 838 838 + iod->cmd.common.flags = NVME_CMD_SGL_METABUF; 839 839 + iod->cmd.common.dptr.sgl.addr = cpu_to_le64(dma_addr); 840 840 + iod->cmd.common.dptr.sgl.length = cpu_to_le32(bv.bv_len); 841 841 + iod->cmd.common.dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4; 842 842 + } else { 843 843 + unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - prp1_offset; 844 844 + 845 845 + iod->cmd.common.dptr.prp1 = cpu_to_le64(dma_addr); 846 846 + iod->cmd.common.dptr.prp2 = 0; 847 847 + if (bv.bv_len > first_prp_len) 848 848 + iod->cmd.common.dptr.prp2 = 849 849 + cpu_to_le64(dma_addr + first_prp_len); 850 850 + } 851 851 + 852 852 + return BLK_STS_OK; 853 853 + } 854 854 + 855 855 + static blk_status_t nvme_map_data(struct request *req) 856 856 + { 857 857 + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 858 858 + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 859 859 + struct nvme_dev *dev = nvmeq->dev; 860 860 + enum nvme_use_sgl use_sgl = nvme_pci_use_sgls(dev, req); 861 861 + struct blk_dma_iter iter; 862 862 + blk_status_t ret; 863 863 + 864 864 + /* 865 865 + * Try to skip the DMA iterator for single segment requests, as that 866 866 + * significantly improves performances for small I/O sizes. 867 867 + */ 868 868 + if (blk_rq_nr_phys_segments(req) == 1) { 869 869 + ret = nvme_pci_setup_data_simple(req, use_sgl); 870 870 + if (ret != BLK_STS_AGAIN) 871 871 + return ret; 872 872 + } 873 873 + 874 874 + if (!blk_rq_dma_map_iter_start(req, dev->dev, &iod->dma_state, &iter)) 875 875 + return iter.status; 876 876 + 877 877 + if (use_sgl == SGL_FORCED || 878 878 + (use_sgl == SGL_SUPPORTED && 879 879 + (sgl_threshold && nvme_pci_avg_seg_size(req) >= sgl_threshold))) 880 880 + return nvme_pci_setup_data_sgl(req, &iter); 881 881 + return nvme_pci_setup_data_prp(req, &iter); 882 882 + } 883 883 + 884 884 + static void nvme_pci_sgl_set_data_sg(struct nvme_sgl_desc *sge, 885 885 + struct scatterlist *sg) 886 886 + { 887 887 + sge->addr = cpu_to_le64(sg_dma_address(sg)); 888 888 + sge->length = cpu_to_le32(sg_dma_len(sg)); 889 889 + sge->type = NVME_SGL_FMT_DATA_DESC << 4; 890 890 + } 891 891 + 892 892 + static blk_status_t nvme_pci_setup_meta_sgls(struct request *req) 1034 893 { 1035 894 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 895 895 + struct nvme_dev *dev = nvmeq->dev; 1036 896 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1037 1037 - struct nvme_rw_command *cmnd = &iod->cmd.rw; 1038 897 struct nvme_sgl_desc *sg_list; 1039 898 struct scatterlist *sgl, *sg; 1040 899 unsigned int entries; ··· 1050 939 iod->meta_descriptor = sg_list; 1051 940 iod->meta_dma = sgl_dma; 1052 941 1053 1053 - cmnd->flags = NVME_CMD_SGL_METASEG; 1054 1054 - cmnd->metadata = cpu_to_le64(sgl_dma); 942 942 + iod->cmd.common.flags = NVME_CMD_SGL_METASEG; 943 943 + iod->cmd.common.metadata = cpu_to_le64(sgl_dma); 1055 944 1056 945 sgl = iod->meta_sgt.sgl; 1057 946 if (entries == 1) { 1058 1058 - nvme_pci_sgl_set_data(sg_list, sgl); 947 947 + nvme_pci_sgl_set_data_sg(sg_list, sgl); 1059 948 return BLK_STS_OK; 1060 949 } 1061 950 1062 951 sgl_dma += sizeof(*sg_list); 1063 952 nvme_pci_sgl_set_seg(sg_list, sgl_dma, entries); 1064 953 for_each_sg(sgl, sg, entries, i) 1065 1065 - nvme_pci_sgl_set_data(&sg_list[i + 1], sg); 954 954 + nvme_pci_sgl_set_data_sg(&sg_list[i + 1], sg); 1066 955 1067 956 return BLK_STS_OK; 1068 957 ··· 1073 962 return BLK_STS_RESOURCE; 1074 963 } 1075 964 1076 1076 - static blk_status_t nvme_pci_setup_meta_mptr(struct nvme_dev *dev, 1077 1077 - struct request *req) 965 965 + static blk_status_t nvme_pci_setup_meta_mptr(struct request *req) 1078 966 { 1079 967 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 968 968 + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1080 969 struct bio_vec bv = rq_integrity_vec(req); 1081 1081 - struct nvme_command *cmnd = &iod->cmd; 1082 970 1083 1083 - iod->meta_dma = dma_map_bvec(dev->dev, &bv, rq_dma_dir(req), 0); 1084 1084 - if (dma_mapping_error(dev->dev, iod->meta_dma)) 971 971 + iod->meta_dma = dma_map_bvec(nvmeq->dev->dev, &bv, rq_dma_dir(req), 0); 972 972 + if (dma_mapping_error(nvmeq->dev->dev, iod->meta_dma)) 1085 973 return BLK_STS_IOERR; 1086 1086 - cmnd->rw.metadata = cpu_to_le64(iod->meta_dma); 974 974 + iod->cmd.common.metadata = cpu_to_le64(iod->meta_dma); 1087 975 return BLK_STS_OK; 1088 976 } 1089 977 1090 1090 - static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req) 978 978 + static blk_status_t nvme_map_metadata(struct request *req) 1091 979 { 1092 980 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1093 981 1094 982 if ((iod->cmd.common.flags & NVME_CMD_SGL_METABUF) && 1095 1095 - nvme_pci_metadata_use_sgls(dev, req)) 1096 1096 - return nvme_pci_setup_meta_sgls(dev, req); 1097 1097 - return nvme_pci_setup_meta_mptr(dev, req); 983 983 + nvme_pci_metadata_use_sgls(req)) 984 984 + return nvme_pci_setup_meta_sgls(req); 985 985 + return nvme_pci_setup_meta_mptr(req); 1098 986 } 1099 987 1100 1100 - static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req) 988 988 + static blk_status_t nvme_prep_rq(struct request *req) 1101 989 { 1102 990 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1103 991 blk_status_t ret; 1104 992 1105 993 iod->flags = 0; 1106 994 iod->nr_descriptors = 0; 1107 1107 - iod->sgt.nents = 0; 995 995 + iod->total_len = 0; 1108 996 iod->meta_sgt.nents = 0; 1109 997 1110 998 ret = nvme_setup_cmd(req->q->queuedata, req); ··· 1111 1001 return ret; 1112 1002 1113 1003 if (blk_rq_nr_phys_segments(req)) { 1114 1114 - ret = nvme_map_data(dev, req, &iod->cmd); 1004 1004 + ret = nvme_map_data(req); 1115 1005 if (ret) 1116 1006 goto out_free_cmd; 1117 1007 } 1118 1008 1119 1009 if (blk_integrity_rq(req)) { 1120 1120 - ret = nvme_map_metadata(dev, req); 1010 1010 + ret = nvme_map_metadata(req); 1121 1011 if (ret) 1122 1012 goto out_unmap_data; 1123 1013 } ··· 1126 1016 return BLK_STS_OK; 1127 1017 out_unmap_data: 1128 1018 if (blk_rq_nr_phys_segments(req)) 1129 1129 - nvme_unmap_data(dev, req->mq_hctx->driver_data, req); 1019 1019 + nvme_unmap_data(req); 1130 1020 out_free_cmd: 1131 1021 nvme_cleanup_cmd(req); 1132 1022 return ret; ··· 1151 1041 if (unlikely(!nvme_check_ready(&dev->ctrl, req, true))) 1152 1042 return nvme_fail_nonready_command(&dev->ctrl, req); 1153 1043 1154 1154 - ret = nvme_prep_rq(dev, req); 1044 1044 + ret = nvme_prep_rq(req); 1155 1045 if (unlikely(ret)) 1156 1046 return ret; 1157 1047 spin_lock(&nvmeq->sq_lock); ··· 1189 1079 if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true))) 1190 1080 return false; 1191 1081 1192 1192 - return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK; 1082 1082 + return nvme_prep_rq(req) == BLK_STS_OK; 1193 1083 } 1194 1084 1195 1085 static void nvme_queue_rqs(struct rq_list *rqlist) ··· 1215 1105 *rqlist = requeue_list; 1216 1106 } 1217 1107 1218 1218 - static __always_inline void nvme_unmap_metadata(struct nvme_dev *dev, 1219 1219 - struct nvme_queue *nvmeq, 1220 1220 - struct request *req) 1108 1108 + static __always_inline void nvme_unmap_metadata(struct request *req) 1221 1109 { 1222 1110 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1111 1111 + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1112 1112 + struct nvme_dev *dev = nvmeq->dev; 1223 1113 1224 1114 if (!iod->meta_sgt.nents) { 1225 1115 dma_unmap_page(dev->dev, iod->meta_dma, ··· 1236 1126 1237 1127 static __always_inline void nvme_pci_unmap_rq(struct request *req) 1238 1128 { 1239 1239 - struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1240 1240 - struct nvme_dev *dev = nvmeq->dev; 1241 1241 - 1242 1129 if (blk_integrity_rq(req)) 1243 1243 - nvme_unmap_metadata(dev, nvmeq, req); 1244 1244 - 1130 1130 + nvme_unmap_metadata(req); 1245 1131 if (blk_rq_nr_phys_segments(req)) 1246 1246 - nvme_unmap_data(dev, nvmeq, req); 1132 1132 + nvme_unmap_data(req); 1247 1133 } 1248 1134 1249 1135 static void nvme_pci_complete_rq(struct request *req) ··· 2064 1958 * might be pointing at! 2065 1959 */ 2066 1960 result = nvme_disable_ctrl(&dev->ctrl, false); 2067 2067 - if (result < 0) 2068 2068 - return result; 1961 1961 + if (result < 0) { 1962 1962 + struct pci_dev *pdev = to_pci_dev(dev->dev); 1963 1963 + 1964 1964 + /* 1965 1965 + * The NVMe Controller Reset method did not get an expected 1966 1966 + * CSTS.RDY transition, so something with the device appears to 1967 1967 + * be stuck. Use the lower level and bigger hammer PCIe 1968 1968 + * Function Level Reset to attempt restoring the device to its 1969 1969 + * initial state, and try again. 1970 1970 + */ 1971 1971 + result = pcie_reset_flr(pdev, false); 1972 1972 + if (result < 0) 1973 1973 + return result; 1974 1974 + 1975 1975 + pci_restore_state(pdev); 1976 1976 + result = nvme_disable_ctrl(&dev->ctrl, false); 1977 1977 + if (result < 0) 1978 1978 + return result; 1979 1979 + 1980 1980 + dev_info(dev->ctrl.device, 1981 1981 + "controller reset completed after pcie flr\n"); 1982 1982 + } 2069 1983 2070 1984 result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH); 2071 1985 if (result) ··· 2457 2331 { 2458 2332 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); 2459 2333 2460 2460 - return sysfs_emit(buf, "cmbloc : x%08x\ncmbsz : x%08x\n", 2334 2334 + return sysfs_emit(buf, "cmbloc : 0x%08x\ncmbsz : 0x%08x\n", 2461 2335 ndev->cmbloc, ndev->cmbsz); 2462 2336 } 2463 2337 static DEVICE_ATTR_RO(cmb); ··· 2644 2518 */ 2645 2519 if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) 2646 2520 return 1; 2647 2647 - return num_possible_cpus() + dev->nr_write_queues + dev->nr_poll_queues; 2521 2521 + return blk_mq_num_possible_queues(0) + dev->nr_write_queues + 2522 2522 + dev->nr_poll_queues; 2648 2523 } 2649 2524 2650 2525 static int nvme_setup_io_queues(struct nvme_dev *dev) ··· 3040 2913 static int nvme_pci_alloc_iod_mempool(struct nvme_dev *dev) 3041 2914 { 3042 2915 size_t meta_size = sizeof(struct scatterlist) * (NVME_MAX_META_SEGS + 1); 3043 3043 - size_t alloc_size = sizeof(struct scatterlist) * NVME_MAX_SEGS; 2916 2916 + size_t alloc_size = sizeof(struct nvme_dma_vec) * NVME_MAX_SEGS; 3044 2917 3045 3045 - dev->iod_mempool = mempool_create_node(1, 2918 2918 + dev->dmavec_mempool = mempool_create_node(1, 3046 2919 mempool_kmalloc, mempool_kfree, 3047 2920 (void *)alloc_size, GFP_KERNEL, 3048 2921 dev_to_node(dev->dev)); 3049 3049 - if (!dev->iod_mempool) 2922 2922 + if (!dev->dmavec_mempool) 3050 2923 return -ENOMEM; 3051 2924 3052 2925 dev->iod_meta_mempool = mempool_create_node(1, ··· 3055 2928 dev_to_node(dev->dev)); 3056 2929 if (!dev->iod_meta_mempool) 3057 2930 goto free; 3058 3058 - 3059 2931 return 0; 3060 2932 free: 3061 3061 - mempool_destroy(dev->iod_mempool); 2933 2933 + mempool_destroy(dev->dmavec_mempool); 3062 2934 return -ENOMEM; 3063 2935 } 3064 2936 ··· 3398 3272 * over a single page. 3399 3273 */ 3400 3274 dev->ctrl.max_hw_sectors = min_t(u32, 3401 3401 - NVME_MAX_KB_SZ << 1, dma_opt_mapping_size(&pdev->dev) >> 9); 3275 3275 + NVME_MAX_BYTES >> SECTOR_SHIFT, 3276 3276 + dma_opt_mapping_size(&pdev->dev) >> 9); 3402 3277 dev->ctrl.max_segments = NVME_MAX_SEGS; 3403 3278 dev->ctrl.max_integrity_segments = 1; 3404 3279 return dev; ··· 3507 3380 nvme_dbbuf_dma_free(dev); 3508 3381 nvme_free_queues(dev, 0); 3509 3382 out_release_iod_mempool: 3510 3510 - mempool_destroy(dev->iod_mempool); 3383 3383 + mempool_destroy(dev->dmavec_mempool); 3511 3384 mempool_destroy(dev->iod_meta_mempool); 3512 3385 out_dev_unmap: 3513 3386 nvme_dev_unmap(dev); ··· 3571 3444 nvme_dev_remove_admin(dev); 3572 3445 nvme_dbbuf_dma_free(dev); 3573 3446 nvme_free_queues(dev, 0); 3574 3574 - mempool_destroy(dev->iod_mempool); 3447 3447 + mempool_destroy(dev->dmavec_mempool); 3575 3448 mempool_destroy(dev->iod_meta_mempool); 3576 3449 nvme_release_descriptor_pools(dev); 3577 3450 nvme_dev_unmap(dev); ··· 3974 3847 BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 3975 3848 BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 3976 3849 BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2); 3977 3977 - BUILD_BUG_ON(nvme_pci_npages_prp() > NVME_MAX_NR_DESCRIPTORS); 3978 3850 3979 3851 return pci_register_driver(&nvme_driver); 3980 3852 }

+1 -1

drivers/nvme/host/rdma.c

reviewed

··· 877 877 878 878 /* 879 879 * Only start IO queues for which we have allocated the tagset 880 880 - * and limitted it to the available queues. On reconnects, the 880 880 + * and limited it to the available queues. On reconnects, the 881 881 * queue number might have changed. 882 882 */ 883 883 nr_queues = min(ctrl->tag_set.nr_hw_queues + 1, ctrl->ctrl.queue_count);

+8 -3

drivers/nvme/host/tcp.c

reviewed

··· 1745 1745 qid, ret); 1746 1746 tls_handshake_cancel(queue->sock->sk); 1747 1747 } else { 1748 1748 - dev_dbg(nctrl->device, 1749 1749 - "queue %d: TLS handshake complete, error %d\n", 1750 1750 - qid, queue->tls_err); 1748 1748 + if (queue->tls_err) { 1749 1749 + dev_err(nctrl->device, 1750 1750 + "queue %d: TLS handshake complete, error %d\n", 1751 1751 + qid, queue->tls_err); 1752 1752 + } else { 1753 1753 + dev_dbg(nctrl->device, 1754 1754 + "queue %d: TLS handshake complete\n", qid); 1755 1755 + } 1751 1756 ret = queue->tls_err; 1752 1757 } 1753 1758 return ret;

-2

drivers/nvme/target/core.c

reviewed

··· 581 581 if (ns->enabled) 582 582 goto out_unlock; 583 583 584 584 - ret = -EMFILE; 585 585 - 586 584 ret = nvmet_bdev_ns_enable(ns); 587 585 if (ret == -ENOTBLK) 588 586 ret = nvmet_file_ns_enable(ns);

+2 -2

drivers/nvme/target/passthru.c

reviewed

··· 106 106 pctrl->max_hw_sectors); 107 107 108 108 /* 109 109 - * nvmet_passthru_map_sg is limitted to using a single bio so limit 109 109 + * nvmet_passthru_map_sg is limited to using a single bio so limit 110 110 * the mdts based on BIO_MAX_VECS as well 111 111 */ 112 112 max_hw_sectors = min_not_zero(BIO_MAX_VECS << PAGE_SECTORS_SHIFT, ··· 147 147 * When passthru controller is setup using nvme-loop transport it will 148 148 * export the passthru ctrl subsysnqn (PCIe NVMe ctrl) and will fail in 149 149 * the nvme/host/core.c in the nvme_init_subsystem()->nvme_active_ctrl() 150 150 - * code path with duplicate ctr subsynqn. In order to prevent that we 150 150 + * code path with duplicate ctrl subsysnqn. In order to prevent that we 151 151 * mask the passthru-ctrl subsysnqn with the target ctrl subsysnqn. 152 152 */ 153 153 memcpy(id->subnqn, ctrl->subsysnqn, sizeof(id->subnqn));

+17 -8

drivers/nvme/target/pci-epf.c

reviewed

··· 1242 1242 1243 1243 iod->status = le16_to_cpu(req->cqe->status) >> 1; 1244 1244 1245 1245 - /* If we have no data to transfer, directly complete the command. */ 1246 1246 - if (!iod->data_len || iod->dma_dir != DMA_TO_DEVICE) { 1245 1245 + /* 1246 1246 + * If the command failed or we have no data to transfer, complete the 1247 1247 + * command immediately. 1248 1248 + */ 1249 1249 + if (iod->status || !iod->data_len || iod->dma_dir != DMA_TO_DEVICE) { 1247 1250 nvmet_pci_epf_complete_iod(iod); 1248 1251 return; 1249 1252 } ··· 1607 1604 goto complete; 1608 1605 } 1609 1606 1607 1607 + /* 1608 1608 + * If nvmet_req_init() fails (e.g., unsupported opcode) it will call 1609 1609 + * __nvmet_req_complete() internally which will call 1610 1610 + * nvmet_pci_epf_queue_response() and will complete the command directly. 1611 1611 + */ 1610 1612 if (!nvmet_req_init(req, &iod->sq->nvme_sq, &nvmet_pci_epf_fabrics_ops)) 1611 1611 - goto complete; 1613 1613 + return; 1612 1614 1613 1615 iod->data_len = nvmet_req_transfer_len(req); 1614 1616 if (iod->data_len) { ··· 1651 1643 1652 1644 wait_for_completion(&iod->done); 1653 1645 1654 1654 - if (iod->status == NVME_SC_SUCCESS) { 1655 1655 - WARN_ON_ONCE(!iod->data_len || iod->dma_dir != DMA_TO_DEVICE); 1656 1656 - nvmet_pci_epf_transfer_iod_data(iod); 1657 1657 - } 1646 1646 + if (iod->status != NVME_SC_SUCCESS) 1647 1647 + return; 1648 1648 + 1649 1649 + WARN_ON_ONCE(!iod->data_len || iod->dma_dir != DMA_TO_DEVICE); 1650 1650 + nvmet_pci_epf_transfer_iod_data(iod); 1658 1651 1659 1652 complete: 1660 1653 nvmet_pci_epf_complete_iod(iod); ··· 1869 1860 ctrl->io_cqes = 1UL << nvmet_cc_iocqes(ctrl->cc); 1870 1861 if (ctrl->io_cqes < sizeof(struct nvme_completion)) { 1871 1862 dev_err(ctrl->dev, "Unsupported I/O CQES %zu (need %zu)\n", 1872 1872 - ctrl->io_sqes, sizeof(struct nvme_completion)); 1863 1863 + ctrl->io_cqes, sizeof(struct nvme_completion)); 1873 1864 goto err; 1874 1865 } 1875 1866

+1 -1

drivers/nvme/target/zns.c

reviewed

··· 541 541 struct bio *bio; 542 542 int sg_cnt; 543 543 544 544 - /* Request is completed on len mismatch in nvmet_check_transter_len() */ 544 544 + /* Request is completed on len mismatch in nvmet_check_transfer_len() */ 545 545 if (!nvmet_check_transfer_len(req, nvmet_rw_data_len(req))) 546 546 return; 547 547

+9 -6

drivers/scsi/megaraid/megaraid_sas_base.c

reviewed

··· 5971 5971 else 5972 5972 instance->iopoll_q_count = 0; 5973 5973 5974 5974 - num_msix_req = num_online_cpus() + instance->low_latency_index_start; 5974 5974 + num_msix_req = blk_mq_num_online_queues(0) + 5975 5975 + instance->low_latency_index_start; 5975 5976 instance->msix_vectors = min(num_msix_req, 5976 5977 instance->msix_vectors); 5977 5978 ··· 5988 5987 /* Disable Balanced IOPS mode and try realloc vectors */ 5989 5988 instance->perf_mode = MR_LATENCY_PERF_MODE; 5990 5989 instance->low_latency_index_start = 1; 5991 5991 - num_msix_req = num_online_cpus() + instance->low_latency_index_start; 5990 5990 + num_msix_req = blk_mq_num_online_queues(0) + 5991 5991 + instance->low_latency_index_start; 5992 5992 5993 5993 instance->msix_vectors = min(num_msix_req, 5994 5994 instance->msix_vectors); ··· 6245 6243 intr_coalescing = (scratch_pad_1 & MR_INTR_COALESCING_SUPPORT_OFFSET) ? 6246 6244 true : false; 6247 6245 if (intr_coalescing && 6248 6248 - (num_online_cpus() >= MR_HIGH_IOPS_QUEUE_COUNT) && 6246 6246 + (blk_mq_num_online_queues(0) >= MR_HIGH_IOPS_QUEUE_COUNT) && 6249 6247 (instance->msix_vectors == MEGASAS_MAX_MSIX_QUEUES)) 6250 6248 instance->perf_mode = MR_BALANCED_PERF_MODE; 6251 6249 else ··· 6289 6287 else 6290 6288 instance->low_latency_index_start = 1; 6291 6289 6292 6292 - num_msix_req = num_online_cpus() + instance->low_latency_index_start; 6290 6290 + num_msix_req = blk_mq_num_online_queues(0) + 6291 6291 + instance->low_latency_index_start; 6293 6292 6294 6293 instance->msix_vectors = min(num_msix_req, 6295 6294 instance->msix_vectors); ··· 6322 6319 megasas_setup_reply_map(instance); 6323 6320 6324 6321 dev_info(&instance->pdev->dev, 6325 6325 - "current msix/online cpus\t: (%d/%d)\n", 6326 6326 - instance->msix_vectors, (unsigned int)num_online_cpus()); 6322 6322 + "current msix/max num queues\t: (%d/%u)\n", 6323 6323 + instance->msix_vectors, blk_mq_num_online_queues(0)); 6327 6324 dev_info(&instance->pdev->dev, 6328 6325 "RDPQ mode\t: (%s)\n", instance->is_rdpq ? "enabled" : "disabled"); 6329 6326

+5 -5

drivers/scsi/qla2xxx/qla_isr.c

reviewed

··· 4533 4533 if (USER_CTRL_IRQ(ha) || !ha->mqiobase) { 4534 4534 /* user wants to control IRQ setting for target mode */ 4535 4535 ret = pci_alloc_irq_vectors(ha->pdev, min_vecs, 4536 4536 - min((u16)ha->msix_count, (u16)(num_online_cpus() + min_vecs)), 4537 4537 - PCI_IRQ_MSIX); 4536 4536 + blk_mq_num_online_queues(ha->msix_count) + min_vecs, 4537 4537 + PCI_IRQ_MSIX); 4538 4538 } else 4539 4539 ret = pci_alloc_irq_vectors_affinity(ha->pdev, min_vecs, 4540 4540 - min((u16)ha->msix_count, (u16)(num_online_cpus() + min_vecs)), 4541 4541 - PCI_IRQ_MSIX | PCI_IRQ_AFFINITY, 4542 4542 - &desc); 4540 4540 + blk_mq_num_online_queues(ha->msix_count) + min_vecs, 4541 4541 + PCI_IRQ_MSIX | PCI_IRQ_AFFINITY, 4542 4542 + &desc); 4543 4543 4544 4544 if (ret < 0) { 4545 4545 ql_log(ql_log_fatal, vha, 0x00c7,

+2 -3

drivers/scsi/smartpqi/smartpqi_init.c

reviewed

··· 5294 5294 if (is_kdump_kernel()) { 5295 5295 num_queue_groups = 1; 5296 5296 } else { 5297 5297 - int num_cpus; 5298 5297 int max_queue_groups; 5299 5298 5300 5299 max_queue_groups = min(ctrl_info->max_inbound_queues / 2, 5301 5300 ctrl_info->max_outbound_queues - 1); 5302 5301 max_queue_groups = min(max_queue_groups, PQI_MAX_QUEUE_GROUPS); 5303 5302 5304 5304 - num_cpus = num_online_cpus(); 5305 5305 - num_queue_groups = min(num_cpus, ctrl_info->max_msix_vectors); 5303 5303 + num_queue_groups = 5304 5304 + blk_mq_num_online_queues(ctrl_info->max_msix_vectors); 5306 5305 num_queue_groups = min(num_queue_groups, max_queue_groups); 5307 5306 } 5308 5307

+1

drivers/scsi/virtio_scsi.c

reviewed

··· 919 919 /* We need to know how many queues before we allocate. */ 920 920 num_queues = virtscsi_config_get(vdev, num_queues) ? : 1; 921 921 num_queues = min_t(unsigned int, nr_cpu_ids, num_queues); 922 922 + num_queues = blk_mq_num_possible_queues(num_queues); 922 923 923 924 num_targets = virtscsi_config_get(vdev, max_target) + 1; 924 925

+5 -4

drivers/virtio/virtio_vdpa.c

reviewed

··· 329 329 330 330 for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) { 331 331 unsigned int this_vecs = affd->set_size[i]; 332 332 + unsigned int nr_masks; 332 333 int j; 333 333 - struct cpumask *result = group_cpus_evenly(this_vecs); 334 334 + struct cpumask *result = group_cpus_evenly(this_vecs, &nr_masks); 334 335 335 336 if (!result) { 336 337 kfree(masks); 337 338 return NULL; 338 339 } 339 340 340 340 - for (j = 0; j < this_vecs; j++) 341 341 + for (j = 0; j < nr_masks; j++) 341 342 cpumask_copy(&masks[curvec + j], &result[j]); 342 343 kfree(result); 343 344 344 344 - curvec += this_vecs; 345 345 - usedvecs += this_vecs; 345 345 + curvec += nr_masks; 346 346 + usedvecs += nr_masks; 346 347 } 347 348 348 349 /* Fill out vectors at the end that don't need affinity */

+3 -3

fs/fuse/virtio_fs.c

reviewed

··· 862 862 static void virtio_fs_map_queues(struct virtio_device *vdev, struct virtio_fs *fs) 863 863 { 864 864 const struct cpumask *mask, *masks; 865 865 - unsigned int q, cpu; 865 865 + unsigned int q, cpu, nr_masks; 866 866 867 867 /* First attempt to map using existing transport layer affinities 868 868 * e.g. PCIe MSI-X ··· 882 882 return; 883 883 fallback: 884 884 /* Attempt to map evenly in groups over the CPUs */ 885 885 - masks = group_cpus_evenly(fs->num_request_queues); 885 885 + masks = group_cpus_evenly(fs->num_request_queues, &nr_masks); 886 886 /* If even this fails we default to all CPUs use first request queue */ 887 887 if (!masks) { 888 888 for_each_possible_cpu(cpu) ··· 891 891 } 892 892 893 893 for (q = 0; q < fs->num_request_queues; q++) { 894 894 - for_each_cpu(cpu, &masks[q]) 894 894 + for_each_cpu(cpu, &masks[q % nr_masks]) 895 895 fs->mq_map[cpu] = q + VQ_REQUEST; 896 896 } 897 897 kfree(masks);

-5

fs/xfs/xfs_mount.c

reviewed

··· 673 673 return rounddown_pow_of_two(XFS_B_TO_FSB(mp, MAX_RW_COUNT)); 674 674 } 675 675 676 676 - static inline unsigned int max_pow_of_two_factor(const unsigned int nr) 677 677 - { 678 678 - return 1 << (ffs(nr) - 1); 679 679 - } 680 680 - 681 676 /* 682 677 * If the underlying device advertises atomic write support, limit the size of 683 678 * atomic writes to the greatest power-of-two factor of the group size so

+63

include/linux/blk-mq-dma.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 2 + #ifndef BLK_MQ_DMA_H 3 3 + #define BLK_MQ_DMA_H 4 4 + 5 5 + #include <linux/blk-mq.h> 6 6 + #include <linux/pci-p2pdma.h> 7 7 + 8 8 + struct blk_dma_iter { 9 9 + /* Output address range for this iteration */ 10 10 + dma_addr_t addr; 11 11 + u32 len; 12 12 + 13 13 + /* Status code. Only valid when blk_rq_dma_map_iter_* returned false */ 14 14 + blk_status_t status; 15 15 + 16 16 + /* Internal to blk_rq_dma_map_iter_* */ 17 17 + struct req_iterator iter; 18 18 + struct pci_p2pdma_map_state p2pdma; 19 19 + }; 20 20 + 21 21 + bool blk_rq_dma_map_iter_start(struct request *req, struct device *dma_dev, 22 22 + struct dma_iova_state *state, struct blk_dma_iter *iter); 23 23 + bool blk_rq_dma_map_iter_next(struct request *req, struct device *dma_dev, 24 24 + struct dma_iova_state *state, struct blk_dma_iter *iter); 25 25 + 26 26 + /** 27 27 + * blk_rq_dma_map_coalesce - were all segments coalesced? 28 28 + * @state: DMA state to check 29 29 + * 30 30 + * Returns true if blk_rq_dma_map_iter_start coalesced all segments into a 31 31 + * single DMA range. 32 32 + */ 33 33 + static inline bool blk_rq_dma_map_coalesce(struct dma_iova_state *state) 34 34 + { 35 35 + return dma_use_iova(state); 36 36 + } 37 37 + 38 38 + /** 39 39 + * blk_rq_dma_unmap - try to DMA unmap a request 40 40 + * @req: request to unmap 41 41 + * @dma_dev: device to unmap from 42 42 + * @state: DMA IOVA state 43 43 + * @mapped_len: number of bytes to unmap 44 44 + * 45 45 + * Returns %false if the callers need to manually unmap every DMA segment 46 46 + * mapped using @iter or %true if no work is left to be done. 47 47 + */ 48 48 + static inline bool blk_rq_dma_unmap(struct request *req, struct device *dma_dev, 49 49 + struct dma_iova_state *state, size_t mapped_len) 50 50 + { 51 51 + if (req->cmd_flags & REQ_P2PDMA) 52 52 + return true; 53 53 + 54 54 + if (dma_use_iova(state)) { 55 55 + dma_iova_destroy(dma_dev, state, mapped_len, rq_dma_dir(req), 56 56 + 0); 57 57 + return true; 58 58 + } 59 59 + 60 60 + return !dma_need_unmap(dma_dev); 61 61 + } 62 62 + 63 63 + #endif /* BLK_MQ_DMA_H */

+2

include/linux/blk-mq.h

reviewed

··· 947 947 void blk_mq_unfreeze_queue_non_owner(struct request_queue *q); 948 948 void blk_freeze_queue_start_non_owner(struct request_queue *q); 949 949 950 950 + unsigned int blk_mq_num_possible_queues(unsigned int max_queues); 951 951 + unsigned int blk_mq_num_online_queues(unsigned int max_queues); 950 952 void blk_mq_map_queues(struct blk_mq_queue_map *qmap); 951 953 void blk_mq_map_hw_queues(struct blk_mq_queue_map *qmap, 952 954 struct device *dev, unsigned int offset);

+5 -3

include/linux/blk_types.h

reviewed

··· 350 350 /* Close a zone */ 351 351 REQ_OP_ZONE_CLOSE = (__force blk_opf_t)11, 352 352 /* Transition a zone to full */ 353 353 - REQ_OP_ZONE_FINISH = (__force blk_opf_t)12, 353 353 + REQ_OP_ZONE_FINISH = (__force blk_opf_t)13, 354 354 /* reset a zone write pointer */ 355 355 - REQ_OP_ZONE_RESET = (__force blk_opf_t)13, 355 355 + REQ_OP_ZONE_RESET = (__force blk_opf_t)15, 356 356 /* reset all the zone present on the device */ 357 357 - REQ_OP_ZONE_RESET_ALL = (__force blk_opf_t)15, 357 357 + REQ_OP_ZONE_RESET_ALL = (__force blk_opf_t)17, 358 358 359 359 /* Driver private requests */ 360 360 REQ_OP_DRV_IN = (__force blk_opf_t)34, ··· 386 386 __REQ_DRV, /* for driver use */ 387 387 __REQ_FS_PRIVATE, /* for file system (submitter) use */ 388 388 __REQ_ATOMIC, /* for atomic write operations */ 389 389 + __REQ_P2PDMA, /* contains P2P DMA pages */ 389 390 /* 390 391 * Command specific flags, keep last: 391 392 */ ··· 419 418 #define REQ_DRV (__force blk_opf_t)(1ULL << __REQ_DRV) 420 419 #define REQ_FS_PRIVATE (__force blk_opf_t)(1ULL << __REQ_FS_PRIVATE) 421 420 #define REQ_ATOMIC (__force blk_opf_t)(1ULL << __REQ_ATOMIC) 421 421 + #define REQ_P2PDMA (__force blk_opf_t)(1ULL << __REQ_P2PDMA) 422 422 423 423 #define REQ_NOUNMAP (__force blk_opf_t)(1ULL << __REQ_NOUNMAP) 424 424

+55 -9

include/linux/blkdev.h

reviewed

··· 846 846 { 847 847 return disk->nr_zones; 848 848 } 849 849 + 850 850 + /** 851 851 + * bio_needs_zone_write_plugging - Check if a BIO needs to be handled with zone 852 852 + * write plugging 853 853 + * @bio: The BIO being submitted 854 854 + * 855 855 + * Return true whenever @bio execution needs to be handled through zone 856 856 + * write plugging (using blk_zone_plug_bio()). Return false otherwise. 857 857 + */ 858 858 + static inline bool bio_needs_zone_write_plugging(struct bio *bio) 859 859 + { 860 860 + enum req_op op = bio_op(bio); 861 861 + 862 862 + /* 863 863 + * Only zoned block devices have a zone write plug hash table. But not 864 864 + * all of them have one (e.g. DM devices may not need one). 865 865 + */ 866 866 + if (!bio->bi_bdev->bd_disk->zone_wplugs_hash) 867 867 + return false; 868 868 + 869 869 + /* Only write operations need zone write plugging. */ 870 870 + if (!op_is_write(op)) 871 871 + return false; 872 872 + 873 873 + /* Ignore empty flush */ 874 874 + if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) 875 875 + return false; 876 876 + 877 877 + /* Ignore BIOs that already have been handled by zone write plugging. */ 878 878 + if (bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING)) 879 879 + return false; 880 880 + 881 881 + /* 882 882 + * All zone write operations must be handled through zone write plugging 883 883 + * using blk_zone_plug_bio(). 884 884 + */ 885 885 + switch (op) { 886 886 + case REQ_OP_ZONE_APPEND: 887 887 + case REQ_OP_WRITE: 888 888 + case REQ_OP_WRITE_ZEROES: 889 889 + case REQ_OP_ZONE_FINISH: 890 890 + case REQ_OP_ZONE_RESET: 891 891 + case REQ_OP_ZONE_RESET_ALL: 892 892 + return true; 893 893 + default: 894 894 + return false; 895 895 + } 896 896 + } 897 897 + 849 898 bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs); 850 899 851 900 /** ··· 924 875 { 925 876 return 0; 926 877 } 878 878 + 879 879 + static inline bool bio_needs_zone_write_plugging(struct bio *bio) 880 880 + { 881 881 + return false; 882 882 + } 883 883 + 927 884 static inline bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs) 928 885 { 929 886 return false; ··· 1284 1229 BLK_MAX_SEGMENT_SIZE = 65536, 1285 1230 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, 1286 1231 }; 1287 1287 - 1288 1288 - /* 1289 1289 - * Default upper limit for the software max_sectors limit used for 1290 1290 - * regular file system I/O. This can be increased through sysfs. 1291 1291 - * 1292 1292 - * Not to be confused with the max_hw_sector limit that is entirely 1293 1293 - * controlled by the driver, usually based on hardware limits. 1294 1294 - */ 1295 1295 - #define BLK_DEF_MAX_SECTORS_CAP 2560u 1296 1232 1297 1233 static inline struct queue_limits *bdev_limits(struct block_device *bdev) 1298 1234 {

-1

include/linux/cdrom.h

reviewed

··· 62 62 __u8 last_sense; 63 63 __u8 media_written; /* dirty flag, DVD+RW bookkeeping */ 64 64 unsigned short mmc3_profile; /* current MMC3 profile */ 65 65 - int (*exit)(struct cdrom_device_info *); 66 65 int mrw_mode_page; 67 66 bool opened_for_data; 68 67 __s64 last_media_change_ms;

+1 -1

include/linux/group_cpus.h

reviewed

··· 9 9 #include <linux/kernel.h> 10 10 #include <linux/cpu.h> 11 11 12 12 - struct cpumask *group_cpus_evenly(unsigned int numgrps); 12 12 + struct cpumask *group_cpus_evenly(unsigned int numgrps, unsigned int *nummasks); 13 13 14 14 #endif

+14

include/linux/log2.h

reviewed

··· 255 255 ) : \ 256 256 __bits_per(n) \ 257 257 ) 258 258 + 259 259 + /** 260 260 + * max_pow_of_two_factor - return highest power-of-2 factor 261 261 + * @n: parameter 262 262 + * 263 263 + * find highest power-of-2 which is evenly divisible into n. 264 264 + * 0 is returned for n == 0 or 1. 265 265 + */ 266 266 + static inline __attribute__((const)) 267 267 + unsigned int max_pow_of_two_factor(unsigned int n) 268 268 + { 269 269 + return n & -n; 270 270 + } 271 271 + 258 272 #endif /* _LINUX_LOG2_H */

+1 -1

include/linux/nvme.h

reviewed

··· 2155 2155 NVME_SC_NS_NOT_ATTACHED = 0x11a, 2156 2156 NVME_SC_THIN_PROV_NOT_SUPP = 0x11b, 2157 2157 NVME_SC_CTRL_LIST_INVALID = 0x11c, 2158 2158 - NVME_SC_SELT_TEST_IN_PROGRESS = 0x11d, 2158 2158 + NVME_SC_SELF_TEST_IN_PROGRESS = 0x11d, 2159 2159 NVME_SC_BP_WRITE_PROHIBITED = 0x11e, 2160 2160 NVME_SC_CTRL_ID_INVALID = 0x11f, 2161 2161 NVME_SC_SEC_CTRL_STATE_INVALID = 0x120,

-198

include/linux/pktcdvd.h

reviewed

··· 1 1 - /* 2 2 - * Copyright (C) 2000 Jens Axboe <axboe@suse.de> 3 3 - * Copyright (C) 2001-2004 Peter Osterlund <petero2@telia.com> 4 4 - * 5 5 - * May be copied or modified under the terms of the GNU General Public 6 6 - * License. See linux/COPYING for more information. 7 7 - * 8 8 - * Packet writing layer for ATAPI and SCSI CD-R, CD-RW, DVD-R, and 9 9 - * DVD-RW devices. 10 10 - * 11 11 - */ 12 12 - #ifndef __PKTCDVD_H 13 13 - #define __PKTCDVD_H 14 14 - 15 15 - #include <linux/blkdev.h> 16 16 - #include <linux/completion.h> 17 17 - #include <linux/cdrom.h> 18 18 - #include <linux/kobject.h> 19 19 - #include <linux/sysfs.h> 20 20 - #include <linux/mempool.h> 21 21 - #include <uapi/linux/pktcdvd.h> 22 22 - 23 23 - /* default bio write queue congestion marks */ 24 24 - #define PKT_WRITE_CONGESTION_ON 10000 25 25 - #define PKT_WRITE_CONGESTION_OFF 9000 26 26 - 27 27 - 28 28 - struct packet_settings 29 29 - { 30 30 - __u32 size; /* packet size in (512 byte) sectors */ 31 31 - __u8 fp; /* fixed packets */ 32 32 - __u8 link_loss; /* the rest is specified 33 33 - * as per Mt Fuji */ 34 34 - __u8 write_type; 35 35 - __u8 track_mode; 36 36 - __u8 block_mode; 37 37 - }; 38 38 - 39 39 - /* 40 40 - * Very crude stats for now 41 41 - */ 42 42 - struct packet_stats 43 43 - { 44 44 - unsigned long pkt_started; 45 45 - unsigned long pkt_ended; 46 46 - unsigned long secs_w; 47 47 - unsigned long secs_rg; 48 48 - unsigned long secs_r; 49 49 - }; 50 50 - 51 51 - struct packet_cdrw 52 52 - { 53 53 - struct list_head pkt_free_list; 54 54 - struct list_head pkt_active_list; 55 55 - spinlock_t active_list_lock; /* Serialize access to pkt_active_list */ 56 56 - struct task_struct *thread; 57 57 - atomic_t pending_bios; 58 58 - }; 59 59 - 60 60 - /* 61 61 - * Switch to high speed reading after reading this many kilobytes 62 62 - * with no interspersed writes. 63 63 - */ 64 64 - #define HI_SPEED_SWITCH 512 65 65 - 66 66 - struct packet_iosched 67 67 - { 68 68 - atomic_t attention; /* Set to non-zero when queue processing is needed */ 69 69 - int writing; /* Non-zero when writing, zero when reading */ 70 70 - spinlock_t lock; /* Protecting read/write queue manipulations */ 71 71 - struct bio_list read_queue; 72 72 - struct bio_list write_queue; 73 73 - sector_t last_write; /* The sector where the last write ended */ 74 74 - int successive_reads; 75 75 - }; 76 76 - 77 77 - /* 78 78 - * 32 buffers of 2048 bytes 79 79 - */ 80 80 - #if (PAGE_SIZE % CD_FRAMESIZE) != 0 81 81 - #error "PAGE_SIZE must be a multiple of CD_FRAMESIZE" 82 82 - #endif 83 83 - #define PACKET_MAX_SIZE 128 84 84 - #define FRAMES_PER_PAGE (PAGE_SIZE / CD_FRAMESIZE) 85 85 - #define PACKET_MAX_SECTORS (PACKET_MAX_SIZE * CD_FRAMESIZE >> 9) 86 86 - 87 87 - enum packet_data_state { 88 88 - PACKET_IDLE_STATE, /* Not used at the moment */ 89 89 - PACKET_WAITING_STATE, /* Waiting for more bios to arrive, so */ 90 90 - /* we don't have to do as much */ 91 91 - /* data gathering */ 92 92 - PACKET_READ_WAIT_STATE, /* Waiting for reads to fill in holes */ 93 93 - PACKET_WRITE_WAIT_STATE, /* Waiting for the write to complete */ 94 94 - PACKET_RECOVERY_STATE, /* Recover after read/write errors */ 95 95 - PACKET_FINISHED_STATE, /* After write has finished */ 96 96 - 97 97 - PACKET_NUM_STATES /* Number of possible states */ 98 98 - }; 99 99 - 100 100 - /* 101 101 - * Information needed for writing a single packet 102 102 - */ 103 103 - struct pktcdvd_device; 104 104 - 105 105 - struct packet_data 106 106 - { 107 107 - struct list_head list; 108 108 - 109 109 - spinlock_t lock; /* Lock protecting state transitions and */ 110 110 - /* orig_bios list */ 111 111 - 112 112 - struct bio_list orig_bios; /* Original bios passed to pkt_make_request */ 113 113 - /* that will be handled by this packet */ 114 114 - int write_size; /* Total size of all bios in the orig_bios */ 115 115 - /* list, measured in number of frames */ 116 116 - 117 117 - struct bio *w_bio; /* The bio we will send to the real CD */ 118 118 - /* device once we have all data for the */ 119 119 - /* packet we are going to write */ 120 120 - sector_t sector; /* First sector in this packet */ 121 121 - int frames; /* Number of frames in this packet */ 122 122 - 123 123 - enum packet_data_state state; /* Current state */ 124 124 - atomic_t run_sm; /* Incremented whenever the state */ 125 125 - /* machine needs to be run */ 126 126 - long sleep_time; /* Set this to non-zero to make the state */ 127 127 - /* machine run after this many jiffies. */ 128 128 - 129 129 - atomic_t io_wait; /* Number of pending IO operations */ 130 130 - atomic_t io_errors; /* Number of read/write errors during IO */ 131 131 - 132 132 - struct bio *r_bios[PACKET_MAX_SIZE]; /* bios to use during data gathering */ 133 133 - struct page *pages[PACKET_MAX_SIZE / FRAMES_PER_PAGE]; 134 134 - 135 135 - int cache_valid; /* If non-zero, the data for the zone defined */ 136 136 - /* by the sector variable is completely cached */ 137 137 - /* in the pages[] vector. */ 138 138 - 139 139 - int id; /* ID number for debugging */ 140 140 - struct pktcdvd_device *pd; 141 141 - }; 142 142 - 143 143 - struct pkt_rb_node { 144 144 - struct rb_node rb_node; 145 145 - struct bio *bio; 146 146 - }; 147 147 - 148 148 - struct packet_stacked_data 149 149 - { 150 150 - struct bio *bio; /* Original read request bio */ 151 151 - struct pktcdvd_device *pd; 152 152 - }; 153 153 - #define PSD_POOL_SIZE 64 154 154 - 155 155 - struct pktcdvd_device 156 156 - { 157 157 - struct file *bdev_file; /* dev attached */ 158 158 - /* handle acquired for bdev during pkt_open_dev() */ 159 159 - struct file *f_open_bdev; 160 160 - dev_t pkt_dev; /* our dev */ 161 161 - struct packet_settings settings; 162 162 - struct packet_stats stats; 163 163 - int refcnt; /* Open count */ 164 164 - int write_speed; /* current write speed, kB/s */ 165 165 - int read_speed; /* current read speed, kB/s */ 166 166 - unsigned long offset; /* start offset */ 167 167 - __u8 mode_offset; /* 0 / 8 */ 168 168 - __u8 type; 169 169 - unsigned long flags; 170 170 - __u16 mmc3_profile; 171 171 - __u32 nwa; /* next writable address */ 172 172 - __u32 lra; /* last recorded address */ 173 173 - struct packet_cdrw cdrw; 174 174 - wait_queue_head_t wqueue; 175 175 - 176 176 - spinlock_t lock; /* Serialize access to bio_queue */ 177 177 - struct rb_root bio_queue; /* Work queue of bios we need to handle */ 178 178 - int bio_queue_size; /* Number of nodes in bio_queue */ 179 179 - bool congested; /* Someone is waiting for bio_queue_size 180 180 - * to drop. */ 181 181 - sector_t current_sector; /* Keep track of where the elevator is */ 182 182 - atomic_t scan_queue; /* Set to non-zero when pkt_handle_queue */ 183 183 - /* needs to be run. */ 184 184 - mempool_t rb_pool; /* mempool for pkt_rb_node allocations */ 185 185 - 186 186 - struct packet_iosched iosched; 187 187 - struct gendisk *disk; 188 188 - 189 189 - int write_congestion_off; 190 190 - int write_congestion_on; 191 191 - 192 192 - struct device *dev; /* sysfs pktcdvd[0-7] dev */ 193 193 - 194 194 - struct dentry *dfs_d_root; /* debugfs: devname directory */ 195 195 - struct dentry *dfs_f_info; /* debugfs: info file */ 196 196 - }; 197 197 - 198 198 - #endif /* __PKTCDVD_H */

+90 -1

include/trace/events/block.h

reviewed

··· 11 11 #include <linux/tracepoint.h> 12 12 #include <uapi/linux/ioprio.h> 13 13 14 14 - #define RWBS_LEN 9 14 14 + #define RWBS_LEN 10 15 15 16 16 #define IOPRIO_CLASS_STRINGS \ 17 17 { IOPRIO_CLASS_NONE, "none" }, \ ··· 405 405 ); 406 406 407 407 /** 408 408 + * blk_zone_append_update_request_bio - update bio sector after zone append 409 409 + * @rq: the completed request that sets the bio sector 410 410 + * 411 411 + * Update the bio's bi_sector after a zone append command has been completed. 412 412 + */ 413 413 + DEFINE_EVENT(block_rq, blk_zone_append_update_request_bio, 414 414 + TP_PROTO(struct request *rq), 415 415 + TP_ARGS(rq) 416 416 + ); 417 417 + 418 418 + /** 408 419 * block_plug - keep operations requests in request queue 409 420 * @q: request queue to plug 410 421 * ··· 597 586 __entry->nr_sector, 598 587 MAJOR(__entry->old_dev), MINOR(__entry->old_dev), 599 588 (unsigned long long)__entry->old_sector, __entry->nr_bios) 589 589 + ); 590 590 + 591 591 + /** 592 592 + * blkdev_zone_mgmt - Execute a zone management operation on a range of zones 593 593 + * @bio: The block IO operation sent down to the device 594 594 + * @nr_sectors: The number of sectors affected by this operation 595 595 + * 596 596 + * Execute a zone management operation on a specified range of zones. This 597 597 + * range is encoded in %nr_sectors, which has to be a multiple of the zone 598 598 + * size. 599 599 + */ 600 600 + TRACE_EVENT(blkdev_zone_mgmt, 601 601 + 602 602 + TP_PROTO(struct bio *bio, sector_t nr_sectors), 603 603 + 604 604 + TP_ARGS(bio, nr_sectors), 605 605 + 606 606 + TP_STRUCT__entry( 607 607 + __field( dev_t, dev ) 608 608 + __field( sector_t, sector ) 609 609 + __field( sector_t, nr_sectors ) 610 610 + __array( char, rwbs, RWBS_LEN) 611 611 + ), 612 612 + 613 613 + TP_fast_assign( 614 614 + __entry->dev = bio_dev(bio); 615 615 + __entry->sector = bio->bi_iter.bi_sector; 616 616 + __entry->nr_sectors = bio_sectors(bio); 617 617 + blk_fill_rwbs(__entry->rwbs, bio->bi_opf); 618 618 + ), 619 619 + 620 620 + TP_printk("%d,%d %s %llu + %llu", 621 621 + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->rwbs, 622 622 + (unsigned long long)__entry->sector, 623 623 + __entry->nr_sectors) 624 624 + ); 625 625 + 626 626 + DECLARE_EVENT_CLASS(block_zwplug, 627 627 + 628 628 + TP_PROTO(struct request_queue *q, unsigned int zno, sector_t sector, 629 629 + unsigned int nr_sectors), 630 630 + 631 631 + TP_ARGS(q, zno, sector, nr_sectors), 632 632 + 633 633 + TP_STRUCT__entry( 634 634 + __field( dev_t, dev ) 635 635 + __field( unsigned int, zno ) 636 636 + __field( sector_t, sector ) 637 637 + __field( unsigned int, nr_sectors ) 638 638 + ), 639 639 + 640 640 + TP_fast_assign( 641 641 + __entry->dev = disk_devt(q->disk); 642 642 + __entry->zno = zno; 643 643 + __entry->sector = sector; 644 644 + __entry->nr_sectors = nr_sectors; 645 645 + ), 646 646 + 647 647 + TP_printk("%d,%d zone %u, BIO %llu + %u", 648 648 + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->zno, 649 649 + (unsigned long long)__entry->sector, 650 650 + __entry->nr_sectors) 651 651 + ); 652 652 + 653 653 + DEFINE_EVENT(block_zwplug, disk_zone_wplug_add_bio, 654 654 + 655 655 + TP_PROTO(struct request_queue *q, unsigned int zno, sector_t sector, 656 656 + unsigned int nr_sectors), 657 657 + 658 658 + TP_ARGS(q, zno, sector, nr_sectors) 659 659 + ); 660 660 + 661 661 + DEFINE_EVENT(block_zwplug, blk_zone_wplug_bio, 662 662 + 663 663 + TP_PROTO(struct request_queue *q, unsigned int zno, sector_t sector, 664 664 + unsigned int nr_sectors), 665 665 + 666 666 + TP_ARGS(q, zno, sector, nr_sectors) 600 667 ); 601 668 602 669 #endif /* _TRACE_BLOCK_H */

+10

include/uapi/linux/ublk_cmd.h

reviewed

··· 301 301 */ 302 302 #define UBLK_F_PER_IO_DAEMON (1ULL << 13) 303 303 304 304 + /* 305 305 + * If this feature is set, UBLK_U_IO_REGISTER_IO_BUF/UBLK_U_IO_UNREGISTER_IO_BUF 306 306 + * can be issued for an I/O on any task. q_id and tag are also ignored in 307 307 + * UBLK_U_IO_UNREGISTER_IO_BUF's ublksrv_io_cmd. 308 308 + * If it is unset, zero-copy buffers can only be registered and unregistered by 309 309 + * the I/O's daemon task. The q_id and tag of the registered buffer are required 310 310 + * in UBLK_U_IO_UNREGISTER_IO_BUF's ublksrv_io_cmd. 311 311 + */ 312 312 + #define UBLK_F_BUF_REG_OFF_DAEMON (1ULL << 14) 313 313 + 304 314 /* device state */ 305 315 #define UBLK_S_DEV_DEAD 0 306 316 #define UBLK_S_DEV_LIVE 1

+5 -6

kernel/irq/affinity.c

reviewed

··· 69 69 * have multiple sets, build each sets affinity mask separately. 70 70 */ 71 71 for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) { 72 72 - unsigned int this_vecs = affd->set_size[i]; 73 73 - int j; 74 74 - struct cpumask *result = group_cpus_evenly(this_vecs); 72 72 + unsigned int nr_masks, this_vecs = affd->set_size[i]; 73 73 + struct cpumask *result = group_cpus_evenly(this_vecs, &nr_masks); 75 74 76 75 if (!result) { 77 76 kfree(masks); 78 77 return NULL; 79 78 } 80 79 81 81 - for (j = 0; j < this_vecs; j++) 80 80 + for (int j = 0; j < nr_masks; j++) 82 81 cpumask_copy(&masks[curvec + j].mask, &result[j]); 83 82 kfree(result); 84 83 85 85 - curvec += this_vecs; 86 86 - usedvecs += this_vecs; 84 84 + curvec += nr_masks; 85 85 + usedvecs += nr_masks; 87 86 } 88 87 89 88 /* Fill out vectors at the end that don't need affinity */

+25

kernel/trace/blktrace.c

reviewed

··· 1875 1875 case REQ_OP_READ: 1876 1876 rwbs[i++] = 'R'; 1877 1877 break; 1878 1878 + case REQ_OP_ZONE_APPEND: 1879 1879 + rwbs[i++] = 'Z'; 1880 1880 + rwbs[i++] = 'A'; 1881 1881 + break; 1882 1882 + case REQ_OP_ZONE_RESET: 1883 1883 + case REQ_OP_ZONE_RESET_ALL: 1884 1884 + rwbs[i++] = 'Z'; 1885 1885 + rwbs[i++] = 'R'; 1886 1886 + if ((opf & REQ_OP_MASK) == REQ_OP_ZONE_RESET_ALL) 1887 1887 + rwbs[i++] = 'A'; 1888 1888 + break; 1889 1889 + case REQ_OP_ZONE_FINISH: 1890 1890 + rwbs[i++] = 'Z'; 1891 1891 + rwbs[i++] = 'F'; 1892 1892 + break; 1893 1893 + case REQ_OP_ZONE_OPEN: 1894 1894 + rwbs[i++] = 'Z'; 1895 1895 + rwbs[i++] = 'O'; 1896 1896 + break; 1897 1897 + case REQ_OP_ZONE_CLOSE: 1898 1898 + rwbs[i++] = 'Z'; 1899 1899 + rwbs[i++] = 'C'; 1900 1900 + break; 1878 1901 default: 1879 1902 rwbs[i++] = 'N'; 1880 1903 } ··· 1912 1889 rwbs[i++] = 'M'; 1913 1890 if (opf & REQ_ATOMIC) 1914 1891 rwbs[i++] = 'U'; 1892 1892 + 1893 1893 + WARN_ON_ONCE(i >= RWBS_LEN); 1915 1894 1916 1895 rwbs[i] = '\0'; 1917 1896 }

+8 -8

lib/group_cpus.c

reviewed

··· 332 332 /** 333 333 * group_cpus_evenly - Group all CPUs evenly per NUMA/CPU locality 334 334 * @numgrps: number of groups 335 335 + * @nummasks: number of initialized cpumasks 335 336 * 336 337 * Return: cpumask array if successful, NULL otherwise. And each element 337 337 - * includes CPUs assigned to this group 338 338 + * includes CPUs assigned to this group. nummasks contains the number 339 339 + * of initialized masks which can be less than numgrps. 338 340 * 339 341 * Try to put close CPUs from viewpoint of CPU and NUMA locality into 340 342 * same group, and run two-stage grouping: ··· 346 344 * We guarantee in the resulted grouping that all CPUs are covered, and 347 345 * no same CPU is assigned to multiple groups 348 346 */ 349 349 - struct cpumask *group_cpus_evenly(unsigned int numgrps) 347 347 + struct cpumask *group_cpus_evenly(unsigned int numgrps, unsigned int *nummasks) 350 348 { 351 349 unsigned int curgrp = 0, nr_present = 0, nr_others = 0; 352 350 cpumask_var_t *node_to_cpumask; ··· 391 389 ret = __group_cpus_evenly(curgrp, numgrps, node_to_cpumask, 392 390 npresmsk, nmsk, masks); 393 391 if (ret < 0) 394 394 - goto fail_build_affinity; 392 392 + goto fail_node_to_cpumask; 395 393 nr_present = ret; 396 394 397 395 /* ··· 410 408 if (ret >= 0) 411 409 nr_others = ret; 412 410 413 413 - fail_build_affinity: 414 414 - if (ret >= 0) 415 415 - WARN_ON(nr_present + nr_others < numgrps); 416 416 - 417 411 fail_node_to_cpumask: 418 412 free_node_to_cpumask(node_to_cpumask); 419 413 ··· 422 424 kfree(masks); 423 425 return NULL; 424 426 } 427 427 + *nummasks = min(nr_present + nr_others, numgrps); 425 428 return masks; 426 429 } 427 430 #else /* CONFIG_SMP */ 428 428 - struct cpumask *group_cpus_evenly(unsigned int numgrps) 431 431 + struct cpumask *group_cpus_evenly(unsigned int numgrps, unsigned int *nummasks) 429 432 { 430 433 struct cpumask *masks; 431 434 ··· 439 440 440 441 /* assign all CPUs(cpu 0) to the 1st group only */ 441 442 cpumask_copy(&masks[0], cpu_possible_mask); 443 443 + *nummasks = 1; 442 444 return masks; 443 445 } 444 446 #endif /* CONFIG_SMP */

+9 -6

tools/testing/selftests/ublk/fault_inject.c

reviewed

··· 38 38 return 0; 39 39 } 40 40 41 41 - static int ublk_fault_inject_queue_io(struct ublk_queue *q, int tag) 41 41 + static int ublk_fault_inject_queue_io(struct ublk_thread *t, 42 42 + struct ublk_queue *q, int tag) 42 43 { 43 44 const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag); 44 45 struct io_uring_sqe *sqe; ··· 47 46 .tv_nsec = (long long)q->dev->private_data, 48 47 }; 49 48 50 50 - ublk_io_alloc_sqes(ublk_get_io(q, tag), &sqe, 1); 49 49 + ublk_io_alloc_sqes(t, &sqe, 1); 51 50 io_uring_prep_timeout(sqe, &ts, 1, 0); 52 51 sqe->user_data = build_user_data(tag, ublksrv_get_op(iod), 0, q->q_id, 1); 53 52 54 54 - ublk_queued_tgt_io(q, tag, 1); 53 53 + ublk_queued_tgt_io(t, q, tag, 1); 55 54 56 55 return 0; 57 56 } 58 57 59 59 - static void ublk_fault_inject_tgt_io_done(struct ublk_queue *q, int tag, 58 58 + static void ublk_fault_inject_tgt_io_done(struct ublk_thread *t, 59 59 + struct ublk_queue *q, 60 60 const struct io_uring_cqe *cqe) 61 61 { 62 62 + unsigned tag = user_data_to_tag(cqe->user_data); 62 63 const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag); 63 64 64 65 if (cqe->res != -ETIME) 65 66 ublk_err("%s: unexpected cqe res %d\n", __func__, cqe->res); 66 67 67 67 - if (ublk_completed_tgt_io(q, tag)) 68 68 - ublk_complete_io(q, tag, iod->nr_sectors << 9); 68 68 + if (ublk_completed_tgt_io(t, q, tag)) 69 69 + ublk_complete_io(t, q, tag, iod->nr_sectors << 9); 69 70 else 70 71 ublk_err("%s: io not complete after 1 cqe\n", __func__); 71 72 }

+18 -14

tools/testing/selftests/ublk/file_backed.c

reviewed

··· 13 13 assert(0); 14 14 } 15 15 16 16 - static int loop_queue_flush_io(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag) 16 16 + static int loop_queue_flush_io(struct ublk_thread *t, struct ublk_queue *q, 17 17 + const struct ublksrv_io_desc *iod, int tag) 17 18 { 18 19 unsigned ublk_op = ublksrv_get_op(iod); 19 20 struct io_uring_sqe *sqe[1]; 20 21 21 21 - ublk_io_alloc_sqes(ublk_get_io(q, tag), sqe, 1); 22 22 + ublk_io_alloc_sqes(t, sqe, 1); 22 23 io_uring_prep_fsync(sqe[0], 1 /*fds[1]*/, IORING_FSYNC_DATASYNC); 23 24 io_uring_sqe_set_flags(sqe[0], IOSQE_FIXED_FILE); 24 25 /* bit63 marks us as tgt io */ ··· 27 26 return 1; 28 27 } 29 28 30 30 - static int loop_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag) 29 29 + static int loop_queue_tgt_rw_io(struct ublk_thread *t, struct ublk_queue *q, 30 30 + const struct ublksrv_io_desc *iod, int tag) 31 31 { 32 32 unsigned ublk_op = ublksrv_get_op(iod); 33 33 unsigned zc = ublk_queue_use_zc(q); ··· 38 36 void *addr = (zc | auto_zc) ? NULL : (void *)iod->addr; 39 37 40 38 if (!zc || auto_zc) { 41 41 - ublk_io_alloc_sqes(ublk_get_io(q, tag), sqe, 1); 39 39 + ublk_io_alloc_sqes(t, sqe, 1); 42 40 if (!sqe[0]) 43 41 return -ENOMEM; 44 42 ··· 54 52 return 1; 55 53 } 56 54 57 57 - ublk_io_alloc_sqes(ublk_get_io(q, tag), sqe, 3); 55 55 + ublk_io_alloc_sqes(t, sqe, 3); 58 56 59 57 io_uring_prep_buf_register(sqe[0], 0, tag, q->q_id, ublk_get_io(q, tag)->buf_index); 60 58 sqe[0]->flags |= IOSQE_CQE_SKIP_SUCCESS | IOSQE_IO_HARDLINK; ··· 74 72 return 2; 75 73 } 76 74 77 77 - static int loop_queue_tgt_io(struct ublk_queue *q, int tag) 75 75 + static int loop_queue_tgt_io(struct ublk_thread *t, struct ublk_queue *q, int tag) 78 76 { 79 77 const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag); 80 78 unsigned ublk_op = ublksrv_get_op(iod); ··· 82 80 83 81 switch (ublk_op) { 84 82 case UBLK_IO_OP_FLUSH: 85 85 - ret = loop_queue_flush_io(q, iod, tag); 83 83 + ret = loop_queue_flush_io(t, q, iod, tag); 86 84 break; 87 85 case UBLK_IO_OP_WRITE_ZEROES: 88 86 case UBLK_IO_OP_DISCARD: ··· 90 88 break; 91 89 case UBLK_IO_OP_READ: 92 90 case UBLK_IO_OP_WRITE: 93 93 - ret = loop_queue_tgt_rw_io(q, iod, tag); 91 91 + ret = loop_queue_tgt_rw_io(t, q, iod, tag); 94 92 break; 95 93 default: 96 94 ret = -EINVAL; ··· 102 100 return ret; 103 101 } 104 102 105 105 - static int ublk_loop_queue_io(struct ublk_queue *q, int tag) 103 103 + static int ublk_loop_queue_io(struct ublk_thread *t, struct ublk_queue *q, 104 104 + int tag) 106 105 { 107 107 - int queued = loop_queue_tgt_io(q, tag); 106 106 + int queued = loop_queue_tgt_io(t, q, tag); 108 107 109 109 - ublk_queued_tgt_io(q, tag, queued); 108 108 + ublk_queued_tgt_io(t, q, tag, queued); 110 109 return 0; 111 110 } 112 111 113 113 - static void ublk_loop_io_done(struct ublk_queue *q, int tag, 112 112 + static void ublk_loop_io_done(struct ublk_thread *t, struct ublk_queue *q, 114 113 const struct io_uring_cqe *cqe) 115 114 { 115 115 + unsigned tag = user_data_to_tag(cqe->user_data); 116 116 unsigned op = user_data_to_op(cqe->user_data); 117 117 struct ublk_io *io = ublk_get_io(q, tag); 118 118 ··· 130 126 if (op == ublk_cmd_op_nr(UBLK_U_IO_REGISTER_IO_BUF)) 131 127 io->tgt_ios += 1; 132 128 133 133 - if (ublk_completed_tgt_io(q, tag)) 134 134 - ublk_complete_io(q, tag, io->result); 129 129 + if (ublk_completed_tgt_io(t, q, tag)) 130 130 + ublk_complete_io(t, q, tag, io->result); 135 131 } 136 132 137 133 static int ublk_loop_tgt_init(const struct dev_ctx *ctx, struct ublk_dev *dev)

+68 -72

tools/testing/selftests/ublk/kublk.c

reviewed

··· 441 441 unsigned long off; 442 442 443 443 q->tgt_ops = dev->tgt.ops; 444 444 - q->state = 0; 444 444 + q->flags = 0; 445 445 q->q_depth = depth; 446 446 - 447 447 - if (dev->dev_info.flags & (UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_AUTO_BUF_REG)) { 448 448 - q->state |= UBLKSRV_NO_BUF; 449 449 - if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY) 450 450 - q->state |= UBLKSRV_ZC; 451 451 - if (dev->dev_info.flags & UBLK_F_AUTO_BUF_REG) 452 452 - q->state |= UBLKSRV_AUTO_BUF_REG; 453 453 - } 454 454 - q->state |= extra_flags; 446 446 + q->flags = dev->dev_info.flags; 447 447 + q->flags |= extra_flags; 455 448 456 449 cmd_buf_size = ublk_queue_cmd_buf_sz(q); 457 450 off = UBLKSRV_CMD_BUF_OFFSET + q->q_id * ublk_queue_max_cmd_buf_sz(); ··· 459 466 io_buf_size = dev->dev_info.max_io_buf_bytes; 460 467 for (i = 0; i < q->q_depth; i++) { 461 468 q->ios[i].buf_addr = NULL; 462 462 - q->ios[i].flags = UBLKSRV_NEED_FETCH_RQ | UBLKSRV_IO_FREE; 469 469 + q->ios[i].flags = UBLKS_IO_NEED_FETCH_RQ | UBLKS_IO_FREE; 463 470 q->ios[i].tag = i; 464 471 465 465 - if (q->state & UBLKSRV_NO_BUF) 472 472 + if (ublk_queue_no_buf(q)) 466 473 continue; 467 474 468 475 if (posix_memalign((void **)&q->ios[i].buf_addr, ··· 576 583 else 577 584 buf.index = q->ios[tag].buf_index; 578 585 579 579 - if (q->state & UBLKSRV_AUTO_BUF_REG_FALLBACK) 586 586 + if (ublk_queue_auto_zc_fallback(q)) 580 587 buf.flags = UBLK_AUTO_BUF_REG_FALLBACK; 581 588 582 589 sqe->addr = ublk_auto_buf_reg_to_sqe_addr(&buf); 583 590 } 584 591 585 585 - int ublk_queue_io_cmd(struct ublk_io *io) 592 592 + int ublk_queue_io_cmd(struct ublk_thread *t, struct ublk_io *io) 586 593 { 587 587 - struct ublk_thread *t = io->t; 588 594 struct ublk_queue *q = ublk_io_to_queue(io); 589 595 struct ublksrv_io_cmd *cmd; 590 596 struct io_uring_sqe *sqe[1]; ··· 591 599 __u64 user_data; 592 600 593 601 /* only freed io can be issued */ 594 594 - if (!(io->flags & UBLKSRV_IO_FREE)) 602 602 + if (!(io->flags & UBLKS_IO_FREE)) 595 603 return 0; 596 604 597 605 /* ··· 599 607 * getting data 600 608 */ 601 609 if (!(io->flags & 602 602 - (UBLKSRV_NEED_FETCH_RQ | UBLKSRV_NEED_COMMIT_RQ_COMP | UBLKSRV_NEED_GET_DATA))) 610 610 + (UBLKS_IO_NEED_FETCH_RQ | UBLKS_IO_NEED_COMMIT_RQ_COMP | UBLKS_IO_NEED_GET_DATA))) 603 611 return 0; 604 612 605 605 - if (io->flags & UBLKSRV_NEED_GET_DATA) 613 613 + if (io->flags & UBLKS_IO_NEED_GET_DATA) 606 614 cmd_op = UBLK_U_IO_NEED_GET_DATA; 607 607 - else if (io->flags & UBLKSRV_NEED_COMMIT_RQ_COMP) 615 615 + else if (io->flags & UBLKS_IO_NEED_COMMIT_RQ_COMP) 608 616 cmd_op = UBLK_U_IO_COMMIT_AND_FETCH_REQ; 609 609 - else if (io->flags & UBLKSRV_NEED_FETCH_RQ) 617 617 + else if (io->flags & UBLKS_IO_NEED_FETCH_RQ) 610 618 cmd_op = UBLK_U_IO_FETCH_REQ; 611 619 612 620 if (io_uring_sq_space_left(&t->ring) < 1) 613 621 io_uring_submit(&t->ring); 614 622 615 615 - ublk_io_alloc_sqes(io, sqe, 1); 623 623 + ublk_io_alloc_sqes(t, sqe, 1); 616 624 if (!sqe[0]) { 617 625 ublk_err("%s: run out of sqe. thread %u, tag %d\n", 618 626 __func__, t->idx, io->tag); ··· 632 640 sqe[0]->rw_flags = 0; 633 641 cmd->tag = io->tag; 634 642 cmd->q_id = q->q_id; 635 635 - if (!(q->state & UBLKSRV_NO_BUF)) 643 643 + if (!ublk_queue_no_buf(q)) 636 644 cmd->addr = (__u64) (uintptr_t) io->buf_addr; 637 645 else 638 646 cmd->addr = 0; 639 647 640 640 - if (q->state & UBLKSRV_AUTO_BUF_REG) 648 648 + if (ublk_queue_use_auto_zc(q)) 641 649 ublk_set_auto_buf_reg(q, sqe[0], io->tag); 642 650 643 651 user_data = build_user_data(io->tag, _IOC_NR(cmd_op), 0, q->q_id, 0); ··· 649 657 650 658 ublk_dbg(UBLK_DBG_IO_CMD, "%s: (thread %u qid %d tag %u cmd_op %u) iof %x stopping %d\n", 651 659 __func__, t->idx, q->q_id, io->tag, cmd_op, 652 652 - io->flags, !!(t->state & UBLKSRV_THREAD_STOPPING)); 660 660 + io->flags, !!(t->state & UBLKS_T_STOPPING)); 653 661 return 1; 654 662 } 655 663 ··· 677 685 int tag = i % dinfo->queue_depth; 678 686 q = &t->dev->q[q_id]; 679 687 io = &q->ios[tag]; 680 680 - io->t = t; 681 688 io->buf_index = j++; 682 682 - ublk_queue_io_cmd(io); 689 689 + ublk_queue_io_cmd(t, io); 683 690 } 684 691 } else { 685 692 /* ··· 688 697 struct ublk_queue *q = &t->dev->q[t->idx]; 689 698 for (i = 0; i < q->q_depth; i++) { 690 699 io = &q->ios[i]; 691 691 - io->t = t; 692 700 io->buf_index = i; 693 693 - ublk_queue_io_cmd(io); 701 701 + ublk_queue_io_cmd(t, io); 694 702 } 695 703 } 696 704 } ··· 701 711 702 712 static int ublk_thread_is_done(struct ublk_thread *t) 703 713 { 704 704 - return (t->state & UBLKSRV_THREAD_STOPPING) && ublk_thread_is_idle(t); 714 714 + return (t->state & UBLKS_T_STOPPING) && ublk_thread_is_idle(t); 705 715 } 706 716 707 707 - static inline void ublksrv_handle_tgt_cqe(struct ublk_queue *q, 708 708 - struct io_uring_cqe *cqe) 717 717 + static inline void ublksrv_handle_tgt_cqe(struct ublk_thread *t, 718 718 + struct ublk_queue *q, 719 719 + struct io_uring_cqe *cqe) 709 720 { 710 710 - unsigned tag = user_data_to_tag(cqe->user_data); 711 711 - 712 721 if (cqe->res < 0 && cqe->res != -EAGAIN) 713 722 ublk_err("%s: failed tgt io: res %d qid %u tag %u, cmd_op %u\n", 714 723 __func__, cqe->res, q->q_id, ··· 715 726 user_data_to_op(cqe->user_data)); 716 727 717 728 if (q->tgt_ops->tgt_io_done) 718 718 - q->tgt_ops->tgt_io_done(q, tag, cqe); 729 729 + q->tgt_ops->tgt_io_done(t, q, cqe); 730 730 + } 731 731 + 732 732 + static void ublk_handle_uring_cmd(struct ublk_thread *t, 733 733 + struct ublk_queue *q, 734 734 + const struct io_uring_cqe *cqe) 735 735 + { 736 736 + int fetch = (cqe->res != UBLK_IO_RES_ABORT) && 737 737 + !(t->state & UBLKS_T_STOPPING); 738 738 + unsigned tag = user_data_to_tag(cqe->user_data); 739 739 + struct ublk_io *io = &q->ios[tag]; 740 740 + 741 741 + if (!fetch) { 742 742 + t->state |= UBLKS_T_STOPPING; 743 743 + io->flags &= ~UBLKS_IO_NEED_FETCH_RQ; 744 744 + } 745 745 + 746 746 + if (cqe->res == UBLK_IO_RES_OK) { 747 747 + assert(tag < q->q_depth); 748 748 + if (q->tgt_ops->queue_io) 749 749 + q->tgt_ops->queue_io(t, q, tag); 750 750 + } else if (cqe->res == UBLK_IO_RES_NEED_GET_DATA) { 751 751 + io->flags |= UBLKS_IO_NEED_GET_DATA | UBLKS_IO_FREE; 752 752 + ublk_queue_io_cmd(t, io); 753 753 + } else { 754 754 + /* 755 755 + * COMMIT_REQ will be completed immediately since no fetching 756 756 + * piggyback is required. 757 757 + * 758 758 + * Marking IO_FREE only, then this io won't be issued since 759 759 + * we only issue io with (UBLKS_IO_FREE | UBLKSRV_NEED_*) 760 760 + * 761 761 + * */ 762 762 + io->flags = UBLKS_IO_FREE; 763 763 + } 719 764 } 720 765 721 766 static void ublk_handle_cqe(struct ublk_thread *t, ··· 758 735 struct ublk_dev *dev = t->dev; 759 736 unsigned q_id = user_data_to_q_id(cqe->user_data); 760 737 struct ublk_queue *q = &dev->q[q_id]; 761 761 - unsigned tag = user_data_to_tag(cqe->user_data); 762 738 unsigned cmd_op = user_data_to_op(cqe->user_data); 763 763 - int fetch = (cqe->res != UBLK_IO_RES_ABORT) && 764 764 - !(t->state & UBLKSRV_THREAD_STOPPING); 765 765 - struct ublk_io *io; 766 739 767 740 if (cqe->res < 0 && cqe->res != -ENODEV) 768 741 ublk_err("%s: res %d userdata %llx queue state %x\n", __func__, 769 769 - cqe->res, cqe->user_data, q->state); 742 742 + cqe->res, cqe->user_data, q->flags); 770 743 771 744 ublk_dbg(UBLK_DBG_IO_CMD, "%s: res %d (qid %d tag %u cmd_op %u target %d/%d) stopping %d\n", 772 772 - __func__, cqe->res, q->q_id, tag, cmd_op, 773 773 - is_target_io(cqe->user_data), 745 745 + __func__, cqe->res, q->q_id, user_data_to_tag(cqe->user_data), 746 746 + cmd_op, is_target_io(cqe->user_data), 774 747 user_data_to_tgt_data(cqe->user_data), 775 775 - (t->state & UBLKSRV_THREAD_STOPPING)); 748 748 + (t->state & UBLKS_T_STOPPING)); 776 749 777 750 /* Don't retrieve io in case of target io */ 778 751 if (is_target_io(cqe->user_data)) { 779 779 - ublksrv_handle_tgt_cqe(q, cqe); 752 752 + ublksrv_handle_tgt_cqe(t, q, cqe); 780 753 return; 781 754 } 782 755 783 783 - io = &q->ios[tag]; 784 756 t->cmd_inflight--; 785 757 786 786 - if (!fetch) { 787 787 - t->state |= UBLKSRV_THREAD_STOPPING; 788 788 - io->flags &= ~UBLKSRV_NEED_FETCH_RQ; 789 789 - } 790 790 - 791 791 - if (cqe->res == UBLK_IO_RES_OK) { 792 792 - assert(tag < q->q_depth); 793 793 - if (q->tgt_ops->queue_io) 794 794 - q->tgt_ops->queue_io(q, tag); 795 795 - } else if (cqe->res == UBLK_IO_RES_NEED_GET_DATA) { 796 796 - io->flags |= UBLKSRV_NEED_GET_DATA | UBLKSRV_IO_FREE; 797 797 - ublk_queue_io_cmd(io); 798 798 - } else { 799 799 - /* 800 800 - * COMMIT_REQ will be completed immediately since no fetching 801 801 - * piggyback is required. 802 802 - * 803 803 - * Marking IO_FREE only, then this io won't be issued since 804 804 - * we only issue io with (UBLKSRV_IO_FREE | UBLKSRV_NEED_*) 805 805 - * 806 806 - * */ 807 807 - io->flags = UBLKSRV_IO_FREE; 808 808 - } 758 758 + ublk_handle_uring_cmd(t, q, cqe); 809 759 } 810 760 811 761 static int ublk_reap_events_uring(struct ublk_thread *t) ··· 804 808 t->dev->dev_info.dev_id, 805 809 t->idx, io_uring_sq_ready(&t->ring), 806 810 t->cmd_inflight, 807 807 - (t->state & UBLKSRV_THREAD_STOPPING)); 811 811 + (t->state & UBLKS_T_STOPPING)); 808 812 809 813 if (ublk_thread_is_done(t)) 810 814 return -ENODEV; ··· 813 817 reapped = ublk_reap_events_uring(t); 814 818 815 819 ublk_dbg(UBLK_DBG_THREAD, "submit result %d, reapped %d stop %d idle %d\n", 816 816 - ret, reapped, (t->state & UBLKSRV_THREAD_STOPPING), 817 817 - (t->state & UBLKSRV_THREAD_IDLE)); 820 820 + ret, reapped, (t->state & UBLKS_T_STOPPING), 821 821 + (t->state & UBLKS_T_IDLE)); 818 822 819 823 return reapped; 820 824 } ··· 911 915 { 912 916 const struct ublksrv_ctrl_dev_info *dinfo = &dev->dev_info; 913 917 struct ublk_thread_info *tinfo; 914 914 - unsigned extra_flags = 0; 918 918 + unsigned long long extra_flags = 0; 915 919 cpu_set_t *affinity_buf; 916 920 void *thread_ret; 917 921 sem_t ready; ··· 933 937 return ret; 934 938 935 939 if (ctx->auto_zc_fallback) 936 936 - extra_flags = UBLKSRV_AUTO_BUF_REG_FALLBACK; 940 940 + extra_flags = UBLKS_Q_AUTO_BUF_REG_FALLBACK; 937 941 938 942 for (i = 0; i < dinfo->nr_hw_queues; i++) { 939 943 dev->q[i].dev = dev;

+41 -94

tools/testing/selftests/ublk/kublk.h

reviewed

··· 29 29 #include "ublk_dep.h" 30 30 #include <linux/ublk_cmd.h> 31 31 32 32 - #define __maybe_unused __attribute__((unused)) 33 33 - #define MAX_BACK_FILES 4 34 34 - #ifndef min 35 35 - #define min(a, b) ((a) < (b) ? (a) : (b)) 36 36 - #endif 32 32 + #include "utils.h" 37 33 38 38 - #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) 34 34 + #define MAX_BACK_FILES 4 39 35 40 36 /****************** part 1: libublk ********************/ 41 37 ··· 41 45 #define UBLK_CTRL_RING_DEPTH 32 42 46 #define ERROR_EVTFD_DEVID -2 43 47 44 44 - /* queue idle timeout */ 45 45 - #define UBLKSRV_IO_IDLE_SECS 20 46 46 - 47 48 #define UBLK_IO_MAX_BYTES (1 << 20) 48 49 #define UBLK_MAX_QUEUES_SHIFT 5 49 50 #define UBLK_MAX_QUEUES (1 << UBLK_MAX_QUEUES_SHIFT) 50 51 #define UBLK_MAX_THREADS_SHIFT 5 51 52 #define UBLK_MAX_THREADS (1 << UBLK_MAX_THREADS_SHIFT) 52 53 #define UBLK_QUEUE_DEPTH 1024 53 53 - 54 54 - #define UBLK_DBG_DEV (1U << 0) 55 55 - #define UBLK_DBG_THREAD (1U << 1) 56 56 - #define UBLK_DBG_IO_CMD (1U << 2) 57 57 - #define UBLK_DBG_IO (1U << 3) 58 58 - #define UBLK_DBG_CTRL_CMD (1U << 4) 59 59 - #define UBLK_LOG (1U << 5) 60 54 61 55 struct ublk_dev; 62 56 struct ublk_queue; ··· 107 121 struct ublk_io { 108 122 char *buf_addr; 109 123 110 110 - #define UBLKSRV_NEED_FETCH_RQ (1UL << 0) 111 111 - #define UBLKSRV_NEED_COMMIT_RQ_COMP (1UL << 1) 112 112 - #define UBLKSRV_IO_FREE (1UL << 2) 113 113 - #define UBLKSRV_NEED_GET_DATA (1UL << 3) 114 114 - #define UBLKSRV_NEED_REG_BUF (1UL << 4) 124 124 + #define UBLKS_IO_NEED_FETCH_RQ (1UL << 0) 125 125 + #define UBLKS_IO_NEED_COMMIT_RQ_COMP (1UL << 1) 126 126 + #define UBLKS_IO_FREE (1UL << 2) 127 127 + #define UBLKS_IO_NEED_GET_DATA (1UL << 3) 128 128 + #define UBLKS_IO_NEED_REG_BUF (1UL << 4) 115 129 unsigned short flags; 116 130 unsigned short refs; /* used by target code only */ 117 131 ··· 122 136 unsigned short buf_index; 123 137 unsigned short tgt_ios; 124 138 void *private_data; 125 125 - struct ublk_thread *t; 126 139 }; 127 140 128 141 struct ublk_tgt_ops { ··· 129 144 int (*init_tgt)(const struct dev_ctx *ctx, struct ublk_dev *); 130 145 void (*deinit_tgt)(struct ublk_dev *); 131 146 132 132 - int (*queue_io)(struct ublk_queue *, int tag); 133 133 - void (*tgt_io_done)(struct ublk_queue *, 134 134 - int tag, const struct io_uring_cqe *); 147 147 + int (*queue_io)(struct ublk_thread *, struct ublk_queue *, int tag); 148 148 + void (*tgt_io_done)(struct ublk_thread *, struct ublk_queue *, 149 149 + const struct io_uring_cqe *); 135 150 136 151 /* 137 152 * Target specific command line handling ··· 164 179 const struct ublk_tgt_ops *tgt_ops; 165 180 struct ublksrv_io_desc *io_cmd_buf; 166 181 182 182 + /* borrow one bit of ublk uapi flags, which may never be used */ 183 183 + #define UBLKS_Q_AUTO_BUF_REG_FALLBACK (1ULL << 63) 184 184 + __u64 flags; 167 185 struct ublk_io ios[UBLK_QUEUE_DEPTH]; 168 168 - #define UBLKSRV_NO_BUF (1U << 2) 169 169 - #define UBLKSRV_ZC (1U << 3) 170 170 - #define UBLKSRV_AUTO_BUF_REG (1U << 4) 171 171 - #define UBLKSRV_AUTO_BUF_REG_FALLBACK (1U << 5) 172 172 - unsigned state; 173 186 }; 174 187 175 188 struct ublk_thread { ··· 179 196 pthread_t thread; 180 197 unsigned idx; 181 198 182 182 - #define UBLKSRV_THREAD_STOPPING (1U << 0) 183 183 - #define UBLKSRV_THREAD_IDLE (1U << 1) 199 199 + #define UBLKS_T_STOPPING (1U << 0) 200 200 + #define UBLKS_T_IDLE (1U << 1) 184 201 unsigned state; 185 202 }; 186 203 ··· 200 217 void *private_data; 201 218 }; 202 219 203 203 - #ifndef offsetof 204 204 - #define offsetof(TYPE, MEMBER) ((size_t)&((TYPE *)0)->MEMBER) 205 205 - #endif 206 206 - 207 207 - #ifndef container_of 208 208 - #define container_of(ptr, type, member) ({ \ 209 209 - unsigned long __mptr = (unsigned long)(ptr); \ 210 210 - ((type *)(__mptr - offsetof(type, member))); }) 211 211 - #endif 212 212 - 213 213 - #define round_up(val, rnd) \ 214 214 - (((val) + ((rnd) - 1)) & ~((rnd) - 1)) 215 215 - 216 216 - 217 217 - extern unsigned int ublk_dbg_mask; 218 218 - extern int ublk_queue_io_cmd(struct ublk_io *io); 220 220 + extern int ublk_queue_io_cmd(struct ublk_thread *t, struct ublk_io *io); 219 221 220 222 221 223 static inline int ublk_io_auto_zc_fallback(const struct ublksrv_io_desc *iod) ··· 249 281 return _IOC_NR(op); 250 282 } 251 283 252 252 - static inline void ublk_err(const char *fmt, ...) 253 253 - { 254 254 - va_list ap; 255 255 - 256 256 - va_start(ap, fmt); 257 257 - vfprintf(stderr, fmt, ap); 258 258 - } 259 259 - 260 260 - static inline void ublk_log(const char *fmt, ...) 261 261 - { 262 262 - if (ublk_dbg_mask & UBLK_LOG) { 263 263 - va_list ap; 264 264 - 265 265 - va_start(ap, fmt); 266 266 - vfprintf(stdout, fmt, ap); 267 267 - } 268 268 - } 269 269 - 270 270 - static inline void ublk_dbg(int level, const char *fmt, ...) 271 271 - { 272 272 - if (level & ublk_dbg_mask) { 273 273 - va_list ap; 274 274 - 275 275 - va_start(ap, fmt); 276 276 - vfprintf(stdout, fmt, ap); 277 277 - } 278 278 - } 279 279 - 280 284 static inline struct ublk_queue *ublk_io_to_queue(const struct ublk_io *io) 281 285 { 282 286 return container_of(io, struct ublk_queue, ios[io->tag]); 283 287 } 284 288 285 285 - static inline int ublk_io_alloc_sqes(struct ublk_io *io, 289 289 + static inline int ublk_io_alloc_sqes(struct ublk_thread *t, 286 290 struct io_uring_sqe *sqes[], int nr_sqes) 287 291 { 288 288 - struct io_uring *ring = &io->t->ring; 292 292 + struct io_uring *ring = &t->ring; 289 293 unsigned left = io_uring_sq_space_left(ring); 290 294 int i; 291 295 ··· 320 380 321 381 static inline void ublk_mark_io_done(struct ublk_io *io, int res) 322 382 { 323 323 - io->flags |= (UBLKSRV_NEED_COMMIT_RQ_COMP | UBLKSRV_IO_FREE); 383 383 + io->flags |= (UBLKS_IO_NEED_COMMIT_RQ_COMP | UBLKS_IO_FREE); 324 384 io->result = res; 325 385 } 326 386 ··· 342 402 return &q->ios[tag]; 343 403 } 344 404 345 345 - static inline int ublk_complete_io(struct ublk_queue *q, unsigned tag, int res) 405 405 + static inline int ublk_complete_io(struct ublk_thread *t, struct ublk_queue *q, 406 406 + unsigned tag, int res) 346 407 { 347 408 struct ublk_io *io = &q->ios[tag]; 348 409 349 410 ublk_mark_io_done(io, res); 350 411 351 351 - return ublk_queue_io_cmd(io); 412 412 + return ublk_queue_io_cmd(t, io); 352 413 } 353 414 354 354 - static inline void ublk_queued_tgt_io(struct ublk_queue *q, unsigned tag, int queued) 415 415 + static inline void ublk_queued_tgt_io(struct ublk_thread *t, struct ublk_queue *q, 416 416 + unsigned tag, int queued) 355 417 { 356 418 if (queued < 0) 357 357 - ublk_complete_io(q, tag, queued); 419 419 + ublk_complete_io(t, q, tag, queued); 358 420 else { 359 421 struct ublk_io *io = ublk_get_io(q, tag); 360 422 361 361 - io->t->io_inflight += queued; 423 423 + t->io_inflight += queued; 362 424 io->tgt_ios = queued; 363 425 io->result = 0; 364 426 } 365 427 } 366 428 367 367 - static inline int ublk_completed_tgt_io(struct ublk_queue *q, unsigned tag) 429 429 + static inline int ublk_completed_tgt_io(struct ublk_thread *t, 430 430 + struct ublk_queue *q, unsigned tag) 368 431 { 369 432 struct ublk_io *io = ublk_get_io(q, tag); 370 433 371 371 - io->t->io_inflight--; 434 434 + t->io_inflight--; 372 435 373 436 return --io->tgt_ios == 0; 374 437 } 375 438 376 439 static inline int ublk_queue_use_zc(const struct ublk_queue *q) 377 440 { 378 378 - return q->state & UBLKSRV_ZC; 441 441 + return q->flags & UBLK_F_SUPPORT_ZERO_COPY; 379 442 } 380 443 381 444 static inline int ublk_queue_use_auto_zc(const struct ublk_queue *q) 382 445 { 383 383 - return q->state & UBLKSRV_AUTO_BUF_REG; 446 446 + return q->flags & UBLK_F_AUTO_BUF_REG; 447 447 + } 448 448 + 449 449 + static inline int ublk_queue_auto_zc_fallback(const struct ublk_queue *q) 450 450 + { 451 451 + return q->flags & UBLKS_Q_AUTO_BUF_REG_FALLBACK; 452 452 + } 453 453 + 454 454 + static inline int ublk_queue_no_buf(const struct ublk_queue *q) 455 455 + { 456 456 + return ublk_queue_use_zc(q) || ublk_queue_use_auto_zc(q); 384 457 } 385 458 386 459 extern const struct ublk_tgt_ops null_tgt_ops; ··· 404 451 void backing_file_tgt_deinit(struct ublk_dev *dev); 405 452 int backing_file_tgt_init(struct ublk_dev *dev); 406 453 407 407 - static inline unsigned int ilog2(unsigned int x) 408 408 - { 409 409 - if (x == 0) 410 410 - return 0; 411 411 - return (sizeof(x) * 8 - 1) - __builtin_clz(x); 412 412 - } 413 454 #endif

+18 -14

tools/testing/selftests/ublk/null.c

reviewed

··· 55 55 sqe->user_data = build_user_data(tag, ublk_op, 0, q_id, 1); 56 56 } 57 57 58 58 - static int null_queue_zc_io(struct ublk_queue *q, int tag) 58 58 + static int null_queue_zc_io(struct ublk_thread *t, struct ublk_queue *q, 59 59 + int tag) 59 60 { 60 61 const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag); 61 62 struct io_uring_sqe *sqe[3]; 62 63 63 63 - ublk_io_alloc_sqes(ublk_get_io(q, tag), sqe, 3); 64 64 + ublk_io_alloc_sqes(t, sqe, 3); 64 65 65 66 io_uring_prep_buf_register(sqe[0], 0, tag, q->q_id, ublk_get_io(q, tag)->buf_index); 66 67 sqe[0]->user_data = build_user_data(tag, ··· 78 77 return 2; 79 78 } 80 79 81 81 - static int null_queue_auto_zc_io(struct ublk_queue *q, int tag) 80 80 + static int null_queue_auto_zc_io(struct ublk_thread *t, struct ublk_queue *q, 81 81 + int tag) 82 82 { 83 83 const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag); 84 84 struct io_uring_sqe *sqe[1]; 85 85 86 86 - ublk_io_alloc_sqes(ublk_get_io(q, tag), sqe, 1); 86 86 + ublk_io_alloc_sqes(t, sqe, 1); 87 87 __setup_nop_io(tag, iod, sqe[0], q->q_id); 88 88 return 1; 89 89 } 90 90 91 91 - static void ublk_null_io_done(struct ublk_queue *q, int tag, 92 92 - const struct io_uring_cqe *cqe) 91 91 + static void ublk_null_io_done(struct ublk_thread *t, struct ublk_queue *q, 92 92 + const struct io_uring_cqe *cqe) 93 93 { 94 94 + unsigned tag = user_data_to_tag(cqe->user_data); 94 95 unsigned op = user_data_to_op(cqe->user_data); 95 96 struct ublk_io *io = ublk_get_io(q, tag); 96 97 ··· 108 105 if (op == ublk_cmd_op_nr(UBLK_U_IO_REGISTER_IO_BUF)) 109 106 io->tgt_ios += 1; 110 107 111 111 - if (ublk_completed_tgt_io(q, tag)) 112 112 - ublk_complete_io(q, tag, io->result); 108 108 + if (ublk_completed_tgt_io(t, q, tag)) 109 109 + ublk_complete_io(t, q, tag, io->result); 113 110 } 114 111 115 115 - static int ublk_null_queue_io(struct ublk_queue *q, int tag) 112 112 + static int ublk_null_queue_io(struct ublk_thread *t, struct ublk_queue *q, 113 113 + int tag) 116 114 { 117 115 const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag); 118 116 unsigned auto_zc = ublk_queue_use_auto_zc(q); ··· 121 117 int queued; 122 118 123 119 if (auto_zc && !ublk_io_auto_zc_fallback(iod)) 124 124 - queued = null_queue_auto_zc_io(q, tag); 120 120 + queued = null_queue_auto_zc_io(t, q, tag); 125 121 else if (zc) 126 126 - queued = null_queue_zc_io(q, tag); 122 122 + queued = null_queue_zc_io(t, q, tag); 127 123 else { 128 128 - ublk_complete_io(q, tag, iod->nr_sectors << 9); 124 124 + ublk_complete_io(t, q, tag, iod->nr_sectors << 9); 129 125 return 0; 130 126 } 131 131 - ublk_queued_tgt_io(q, tag, queued); 127 127 + ublk_queued_tgt_io(t, q, tag, queued); 132 128 return 0; 133 129 } 134 130 ··· 138 134 */ 139 135 static unsigned short ublk_null_buf_index(const struct ublk_queue *q, int tag) 140 136 { 141 141 - if (q->state & UBLKSRV_AUTO_BUF_REG_FALLBACK) 137 137 + if (ublk_queue_auto_zc_fallback(q)) 142 138 return (unsigned short)-1; 143 139 return q->ios[tag].buf_index; 144 140 }

+19 -14

tools/testing/selftests/ublk/stripe.c

reviewed

··· 123 123 assert(0); 124 124 } 125 125 126 126 - static int stripe_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag) 126 126 + static int stripe_queue_tgt_rw_io(struct ublk_thread *t, struct ublk_queue *q, 127 127 + const struct ublksrv_io_desc *iod, int tag) 127 128 { 128 129 const struct stripe_conf *conf = get_chunk_shift(q); 129 130 unsigned auto_zc = (ublk_queue_use_auto_zc(q) != 0); ··· 139 138 io->private_data = s; 140 139 calculate_stripe_array(conf, iod, s, base); 141 140 142 142 - ublk_io_alloc_sqes(ublk_get_io(q, tag), sqe, s->nr + extra); 141 141 + ublk_io_alloc_sqes(t, sqe, s->nr + extra); 143 142 144 143 if (zc) { 145 144 io_uring_prep_buf_register(sqe[0], 0, tag, q->q_id, io->buf_index); ··· 177 176 return s->nr + zc; 178 177 } 179 178 180 180 - static int handle_flush(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag) 179 179 + static int handle_flush(struct ublk_thread *t, struct ublk_queue *q, 180 180 + const struct ublksrv_io_desc *iod, int tag) 181 181 { 182 182 const struct stripe_conf *conf = get_chunk_shift(q); 183 183 struct io_uring_sqe *sqe[NR_STRIPE]; 184 184 int i; 185 185 186 186 - ublk_io_alloc_sqes(ublk_get_io(q, tag), sqe, conf->nr_files); 186 186 + ublk_io_alloc_sqes(t, sqe, conf->nr_files); 187 187 for (i = 0; i < conf->nr_files; i++) { 188 188 io_uring_prep_fsync(sqe[i], i + 1, IORING_FSYNC_DATASYNC); 189 189 io_uring_sqe_set_flags(sqe[i], IOSQE_FIXED_FILE); ··· 193 191 return conf->nr_files; 194 192 } 195 193 196 196 - static int stripe_queue_tgt_io(struct ublk_queue *q, int tag) 194 194 + static int stripe_queue_tgt_io(struct ublk_thread *t, struct ublk_queue *q, 195 195 + int tag) 197 196 { 198 197 const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag); 199 198 unsigned ublk_op = ublksrv_get_op(iod); ··· 202 199 203 200 switch (ublk_op) { 204 201 case UBLK_IO_OP_FLUSH: 205 205 - ret = handle_flush(q, iod, tag); 202 202 + ret = handle_flush(t, q, iod, tag); 206 203 break; 207 204 case UBLK_IO_OP_WRITE_ZEROES: 208 205 case UBLK_IO_OP_DISCARD: ··· 210 207 break; 211 208 case UBLK_IO_OP_READ: 212 209 case UBLK_IO_OP_WRITE: 213 213 - ret = stripe_queue_tgt_rw_io(q, iod, tag); 210 210 + ret = stripe_queue_tgt_rw_io(t, q, iod, tag); 214 211 break; 215 212 default: 216 213 ret = -EINVAL; ··· 221 218 return ret; 222 219 } 223 220 224 224 - static int ublk_stripe_queue_io(struct ublk_queue *q, int tag) 221 221 + static int ublk_stripe_queue_io(struct ublk_thread *t, struct ublk_queue *q, 222 222 + int tag) 225 223 { 226 226 - int queued = stripe_queue_tgt_io(q, tag); 224 224 + int queued = stripe_queue_tgt_io(t, q, tag); 227 225 228 228 - ublk_queued_tgt_io(q, tag, queued); 226 226 + ublk_queued_tgt_io(t, q, tag, queued); 229 227 return 0; 230 228 } 231 229 232 232 - static void ublk_stripe_io_done(struct ublk_queue *q, int tag, 233 233 - const struct io_uring_cqe *cqe) 230 230 + static void ublk_stripe_io_done(struct ublk_thread *t, struct ublk_queue *q, 231 231 + const struct io_uring_cqe *cqe) 234 232 { 233 233 + unsigned tag = user_data_to_tag(cqe->user_data); 235 234 const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag); 236 235 unsigned op = user_data_to_op(cqe->user_data); 237 236 struct ublk_io *io = ublk_get_io(q, tag); ··· 262 257 } 263 258 } 264 259 265 265 - if (ublk_completed_tgt_io(q, tag)) { 260 260 + if (ublk_completed_tgt_io(t, q, tag)) { 266 261 int res = io->result; 267 262 268 263 if (!res) 269 264 res = iod->nr_sectors << 9; 270 265 271 271 - ublk_complete_io(q, tag, res); 266 266 + ublk_complete_io(t, q, tag, res); 272 267 273 268 free_stripe_array(io->private_data); 274 269 io->private_data = NULL;

+70

tools/testing/selftests/ublk/utils.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 + #ifndef KUBLK_UTILS_H 3 3 + #define KUBLK_UTILS_H 4 4 + 5 5 + #define __maybe_unused __attribute__((unused)) 6 6 + 7 7 + #ifndef min 8 8 + #define min(a, b) ((a) < (b) ? (a) : (b)) 9 9 + #endif 10 10 + 11 11 + #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) 12 12 + 13 13 + #ifndef offsetof 14 14 + #define offsetof(TYPE, MEMBER) ((size_t)&((TYPE *)0)->MEMBER) 15 15 + #endif 16 16 + 17 17 + #ifndef container_of 18 18 + #define container_of(ptr, type, member) ({ \ 19 19 + unsigned long __mptr = (unsigned long)(ptr); \ 20 20 + ((type *)(__mptr - offsetof(type, member))); }) 21 21 + #endif 22 22 + 23 23 + #define round_up(val, rnd) \ 24 24 + (((val) + ((rnd) - 1)) & ~((rnd) - 1)) 25 25 + 26 26 + static inline unsigned int ilog2(unsigned int x) 27 27 + { 28 28 + if (x == 0) 29 29 + return 0; 30 30 + return (sizeof(x) * 8 - 1) - __builtin_clz(x); 31 31 + } 32 32 + 33 33 + #define UBLK_DBG_DEV (1U << 0) 34 34 + #define UBLK_DBG_THREAD (1U << 1) 35 35 + #define UBLK_DBG_IO_CMD (1U << 2) 36 36 + #define UBLK_DBG_IO (1U << 3) 37 37 + #define UBLK_DBG_CTRL_CMD (1U << 4) 38 38 + #define UBLK_LOG (1U << 5) 39 39 + 40 40 + extern unsigned int ublk_dbg_mask; 41 41 + 42 42 + static inline void ublk_err(const char *fmt, ...) 43 43 + { 44 44 + va_list ap; 45 45 + 46 46 + va_start(ap, fmt); 47 47 + vfprintf(stderr, fmt, ap); 48 48 + } 49 49 + 50 50 + static inline void ublk_log(const char *fmt, ...) 51 51 + { 52 52 + if (ublk_dbg_mask & UBLK_LOG) { 53 53 + va_list ap; 54 54 + 55 55 + va_start(ap, fmt); 56 56 + vfprintf(stdout, fmt, ap); 57 57 + } 58 58 + } 59 59 + 60 60 + static inline void ublk_dbg(int level, const char *fmt, ...) 61 61 + { 62 62 + if (level & ublk_dbg_mask) { 63 63 + va_list ap; 64 64 + 65 65 + va_start(ap, fmt); 66 66 + vfprintf(stdout, fmt, ap); 67 67 + } 68 68 + } 69 69 + 70 70 + #endif