Merge tag 'for-6.8/block-2024-01-08' of git://git.kernel.dk/linux

+1

MAINTAINERS

··· 20079 20079 20080 20080 SOFTWARE RAID (Multiple Disks) SUPPORT 20081 20081 M: Song Liu <song@kernel.org> 20082 + R: Yu Kuai <yukuai3@huawei.com> 20082 20083 L: linux-raid@vger.kernel.org 20083 20084 S: Supported 20084 20085 Q: https://patchwork.kernel.org/project/linux-raid/list/

-1

arch/um/drivers/ubd_kern.c

··· 798 798 ubd_dev->cow.fd = err; 799 799 } 800 800 if (ubd_dev->no_trim == 0) { 801 - ubd_dev->queue->limits.discard_granularity = SECTOR_SIZE; 802 801 blk_queue_max_discard_sectors(ubd_dev->queue, UBD_MAX_REQUEST); 803 802 blk_queue_max_write_zeroes_sectors(ubd_dev->queue, UBD_MAX_REQUEST); 804 803 }

+216 -2

block/bio-integrity.c

··· 69 69 70 70 memset(bip, 0, sizeof(*bip)); 71 71 72 + /* always report as many vecs as asked explicitly, not inline vecs */ 73 + bip->bip_max_vcnt = nr_vecs; 72 74 if (nr_vecs > inline_vecs) { 73 - bip->bip_max_vcnt = nr_vecs; 74 75 bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool, 75 76 &bip->bip_max_vcnt, gfp_mask); 76 77 if (!bip->bip_vec) 77 78 goto err; 78 79 } else { 79 80 bip->bip_vec = bip->bip_inline_vecs; 80 - bip->bip_max_vcnt = inline_vecs; 81 81 } 82 82 83 83 bip->bip_bio = bio; ··· 90 90 return ERR_PTR(-ENOMEM); 91 91 } 92 92 EXPORT_SYMBOL(bio_integrity_alloc); 93 + 94 + static void bio_integrity_unpin_bvec(struct bio_vec *bv, int nr_vecs, 95 + bool dirty) 96 + { 97 + int i; 98 + 99 + for (i = 0; i < nr_vecs; i++) { 100 + if (dirty && !PageCompound(bv[i].bv_page)) 101 + set_page_dirty_lock(bv[i].bv_page); 102 + unpin_user_page(bv[i].bv_page); 103 + } 104 + } 105 + 106 + static void bio_integrity_uncopy_user(struct bio_integrity_payload *bip) 107 + { 108 + unsigned short nr_vecs = bip->bip_max_vcnt - 1; 109 + struct bio_vec *copy = &bip->bip_vec[1]; 110 + size_t bytes = bip->bip_iter.bi_size; 111 + struct iov_iter iter; 112 + int ret; 113 + 114 + iov_iter_bvec(&iter, ITER_DEST, copy, nr_vecs, bytes); 115 + ret = copy_to_iter(bvec_virt(bip->bip_vec), bytes, &iter); 116 + WARN_ON_ONCE(ret != bytes); 117 + 118 + bio_integrity_unpin_bvec(copy, nr_vecs, true); 119 + } 120 + 121 + static void bio_integrity_unmap_user(struct bio_integrity_payload *bip) 122 + { 123 + bool dirty = bio_data_dir(bip->bip_bio) == READ; 124 + 125 + if (bip->bip_flags & BIP_COPY_USER) { 126 + if (dirty) 127 + bio_integrity_uncopy_user(bip); 128 + kfree(bvec_virt(bip->bip_vec)); 129 + return; 130 + } 131 + 132 + bio_integrity_unpin_bvec(bip->bip_vec, bip->bip_max_vcnt, dirty); 133 + } 93 134 94 135 /** 95 136 * bio_integrity_free - Free bio integrity payload ··· 146 105 147 106 if (bip->bip_flags & BIP_BLOCK_INTEGRITY) 148 107 kfree(bvec_virt(bip->bip_vec)); 108 + else if (bip->bip_flags & BIP_INTEGRITY_USER) 109 + bio_integrity_unmap_user(bip); 149 110 150 111 __bio_integrity_free(bs, bip); 151 112 bio->bi_integrity = NULL; ··· 202 159 return len; 203 160 } 204 161 EXPORT_SYMBOL(bio_integrity_add_page); 162 + 163 + static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec, 164 + int nr_vecs, unsigned int len, 165 + unsigned int direction, u32 seed) 166 + { 167 + bool write = direction == ITER_SOURCE; 168 + struct bio_integrity_payload *bip; 169 + struct iov_iter iter; 170 + void *buf; 171 + int ret; 172 + 173 + buf = kmalloc(len, GFP_KERNEL); 174 + if (!buf) 175 + return -ENOMEM; 176 + 177 + if (write) { 178 + iov_iter_bvec(&iter, direction, bvec, nr_vecs, len); 179 + if (!copy_from_iter_full(buf, len, &iter)) { 180 + ret = -EFAULT; 181 + goto free_buf; 182 + } 183 + 184 + bip = bio_integrity_alloc(bio, GFP_KERNEL, 1); 185 + } else { 186 + memset(buf, 0, len); 187 + 188 + /* 189 + * We need to preserve the original bvec and the number of vecs 190 + * in it for completion handling 191 + */ 192 + bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs + 1); 193 + } 194 + 195 + if (IS_ERR(bip)) { 196 + ret = PTR_ERR(bip); 197 + goto free_buf; 198 + } 199 + 200 + if (write) 201 + bio_integrity_unpin_bvec(bvec, nr_vecs, false); 202 + else 203 + memcpy(&bip->bip_vec[1], bvec, nr_vecs * sizeof(*bvec)); 204 + 205 + ret = bio_integrity_add_page(bio, virt_to_page(buf), len, 206 + offset_in_page(buf)); 207 + if (ret != len) { 208 + ret = -ENOMEM; 209 + goto free_bip; 210 + } 211 + 212 + bip->bip_flags |= BIP_INTEGRITY_USER | BIP_COPY_USER; 213 + bip->bip_iter.bi_sector = seed; 214 + return 0; 215 + free_bip: 216 + bio_integrity_free(bio); 217 + free_buf: 218 + kfree(buf); 219 + return ret; 220 + } 221 + 222 + static int bio_integrity_init_user(struct bio *bio, struct bio_vec *bvec, 223 + int nr_vecs, unsigned int len, u32 seed) 224 + { 225 + struct bio_integrity_payload *bip; 226 + 227 + bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs); 228 + if (IS_ERR(bip)) 229 + return PTR_ERR(bip); 230 + 231 + memcpy(bip->bip_vec, bvec, nr_vecs * sizeof(*bvec)); 232 + bip->bip_flags |= BIP_INTEGRITY_USER; 233 + bip->bip_iter.bi_sector = seed; 234 + bip->bip_iter.bi_size = len; 235 + return 0; 236 + } 237 + 238 + static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages, 239 + int nr_vecs, ssize_t bytes, ssize_t offset) 240 + { 241 + unsigned int nr_bvecs = 0; 242 + int i, j; 243 + 244 + for (i = 0; i < nr_vecs; i = j) { 245 + size_t size = min_t(size_t, bytes, PAGE_SIZE - offset); 246 + struct folio *folio = page_folio(pages[i]); 247 + 248 + bytes -= size; 249 + for (j = i + 1; j < nr_vecs; j++) { 250 + size_t next = min_t(size_t, PAGE_SIZE, bytes); 251 + 252 + if (page_folio(pages[j]) != folio || 253 + pages[j] != pages[j - 1] + 1) 254 + break; 255 + unpin_user_page(pages[j]); 256 + size += next; 257 + bytes -= next; 258 + } 259 + 260 + bvec_set_page(&bvec[nr_bvecs], pages[i], size, offset); 261 + offset = 0; 262 + nr_bvecs++; 263 + } 264 + 265 + return nr_bvecs; 266 + } 267 + 268 + int bio_integrity_map_user(struct bio *bio, void __user *ubuf, ssize_t bytes, 269 + u32 seed) 270 + { 271 + struct request_queue *q = bdev_get_queue(bio->bi_bdev); 272 + unsigned int align = q->dma_pad_mask | queue_dma_alignment(q); 273 + struct page *stack_pages[UIO_FASTIOV], **pages = stack_pages; 274 + struct bio_vec stack_vec[UIO_FASTIOV], *bvec = stack_vec; 275 + unsigned int direction, nr_bvecs; 276 + struct iov_iter iter; 277 + int ret, nr_vecs; 278 + size_t offset; 279 + bool copy; 280 + 281 + if (bio_integrity(bio)) 282 + return -EINVAL; 283 + if (bytes >> SECTOR_SHIFT > queue_max_hw_sectors(q)) 284 + return -E2BIG; 285 + 286 + if (bio_data_dir(bio) == READ) 287 + direction = ITER_DEST; 288 + else 289 + direction = ITER_SOURCE; 290 + 291 + iov_iter_ubuf(&iter, direction, ubuf, bytes); 292 + nr_vecs = iov_iter_npages(&iter, BIO_MAX_VECS + 1); 293 + if (nr_vecs > BIO_MAX_VECS) 294 + return -E2BIG; 295 + if (nr_vecs > UIO_FASTIOV) { 296 + bvec = kcalloc(sizeof(*bvec), nr_vecs, GFP_KERNEL); 297 + if (!bvec) 298 + return -ENOMEM; 299 + pages = NULL; 300 + } 301 + 302 + copy = !iov_iter_is_aligned(&iter, align, align); 303 + ret = iov_iter_extract_pages(&iter, &pages, bytes, nr_vecs, 0, &offset); 304 + if (unlikely(ret < 0)) 305 + goto free_bvec; 306 + 307 + nr_bvecs = bvec_from_pages(bvec, pages, nr_vecs, bytes, offset); 308 + if (pages != stack_pages) 309 + kvfree(pages); 310 + if (nr_bvecs > queue_max_integrity_segments(q)) 311 + copy = true; 312 + 313 + if (copy) 314 + ret = bio_integrity_copy_user(bio, bvec, nr_bvecs, bytes, 315 + direction, seed); 316 + else 317 + ret = bio_integrity_init_user(bio, bvec, nr_bvecs, bytes, seed); 318 + if (ret) 319 + goto release_pages; 320 + if (bvec != stack_vec) 321 + kfree(bvec); 322 + 323 + return 0; 324 + 325 + release_pages: 326 + bio_integrity_unpin_bvec(bvec, nr_bvecs, false); 327 + free_bvec: 328 + if (bvec != stack_vec) 329 + kfree(bvec); 330 + return ret; 331 + } 332 + EXPORT_SYMBOL_GPL(bio_integrity_map_user); 205 333 206 334 /** 207 335 * bio_integrity_process - Process integrity metadata for a bio

+29 -24

block/bio.c

··· 944 944 945 945 if ((addr1 | mask) != (addr2 | mask)) 946 946 return false; 947 - if (bv->bv_len + len > queue_max_segment_size(q)) 947 + if (len > queue_max_segment_size(q) - bv->bv_len) 948 948 return false; 949 949 return bvec_try_merge_page(bv, page, len, offset, same_page); 950 950 } ··· 966 966 struct page *page, unsigned int len, unsigned int offset, 967 967 unsigned int max_sectors, bool *same_page) 968 968 { 969 + unsigned int max_size = max_sectors << SECTOR_SHIFT; 970 + 969 971 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) 970 972 return 0; 971 973 972 - if (((bio->bi_iter.bi_size + len) >> SECTOR_SHIFT) > max_sectors) 974 + len = min3(len, max_size, queue_max_segment_size(q)); 975 + if (len > max_size - bio->bi_iter.bi_size) 973 976 return 0; 974 977 975 978 if (bio->bi_vcnt > 0) { ··· 1148 1145 1149 1146 void __bio_release_pages(struct bio *bio, bool mark_dirty) 1150 1147 { 1151 - struct bvec_iter_all iter_all; 1152 - struct bio_vec *bvec; 1148 + struct folio_iter fi; 1153 1149 1154 - bio_for_each_segment_all(bvec, bio, iter_all) { 1155 - if (mark_dirty && !PageCompound(bvec->bv_page)) 1156 - set_page_dirty_lock(bvec->bv_page); 1157 - bio_release_page(bio, bvec->bv_page); 1150 + bio_for_each_folio_all(fi, bio) { 1151 + struct page *page; 1152 + size_t done = 0; 1153 + 1154 + if (mark_dirty) { 1155 + folio_lock(fi.folio); 1156 + folio_mark_dirty(fi.folio); 1157 + folio_unlock(fi.folio); 1158 + } 1159 + page = folio_page(fi.folio, fi.offset / PAGE_SIZE); 1160 + do { 1161 + bio_release_page(bio, page++); 1162 + done += PAGE_SIZE; 1163 + } while (done < fi.length); 1158 1164 } 1159 1165 } 1160 1166 EXPORT_SYMBOL_GPL(__bio_release_pages); ··· 1451 1439 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions 1452 1440 * for performing direct-IO in BIOs. 1453 1441 * 1454 - * The problem is that we cannot run set_page_dirty() from interrupt context 1442 + * The problem is that we cannot run folio_mark_dirty() from interrupt context 1455 1443 * because the required locks are not interrupt-safe. So what we can do is to 1456 1444 * mark the pages dirty _before_ performing IO. And in interrupt context, 1457 1445 * check that the pages are still dirty. If so, fine. If not, redirty them 1458 1446 * in process context. 1459 - * 1460 - * We special-case compound pages here: normally this means reads into hugetlb 1461 - * pages. The logic in here doesn't really work right for compound pages 1462 - * because the VM does not uniformly chase down the head page in all cases. 1463 - * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't 1464 - * handle them at all. So we skip compound pages here at an early stage. 1465 1447 * 1466 1448 * Note that this code is very hard to test under normal circumstances because 1467 1449 * direct-io pins the pages with get_user_pages(). This makes ··· 1472 1466 */ 1473 1467 void bio_set_pages_dirty(struct bio *bio) 1474 1468 { 1475 - struct bio_vec *bvec; 1476 - struct bvec_iter_all iter_all; 1469 + struct folio_iter fi; 1477 1470 1478 - bio_for_each_segment_all(bvec, bio, iter_all) { 1479 - if (!PageCompound(bvec->bv_page)) 1480 - set_page_dirty_lock(bvec->bv_page); 1471 + bio_for_each_folio_all(fi, bio) { 1472 + folio_lock(fi.folio); 1473 + folio_mark_dirty(fi.folio); 1474 + folio_unlock(fi.folio); 1481 1475 } 1482 1476 } 1483 1477 EXPORT_SYMBOL_GPL(bio_set_pages_dirty); ··· 1521 1515 1522 1516 void bio_check_pages_dirty(struct bio *bio) 1523 1517 { 1524 - struct bio_vec *bvec; 1518 + struct folio_iter fi; 1525 1519 unsigned long flags; 1526 - struct bvec_iter_all iter_all; 1527 1520 1528 - bio_for_each_segment_all(bvec, bio, iter_all) { 1529 - if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page)) 1521 + bio_for_each_folio_all(fi, bio) { 1522 + if (!folio_test_dirty(fi.folio)) 1530 1523 goto defer; 1531 1524 } 1532 1525

+5 -2

block/blk-cgroup.c

··· 575 575 static void blkg_destroy_all(struct gendisk *disk) 576 576 { 577 577 struct request_queue *q = disk->queue; 578 - struct blkcg_gq *blkg, *n; 578 + struct blkcg_gq *blkg; 579 579 int count = BLKG_DESTROY_BATCH_SIZE; 580 580 int i; 581 581 582 582 restart: 583 583 spin_lock_irq(&q->queue_lock); 584 - list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { 584 + list_for_each_entry(blkg, &q->blkg_list, q_node) { 585 585 struct blkcg *blkcg = blkg->blkcg; 586 586 587 587 if (hlist_unhashed(&blkg->blkcg_node)) ··· 2063 2063 void bio_associate_blkg(struct bio *bio) 2064 2064 { 2065 2065 struct cgroup_subsys_state *css; 2066 + 2067 + if (blk_op_is_passthrough(bio->bi_opf)) 2068 + return; 2066 2069 2067 2070 rcu_read_lock(); 2068 2071

+2 -1

block/blk-cgroup.h

··· 252 252 if (blkcg == &blkcg_root) 253 253 return q->root_blkg; 254 254 255 - blkg = rcu_dereference(blkcg->blkg_hint); 255 + blkg = rcu_dereference_check(blkcg->blkg_hint, 256 + lockdep_is_held(&q->queue_lock)); 256 257 if (blkg && blkg->q == q) 257 258 return blkg; 258 259

+21 -5

block/blk-core.c

··· 772 772 bio_clear_polled(bio); 773 773 774 774 switch (bio_op(bio)) { 775 + case REQ_OP_READ: 776 + case REQ_OP_WRITE: 777 + break; 778 + case REQ_OP_FLUSH: 779 + /* 780 + * REQ_OP_FLUSH can't be submitted through bios, it is only 781 + * synthetized in struct request by the flush state machine. 782 + */ 783 + goto not_supported; 775 784 case REQ_OP_DISCARD: 776 785 if (!bdev_max_discard_sectors(bdev)) 777 786 goto not_supported; ··· 794 785 if (status != BLK_STS_OK) 795 786 goto end_io; 796 787 break; 788 + case REQ_OP_WRITE_ZEROES: 789 + if (!q->limits.max_write_zeroes_sectors) 790 + goto not_supported; 791 + break; 797 792 case REQ_OP_ZONE_RESET: 798 793 case REQ_OP_ZONE_OPEN: 799 794 case REQ_OP_ZONE_CLOSE: ··· 809 796 if (!bdev_is_zoned(bio->bi_bdev) || !blk_queue_zone_resetall(q)) 810 797 goto not_supported; 811 798 break; 812 - case REQ_OP_WRITE_ZEROES: 813 - if (!q->limits.max_write_zeroes_sectors) 814 - goto not_supported; 815 - break; 799 + case REQ_OP_DRV_IN: 800 + case REQ_OP_DRV_OUT: 801 + /* 802 + * Driver private operations are only used with passthrough 803 + * requests. 804 + */ 805 + fallthrough; 816 806 default: 817 - break; 807 + goto not_supported; 818 808 } 819 809 820 810 if (blk_throtl_bio(bio))

+1 -5

block/blk-merge.c

··· 115 115 116 116 *nsegs = 1; 117 117 118 - /* Zero-sector (unknown) and one-sector granularities are the same. */ 119 118 granularity = max(lim->discard_granularity >> 9, 1U); 120 119 121 120 max_discard_sectors = 122 121 min(lim->max_discard_sectors, bio_allowed_max_sectors(lim)); 123 122 max_discard_sectors -= max_discard_sectors % granularity; 124 - 125 - if (unlikely(!max_discard_sectors)) { 126 - /* XXX: warn */ 123 + if (unlikely(!max_discard_sectors)) 127 124 return NULL; 128 - } 129 125 130 126 if (bio_sectors(bio) <= max_discard_sectors) 131 127 return NULL;

+2 -1

block/blk-mq.c

··· 1248 1248 1249 1249 trace_block_rq_issue(rq); 1250 1250 1251 - if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 1251 + if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags) && 1252 + !blk_rq_is_passthrough(rq)) { 1252 1253 rq->io_start_time_ns = ktime_get_ns(); 1253 1254 rq->stats_sectors = blk_rq_sectors(rq); 1254 1255 rq->rq_flags |= RQF_STATS;

+1 -1

block/blk-rq-qos.h

··· 118 118 119 119 static inline void rq_qos_done(struct request_queue *q, struct request *rq) 120 120 { 121 - if (q->rq_qos) 121 + if (q->rq_qos && !blk_rq_is_passthrough(rq)) 122 122 __rq_qos_done(q->rq_qos, rq); 123 123 } 124 124

+23 -82

block/blk-settings.c

··· 48 48 lim->max_discard_sectors = 0; 49 49 lim->max_hw_discard_sectors = 0; 50 50 lim->max_secure_erase_sectors = 0; 51 - lim->discard_granularity = 0; 51 + lim->discard_granularity = 512; 52 52 lim->discard_alignment = 0; 53 53 lim->discard_misaligned = 0; 54 54 lim->logical_block_size = lim->physical_block_size = lim->io_min = 512; ··· 56 56 lim->alignment_offset = 0; 57 57 lim->io_opt = 0; 58 58 lim->misaligned = 0; 59 - lim->zoned = BLK_ZONED_NONE; 59 + lim->zoned = false; 60 60 lim->zone_write_granularity = 0; 61 61 lim->dma_alignment = 511; 62 62 } ··· 127 127 128 128 if ((max_hw_sectors << 9) < PAGE_SIZE) { 129 129 max_hw_sectors = 1 << (PAGE_SHIFT - 9); 130 - printk(KERN_INFO "%s: set to minimum %d\n", 131 - __func__, max_hw_sectors); 130 + pr_info("%s: set to minimum %u\n", __func__, max_hw_sectors); 132 131 } 133 132 134 133 max_hw_sectors = round_down(max_hw_sectors, ··· 139 140 if (limits->max_user_sectors) 140 141 max_sectors = min(max_sectors, limits->max_user_sectors); 141 142 else 142 - max_sectors = min(max_sectors, BLK_DEF_MAX_SECTORS); 143 + max_sectors = min(max_sectors, BLK_DEF_MAX_SECTORS_CAP); 143 144 144 145 max_sectors = round_down(max_sectors, 145 146 limits->logical_block_size >> SECTOR_SHIFT); ··· 247 248 { 248 249 if (!max_segments) { 249 250 max_segments = 1; 250 - printk(KERN_INFO "%s: set to minimum %d\n", 251 - __func__, max_segments); 251 + pr_info("%s: set to minimum %u\n", __func__, max_segments); 252 252 } 253 253 254 254 q->limits.max_segments = max_segments; ··· 283 285 { 284 286 if (max_size < PAGE_SIZE) { 285 287 max_size = PAGE_SIZE; 286 - printk(KERN_INFO "%s: set to minimum %d\n", 287 - __func__, max_size); 288 + pr_info("%s: set to minimum %u\n", __func__, max_size); 288 289 } 289 290 290 291 /* see blk_queue_virt_boundary() for the explanation */ ··· 308 311 struct queue_limits *limits = &q->limits; 309 312 310 313 limits->logical_block_size = size; 314 + 315 + if (limits->discard_granularity < limits->logical_block_size) 316 + limits->discard_granularity = limits->logical_block_size; 311 317 312 318 if (limits->physical_block_size < size) 313 319 limits->physical_block_size = size; ··· 341 341 342 342 if (q->limits.physical_block_size < q->limits.logical_block_size) 343 343 q->limits.physical_block_size = q->limits.logical_block_size; 344 + 345 + if (q->limits.discard_granularity < q->limits.physical_block_size) 346 + q->limits.discard_granularity = q->limits.physical_block_size; 344 347 345 348 if (q->limits.io_min < q->limits.physical_block_size) 346 349 q->limits.io_min = q->limits.physical_block_size; ··· 743 740 { 744 741 if (mask < PAGE_SIZE - 1) { 745 742 mask = PAGE_SIZE - 1; 746 - printk(KERN_INFO "%s: set to minimum %lx\n", 747 - __func__, mask); 743 + pr_info("%s: set to minimum %lx\n", __func__, mask); 748 744 } 749 745 750 746 q->limits.seg_boundary_mask = mask; ··· 843 841 blk_queue_flag_set(QUEUE_FLAG_FUA, q); 844 842 else 845 843 blk_queue_flag_clear(QUEUE_FLAG_FUA, q); 846 - 847 - wbt_set_write_cache(q, test_bit(QUEUE_FLAG_WC, &q->queue_flags)); 848 844 } 849 845 EXPORT_SYMBOL_GPL(blk_queue_write_cache); 850 846 ··· 884 884 } 885 885 EXPORT_SYMBOL_GPL(blk_queue_can_use_dma_map_merging); 886 886 887 - static bool disk_has_partitions(struct gendisk *disk) 888 - { 889 - unsigned long idx; 890 - struct block_device *part; 891 - bool ret = false; 892 - 893 - rcu_read_lock(); 894 - xa_for_each(&disk->part_tbl, idx, part) { 895 - if (bdev_is_partition(part)) { 896 - ret = true; 897 - break; 898 - } 899 - } 900 - rcu_read_unlock(); 901 - 902 - return ret; 903 - } 904 - 905 887 /** 906 - * disk_set_zoned - configure the zoned model for a disk 907 - * @disk: the gendisk of the queue to configure 908 - * @model: the zoned model to set 909 - * 910 - * Set the zoned model of @disk to @model. 911 - * 912 - * When @model is BLK_ZONED_HM (host managed), this should be called only 913 - * if zoned block device support is enabled (CONFIG_BLK_DEV_ZONED option). 914 - * If @model specifies BLK_ZONED_HA (host aware), the effective model used 915 - * depends on CONFIG_BLK_DEV_ZONED settings and on the existence of partitions 916 - * on the disk. 888 + * disk_set_zoned - inidicate a zoned device 889 + * @disk: gendisk to configure 917 890 */ 918 - void disk_set_zoned(struct gendisk *disk, enum blk_zoned_model model) 891 + void disk_set_zoned(struct gendisk *disk) 919 892 { 920 893 struct request_queue *q = disk->queue; 921 - unsigned int old_model = q->limits.zoned; 922 894 923 - switch (model) { 924 - case BLK_ZONED_HM: 925 - /* 926 - * Host managed devices are supported only if 927 - * CONFIG_BLK_DEV_ZONED is enabled. 928 - */ 929 - WARN_ON_ONCE(!IS_ENABLED(CONFIG_BLK_DEV_ZONED)); 930 - break; 931 - case BLK_ZONED_HA: 932 - /* 933 - * Host aware devices can be treated either as regular block 934 - * devices (similar to drive managed devices) or as zoned block 935 - * devices to take advantage of the zone command set, similarly 936 - * to host managed devices. We try the latter if there are no 937 - * partitions and zoned block device support is enabled, else 938 - * we do nothing special as far as the block layer is concerned. 939 - */ 940 - if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) || 941 - disk_has_partitions(disk)) 942 - model = BLK_ZONED_NONE; 943 - break; 944 - case BLK_ZONED_NONE: 945 - default: 946 - if (WARN_ON_ONCE(model != BLK_ZONED_NONE)) 947 - model = BLK_ZONED_NONE; 948 - break; 949 - } 895 + WARN_ON_ONCE(!IS_ENABLED(CONFIG_BLK_DEV_ZONED)); 950 896 951 - q->limits.zoned = model; 952 - if (model != BLK_ZONED_NONE) { 953 - /* 954 - * Set the zone write granularity to the device logical block 955 - * size by default. The driver can change this value if needed. 956 - */ 957 - blk_queue_zone_write_granularity(q, 958 - queue_logical_block_size(q)); 959 - } else if (old_model != BLK_ZONED_NONE) { 960 - disk_clear_zone_settings(disk); 961 - } 897 + /* 898 + * Set the zone write granularity to the device logical block 899 + * size by default. The driver can change this value if needed. 900 + */ 901 + q->limits.zoned = true; 902 + blk_queue_zone_write_granularity(q, queue_logical_block_size(q)); 962 903 } 963 904 EXPORT_SYMBOL_GPL(disk_set_zoned); 964 905

+3 -8

block/blk-sysfs.c

··· 241 241 if (max_sectors_kb == 0) { 242 242 q->limits.max_user_sectors = 0; 243 243 max_sectors_kb = min(max_hw_sectors_kb, 244 - BLK_DEF_MAX_SECTORS >> 1); 244 + BLK_DEF_MAX_SECTORS_CAP >> 1); 245 245 } else { 246 246 if (max_sectors_kb > max_hw_sectors_kb || 247 247 max_sectors_kb < page_kb) ··· 309 309 310 310 static ssize_t queue_zoned_show(struct request_queue *q, char *page) 311 311 { 312 - switch (blk_queue_zoned_model(q)) { 313 - case BLK_ZONED_HA: 314 - return sprintf(page, "host-aware\n"); 315 - case BLK_ZONED_HM: 312 + if (blk_queue_is_zoned(q)) 316 313 return sprintf(page, "host-managed\n"); 317 - default: 318 - return sprintf(page, "none\n"); 319 - } 314 + return sprintf(page, "none\n"); 320 315 } 321 316 322 317 static ssize_t queue_nr_zones_show(struct request_queue *q, char *page)

+2 -11

block/blk-wbt.c

··· 84 84 u64 sync_issue; 85 85 void *sync_cookie; 86 86 87 - unsigned int wc; 88 - 89 87 unsigned long last_issue; /* last non-throttled issue */ 90 88 unsigned long last_comp; /* last non-throttled comp */ 91 89 unsigned long min_lat_nsec; ··· 205 207 */ 206 208 if (wb_acct & WBT_DISCARD) 207 209 limit = rwb->wb_background; 208 - else if (rwb->wc && !wb_recent_wait(rwb)) 210 + else if (test_bit(QUEUE_FLAG_WC, &rwb->rqos.disk->queue->queue_flags) && 211 + !wb_recent_wait(rwb)) 209 212 limit = 0; 210 213 else 211 214 limit = rwb->wb_normal; ··· 698 699 } 699 700 } 700 701 701 - void wbt_set_write_cache(struct request_queue *q, bool write_cache_on) 702 - { 703 - struct rq_qos *rqos = wbt_rq_qos(q); 704 - if (rqos) 705 - RQWB(rqos)->wc = write_cache_on; 706 - } 707 - 708 702 /* 709 703 * Enable wbt if defaults are configured that way 710 704 */ ··· 910 918 rwb->last_comp = rwb->last_issue = jiffies; 911 919 rwb->win_nsec = RWB_WINDOW_NSEC; 912 920 rwb->enable_state = WBT_STATE_ON_DEFAULT; 913 - rwb->wc = test_bit(QUEUE_FLAG_WC, &q->queue_flags); 914 921 rwb->rq_depth.default_depth = RWB_DEF_DEPTH; 915 922 rwb->min_lat_nsec = wbt_default_latency_nsec(q); 916 923 rwb->rq_depth.queue_depth = blk_queue_depth(q);

-5

block/blk-wbt.h

··· 12 12 void wbt_set_min_lat(struct request_queue *q, u64 val); 13 13 bool wbt_disabled(struct request_queue *); 14 14 15 - void wbt_set_write_cache(struct request_queue *, bool); 16 - 17 15 u64 wbt_default_latency_nsec(struct request_queue *); 18 16 19 17 #else ··· 20 22 { 21 23 } 22 24 static inline void wbt_enable_default(struct gendisk *disk) 23 - { 24 - } 25 - static inline void wbt_set_write_cache(struct request_queue *q, bool wc) 26 25 { 27 26 } 28 27

+1 -20

block/blk-zoned.c

··· 498 498 set_bit(idx, args->conv_zones_bitmap); 499 499 break; 500 500 case BLK_ZONE_TYPE_SEQWRITE_REQ: 501 - case BLK_ZONE_TYPE_SEQWRITE_PREF: 502 501 if (!args->seq_zones_wlock) { 503 502 args->seq_zones_wlock = 504 503 blk_alloc_zone_bitmap(q->node, args->nr_zones); ··· 505 506 return -ENOMEM; 506 507 } 507 508 break; 509 + case BLK_ZONE_TYPE_SEQWRITE_PREF: 508 510 default: 509 511 pr_warn("%s: Invalid zone type 0x%x at sectors %llu\n", 510 512 disk->disk_name, (int)zone->type, zone->start); ··· 615 615 return ret; 616 616 } 617 617 EXPORT_SYMBOL_GPL(blk_revalidate_disk_zones); 618 - 619 - void disk_clear_zone_settings(struct gendisk *disk) 620 - { 621 - struct request_queue *q = disk->queue; 622 - 623 - blk_mq_freeze_queue(q); 624 - 625 - disk_free_zone_bitmaps(disk); 626 - blk_queue_flag_clear(QUEUE_FLAG_ZONE_RESETALL, q); 627 - q->required_elevator_features &= ~ELEVATOR_F_ZBD_SEQ_WRITE; 628 - disk->nr_zones = 0; 629 - disk->max_open_zones = 0; 630 - disk->max_active_zones = 0; 631 - q->limits.chunk_sectors = 0; 632 - q->limits.zone_write_granularity = 0; 633 - q->limits.max_zone_append_sectors = 0; 634 - 635 - blk_mq_unfreeze_queue(q); 636 - }

-2

block/blk.h

··· 395 395 396 396 #ifdef CONFIG_BLK_DEV_ZONED 397 397 void disk_free_zone_bitmaps(struct gendisk *disk); 398 - void disk_clear_zone_settings(struct gendisk *disk); 399 398 int blkdev_report_zones_ioctl(struct block_device *bdev, unsigned int cmd, 400 399 unsigned long arg); 401 400 int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode, 402 401 unsigned int cmd, unsigned long arg); 403 402 #else /* CONFIG_BLK_DEV_ZONED */ 404 403 static inline void disk_free_zone_bitmaps(struct gendisk *disk) {} 405 - static inline void disk_clear_zone_settings(struct gendisk *disk) {} 406 404 static inline int blkdev_report_zones_ioctl(struct block_device *bdev, 407 405 unsigned int cmd, unsigned long arg) 408 406 {

+4 -1

block/genhd.c

··· 432 432 DISK_MAX_PARTS); 433 433 disk->minors = DISK_MAX_PARTS; 434 434 } 435 - if (disk->first_minor + disk->minors > MINORMASK + 1) 435 + if (disk->first_minor > MINORMASK || 436 + disk->minors > MINORMASK + 1 || 437 + disk->first_minor + disk->minors > MINORMASK + 1) 436 438 goto out_exit_elevator; 437 439 } else { 438 440 if (WARN_ON(disk->minors)) ··· 544 542 kobject_put(disk->part0->bd_holder_dir); 545 543 out_del_block_link: 546 544 sysfs_remove_link(block_depr, dev_name(ddev)); 545 + pm_runtime_set_memalloc_noio(ddev, false); 547 546 out_device_del: 548 547 device_del(ddev); 549 548 out_free_ext_minor:

+7 -4

block/ioctl.c

··· 18 18 { 19 19 struct gendisk *disk = bdev->bd_disk; 20 20 struct blkpg_partition p; 21 - long long start, length; 21 + sector_t start, length; 22 22 23 23 if (disk->flags & GENHD_FL_NO_PART) 24 24 return -EINVAL; ··· 35 35 if (op == BLKPG_DEL_PARTITION) 36 36 return bdev_del_partition(disk, p.pno); 37 37 38 + if (p.start < 0 || p.length <= 0 || p.start + p.length < 0) 39 + return -EINVAL; 40 + /* Check that the partition is aligned to the block size */ 41 + if (!IS_ALIGNED(p.start | p.length, bdev_logical_block_size(bdev))) 42 + return -EINVAL; 43 + 38 44 start = p.start >> SECTOR_SHIFT; 39 45 length = p.length >> SECTOR_SHIFT; 40 46 41 47 switch (op) { 42 48 case BLKPG_ADD_PARTITION: 43 - /* check if partition is aligned to blocksize */ 44 - if (p.start & (bdev_logical_block_size(bdev) - 1)) 45 - return -EINVAL; 46 49 return bdev_add_partition(disk, p.pno, start, length); 47 50 case BLKPG_RESIZE_PARTITION: 48 51 return bdev_resize_partition(disk, p.pno, start, length);

+2 -10

block/partitions/core.c

··· 305 305 * Partitions are not supported on zoned block devices that are used as 306 306 * such. 307 307 */ 308 - switch (disk->queue->limits.zoned) { 309 - case BLK_ZONED_HM: 308 + if (bdev_is_zoned(disk->part0)) { 310 309 pr_warn("%s: partitions not supported on host managed zoned block device\n", 311 310 disk->disk_name); 312 311 return ERR_PTR(-ENXIO); 313 - case BLK_ZONED_HA: 314 - pr_info("%s: disabling host aware zoned block device support due to partitions\n", 315 - disk->disk_name); 316 - disk_set_zoned(disk, BLK_ZONED_NONE); 317 - break; 318 - case BLK_ZONED_NONE: 319 - break; 320 312 } 321 313 322 314 if (xa_load(&disk->part_tbl, partno)) ··· 605 613 /* 606 614 * Partitions are not supported on host managed zoned block devices. 607 615 */ 608 - if (disk->queue->limits.zoned == BLK_ZONED_HM) { 616 + if (bdev_is_zoned(disk->part0)) { 609 617 pr_warn("%s: ignoring partition table on host managed zoned block device\n", 610 618 disk->disk_name); 611 619 ret = 0;

+2 -1

drivers/block/aoe/aoeblk.c

··· 383 383 WARN_ON(d->flags & DEVFL_TKILL); 384 384 WARN_ON(d->gd); 385 385 WARN_ON(d->flags & DEVFL_UP); 386 - blk_queue_max_hw_sectors(gd->queue, BLK_DEF_MAX_SECTORS); 386 + /* random number picked from the history block max_sectors cap */ 387 + blk_queue_max_hw_sectors(gd->queue, 2560u); 387 388 blk_queue_io_opt(gd->queue, SZ_2M); 388 389 d->bufpool = mp; 389 390 d->blkq = gd->queue;

+10 -6

drivers/block/drbd/drbd_actlog.c

··· 838 838 } 839 839 840 840 /* clear the bit corresponding to the piece of storage in question: 841 - * size byte of data starting from sector. Only clear a bits of the affected 842 - * one ore more _aligned_ BM_BLOCK_SIZE blocks. 841 + * size byte of data starting from sector. Only clear bits of the affected 842 + * one or more _aligned_ BM_BLOCK_SIZE blocks. 843 843 * 844 844 * called by worker on C_SYNC_TARGET and receiver on SyncSource. 845 845 * ··· 957 957 * @device: DRBD device. 958 958 * @sector: The sector number. 959 959 * 960 - * This functions sleeps on al_wait. Returns 0 on success, -EINTR if interrupted. 960 + * This functions sleeps on al_wait. 961 + * 962 + * Returns: %0 on success, -EINTR if interrupted. 961 963 */ 962 964 int drbd_rs_begin_io(struct drbd_device *device, sector_t sector) 963 965 { ··· 1006 1004 1007 1005 /** 1008 1006 * drbd_try_rs_begin_io() - Gets an extent in the resync LRU cache, does not sleep 1009 - * @device: DRBD device. 1007 + * @peer_device: DRBD device. 1010 1008 * @sector: The sector number. 1011 1009 * 1012 1010 * Gets an extent in the resync LRU cache, sets it to BME_NO_WRITES, then 1013 - * tries to set it to BME_LOCKED. Returns 0 upon success, and -EAGAIN 1011 + * tries to set it to BME_LOCKED. 1012 + * 1013 + * Returns: %0 upon success, and -EAGAIN 1014 1014 * if there is still application IO going on in this area. 1015 1015 */ 1016 1016 int drbd_try_rs_begin_io(struct drbd_peer_device *peer_device, sector_t sector) ··· 1194 1190 * drbd_rs_del_all() - Gracefully remove all extents from the resync LRU 1195 1191 * @device: DRBD device. 1196 1192 * 1197 - * Returns 0 upon success, -EAGAIN if at least one reference count was 1193 + * Returns: %0 upon success, -EAGAIN if at least one reference count was 1198 1194 * not zero. 1199 1195 */ 1200 1196 int drbd_rs_del_all(struct drbd_device *device)

+2 -3

drivers/block/loop.c

··· 1301 1301 loop_set_size(lo, new_size); 1302 1302 } 1303 1303 1304 - loop_config_discard(lo); 1305 - 1306 1304 /* update dio if lo_offset or transfer is changed */ 1307 1305 __loop_update_dio(lo, lo->use_dio); 1308 1306 ··· 2034 2036 } 2035 2037 lo->lo_queue = lo->lo_disk->queue; 2036 2038 2037 - blk_queue_max_hw_sectors(lo->lo_queue, BLK_DEF_MAX_SECTORS); 2039 + /* random number picked from the history block max_sectors cap */ 2040 + blk_queue_max_hw_sectors(lo->lo_queue, 2560u); 2038 2041 2039 2042 /* 2040 2043 * By default, we do buffer IO, so it doesn't make sense to enable

+1 -5

drivers/block/nbd.c

··· 334 334 if (!nbd->pid) 335 335 return 0; 336 336 337 - if (nbd->config->flags & NBD_FLAG_SEND_TRIM) { 338 - nbd->disk->queue->limits.discard_granularity = blksize; 337 + if (nbd->config->flags & NBD_FLAG_SEND_TRIM) 339 338 blk_queue_max_discard_sectors(nbd->disk->queue, UINT_MAX); 340 - } 341 339 blk_queue_logical_block_size(nbd->disk->queue, blksize); 342 340 blk_queue_physical_block_size(nbd->disk->queue, blksize); 343 341 ··· 1355 1357 nbd->config = NULL; 1356 1358 1357 1359 nbd->tag_set.timeout = 0; 1358 - nbd->disk->queue->limits.discard_granularity = 0; 1359 1360 blk_queue_max_discard_sectors(nbd->disk->queue, 0); 1360 1361 1361 1362 mutex_unlock(&nbd->config_lock); ··· 1847 1850 * Tell the block layer that we are not a rotational device 1848 1851 */ 1849 1852 blk_queue_flag_set(QUEUE_FLAG_NONROT, disk->queue); 1850 - disk->queue->limits.discard_granularity = 0; 1851 1853 blk_queue_max_discard_sectors(disk->queue, 0); 1852 1854 blk_queue_max_segment_size(disk->queue, UINT_MAX); 1853 1855 blk_queue_max_segments(disk->queue, USHRT_MAX);

+2 -11

drivers/block/null_blk/main.c

··· 1880 1880 return; 1881 1881 } 1882 1882 1883 - nullb->q->limits.discard_granularity = nullb->dev->blocksize; 1884 1883 blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9); 1885 1884 } 1886 1885 ··· 2185 2186 2186 2187 blk_queue_logical_block_size(nullb->q, dev->blocksize); 2187 2188 blk_queue_physical_block_size(nullb->q, dev->blocksize); 2188 - if (!dev->max_sectors) 2189 - dev->max_sectors = queue_max_hw_sectors(nullb->q); 2190 - dev->max_sectors = min(dev->max_sectors, BLK_DEF_MAX_SECTORS); 2191 - blk_queue_max_hw_sectors(nullb->q, dev->max_sectors); 2189 + if (dev->max_sectors) 2190 + blk_queue_max_hw_sectors(nullb->q, dev->max_sectors); 2192 2191 2193 2192 if (dev->virt_boundary) 2194 2193 blk_queue_virt_boundary(nullb->q, PAGE_SIZE - 1); ··· 2284 2287 pr_warn("invalid block size\n"); 2285 2288 pr_warn("defaults block size to %lu\n", PAGE_SIZE); 2286 2289 g_bs = PAGE_SIZE; 2287 - } 2288 - 2289 - if (g_max_sectors > BLK_DEF_MAX_SECTORS) { 2290 - pr_warn("invalid max sectors\n"); 2291 - pr_warn("defaults max sectors to %u\n", BLK_DEF_MAX_SECTORS); 2292 - g_max_sectors = BLK_DEF_MAX_SECTORS; 2293 2290 } 2294 2291 2295 2292 if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {

+1 -1

drivers/block/null_blk/zoned.c

··· 159 159 struct nullb_device *dev = nullb->dev; 160 160 struct request_queue *q = nullb->q; 161 161 162 - disk_set_zoned(nullb->disk, BLK_ZONED_HM); 162 + disk_set_zoned(nullb->disk); 163 163 blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); 164 164 blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE); 165 165 blk_queue_chunk_sectors(q, dev->zone_size_sects);

+8 -5

drivers/block/rnbd/rnbd-clt.c

··· 1006 1006 msg.prio = cpu_to_le16(req_get_ioprio(rq)); 1007 1007 1008 1008 /* 1009 - * We only support discards with single segment for now. 1009 + * We only support discards/WRITE_ZEROES with single segment for now. 1010 1010 * See queue limits. 1011 1011 */ 1012 - if (req_op(rq) != REQ_OP_DISCARD) 1012 + if ((req_op(rq) != REQ_OP_DISCARD) && (req_op(rq) != REQ_OP_WRITE_ZEROES)) 1013 1013 sg_cnt = blk_rq_map_sg(dev->queue, rq, iu->sgt.sgl); 1014 1014 1015 1015 if (sg_cnt == 0) ··· 1362 1362 blk_queue_write_cache(dev->queue, 1363 1363 !!(rsp->cache_policy & RNBD_WRITEBACK), 1364 1364 !!(rsp->cache_policy & RNBD_FUA)); 1365 + blk_queue_max_write_zeroes_sectors(dev->queue, 1366 + le32_to_cpu(rsp->max_write_zeroes_sectors)); 1365 1367 } 1366 1368 1367 1369 static int rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, ··· 1569 1567 1570 1568 dev = init_dev(sess, access_mode, pathname, nr_poll_queues); 1571 1569 if (IS_ERR(dev)) { 1572 - pr_err("map_device: failed to map device '%s' from session %s, can't initialize device, err: %ld\n", 1573 - pathname, sess->sessname, PTR_ERR(dev)); 1570 + pr_err("map_device: failed to map device '%s' from session %s, can't initialize device, err: %pe\n", 1571 + pathname, sess->sessname, dev); 1574 1572 ret = PTR_ERR(dev); 1575 1573 goto put_sess; 1576 1574 } ··· 1628 1626 } 1629 1627 1630 1628 rnbd_clt_info(dev, 1631 - "map_device: Device mapped as %s (nsectors: %llu, logical_block_size: %d, physical_block_size: %d, max_discard_sectors: %d, discard_granularity: %d, discard_alignment: %d, secure_discard: %d, max_segments: %d, max_hw_sectors: %d, wc: %d, fua: %d)\n", 1629 + "map_device: Device mapped as %s (nsectors: %llu, logical_block_size: %d, physical_block_size: %d, max_write_zeroes_sectors: %d, max_discard_sectors: %d, discard_granularity: %d, discard_alignment: %d, secure_discard: %d, max_segments: %d, max_hw_sectors: %d, wc: %d, fua: %d)\n", 1632 1630 dev->gd->disk_name, le64_to_cpu(rsp->nsectors), 1633 1631 le16_to_cpu(rsp->logical_block_size), 1634 1632 le16_to_cpu(rsp->physical_block_size), 1633 + le32_to_cpu(rsp->max_write_zeroes_sectors), 1635 1634 le32_to_cpu(rsp->max_discard_sectors), 1636 1635 le32_to_cpu(rsp->discard_granularity), 1637 1636 le32_to_cpu(rsp->discard_alignment),

+10 -4

drivers/block/rnbd/rnbd-proto.h

··· 128 128 * @device_id: device_id on server side to identify the device 129 129 * @nsectors: number of sectors in the usual 512b unit 130 130 * @max_hw_sectors: max hardware sectors in the usual 512b unit 131 - * @max_write_same_sectors: max sectors for WRITE SAME in the 512b unit 131 + * @max_write_zeroes_sectors: max sectors for WRITE ZEROES in the 512b unit 132 132 * @max_discard_sectors: max. sectors that can be discarded at once in 512b 133 133 * unit. 134 134 * @discard_granularity: size of the internal discard allocation unit in bytes ··· 145 145 __le32 device_id; 146 146 __le64 nsectors; 147 147 __le32 max_hw_sectors; 148 - __le32 max_write_same_sectors; 148 + __le32 max_write_zeroes_sectors; 149 149 __le32 max_discard_sectors; 150 150 __le32 discard_granularity; 151 151 __le32 discard_alignment; ··· 186 186 * @RNBD_OP_FLUSH: flush the volatile write cache 187 187 * @RNBD_OP_DISCARD: discard sectors 188 188 * @RNBD_OP_SECURE_ERASE: securely erase sectors 189 - * @RNBD_OP_WRITE_SAME: write the same sectors many times 189 + * @RNBD_OP_WRITE_ZEROES: write zeroes sectors 190 190 191 191 * @RNBD_F_SYNC: request is sync (sync write or read) 192 192 * @RNBD_F_FUA: forced unit access ··· 199 199 RNBD_OP_FLUSH = 2, 200 200 RNBD_OP_DISCARD = 3, 201 201 RNBD_OP_SECURE_ERASE = 4, 202 - RNBD_OP_WRITE_SAME = 5, 202 + RNBD_OP_WRITE_ZEROES = 5, 203 203 204 204 /* Flags */ 205 205 RNBD_F_SYNC = 1<<(RNBD_OP_BITS + 0), ··· 236 236 case RNBD_OP_SECURE_ERASE: 237 237 bio_opf = REQ_OP_SECURE_ERASE; 238 238 break; 239 + case RNBD_OP_WRITE_ZEROES: 240 + bio_opf = REQ_OP_WRITE_ZEROES; 241 + break; 239 242 default: 240 243 WARN(1, "Unknown RNBD type: %d (flags %d)\n", 241 244 rnbd_op(rnbd_opf), rnbd_opf); ··· 270 267 break; 271 268 case REQ_OP_SECURE_ERASE: 272 269 rnbd_opf = RNBD_OP_SECURE_ERASE; 270 + break; 271 + case REQ_OP_WRITE_ZEROES: 272 + rnbd_opf = RNBD_OP_WRITE_ZEROES; 273 273 break; 274 274 case REQ_OP_FLUSH: 275 275 rnbd_opf = RNBD_OP_FLUSH;

+23 -21

drivers/block/rnbd/rnbd-srv.c

··· 136 136 137 137 sess_dev = rnbd_get_sess_dev(dev_id, srv_sess); 138 138 if (IS_ERR(sess_dev)) { 139 - pr_err_ratelimited("Got I/O request on session %s for unknown device id %d\n", 140 - srv_sess->sessname, dev_id); 139 + pr_err_ratelimited("Got I/O request on session %s for unknown device id %d: %pe\n", 140 + srv_sess->sessname, dev_id, sess_dev); 141 141 err = -ENOTCONN; 142 142 goto err; 143 143 } ··· 544 544 rsp->max_segments = cpu_to_le16(bdev_max_segments(bdev)); 545 545 rsp->max_hw_sectors = 546 546 cpu_to_le32(queue_max_hw_sectors(bdev_get_queue(bdev))); 547 - rsp->max_write_same_sectors = 0; 547 + rsp->max_write_zeroes_sectors = 548 + cpu_to_le32(bdev_write_zeroes_sectors(bdev)); 548 549 rsp->max_discard_sectors = cpu_to_le32(bdev_max_discard_sectors(bdev)); 549 550 rsp->discard_granularity = cpu_to_le32(bdev_discard_granularity(bdev)); 550 551 rsp->discard_alignment = cpu_to_le32(bdev_discard_alignment(bdev)); ··· 586 585 { 587 586 char *full_path; 588 587 char *a, *b; 588 + int len; 589 589 590 590 full_path = kmalloc(PATH_MAX, GFP_KERNEL); 591 591 if (!full_path) ··· 598 596 */ 599 597 a = strnstr(dev_search_path, "%SESSNAME%", sizeof(dev_search_path)); 600 598 if (a) { 601 - int len = a - dev_search_path; 599 + len = a - dev_search_path; 602 600 603 601 len = snprintf(full_path, PATH_MAX, "%.*s/%s/%s", len, 604 602 dev_search_path, srv_sess->sessname, dev_name); 605 - if (len >= PATH_MAX) { 606 - pr_err("Too long path: %s, %s, %s\n", 607 - dev_search_path, srv_sess->sessname, dev_name); 608 - kfree(full_path); 609 - return ERR_PTR(-EINVAL); 610 - } 611 603 } else { 612 - snprintf(full_path, PATH_MAX, "%s/%s", 613 - dev_search_path, dev_name); 604 + len = snprintf(full_path, PATH_MAX, "%s/%s", 605 + dev_search_path, dev_name); 606 + } 607 + if (len >= PATH_MAX) { 608 + pr_err("Too long path: %s, %s, %s\n", 609 + dev_search_path, srv_sess->sessname, dev_name); 610 + kfree(full_path); 611 + return ERR_PTR(-EINVAL); 614 612 } 615 613 616 614 /* eliminitate duplicated slashes */ ··· 711 709 full_path = rnbd_srv_get_full_path(srv_sess, open_msg->dev_name); 712 710 if (IS_ERR(full_path)) { 713 711 ret = PTR_ERR(full_path); 714 - pr_err("Opening device '%s' for client %s failed, failed to get device full path, err: %d\n", 715 - open_msg->dev_name, srv_sess->sessname, ret); 712 + pr_err("Opening device '%s' for client %s failed, failed to get device full path, err: %pe\n", 713 + open_msg->dev_name, srv_sess->sessname, full_path); 716 714 goto reject; 717 715 } 718 716 719 717 bdev_handle = bdev_open_by_path(full_path, open_flags, NULL, NULL); 720 718 if (IS_ERR(bdev_handle)) { 721 719 ret = PTR_ERR(bdev_handle); 722 - pr_err("Opening device '%s' on session %s failed, failed to open the block device, err: %d\n", 723 - full_path, srv_sess->sessname, ret); 720 + pr_err("Opening device '%s' on session %s failed, failed to open the block device, err: %pe\n", 721 + full_path, srv_sess->sessname, bdev_handle); 724 722 goto free_path; 725 723 } 726 724 727 725 srv_dev = rnbd_srv_get_or_create_srv_dev(bdev_handle->bdev, srv_sess, 728 726 open_msg->access_mode); 729 727 if (IS_ERR(srv_dev)) { 730 - pr_err("Opening device '%s' on session %s failed, creating srv_dev failed, err: %ld\n", 731 - full_path, srv_sess->sessname, PTR_ERR(srv_dev)); 728 + pr_err("Opening device '%s' on session %s failed, creating srv_dev failed, err: %pe\n", 729 + full_path, srv_sess->sessname, srv_dev); 732 730 ret = PTR_ERR(srv_dev); 733 731 goto blkdev_put; 734 732 } ··· 738 736 open_msg->access_mode == RNBD_ACCESS_RO, 739 737 srv_dev); 740 738 if (IS_ERR(srv_sess_dev)) { 741 - pr_err("Opening device '%s' on session %s failed, creating sess_dev failed, err: %ld\n", 742 - full_path, srv_sess->sessname, PTR_ERR(srv_sess_dev)); 739 + pr_err("Opening device '%s' on session %s failed, creating sess_dev failed, err: %pe\n", 740 + full_path, srv_sess->sessname, srv_sess_dev); 743 741 ret = PTR_ERR(srv_sess_dev); 744 742 goto srv_dev_put; 745 743 } ··· 820 818 }; 821 819 rtrs_ctx = rtrs_srv_open(&rtrs_ops, port_nr); 822 820 if (IS_ERR(rtrs_ctx)) { 823 - pr_err("rtrs_srv_open(), err: %d\n", err); 821 + pr_err("rtrs_srv_open(), err: %pe\n", rtrs_ctx); 824 822 return PTR_ERR(rtrs_ctx); 825 823 } 826 824

+1 -1

drivers/block/ublk_drv.c

··· 250 250 { 251 251 const struct ublk_param_zoned *p = &ub->params.zoned; 252 252 253 - disk_set_zoned(ub->ub_disk, BLK_ZONED_HM); 253 + disk_set_zoned(ub->ub_disk); 254 254 blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, ub->ub_disk->queue); 255 255 blk_queue_required_elevator_features(ub->ub_disk->queue, 256 256 ELEVATOR_F_ZBD_SEQ_WRITE);

+23 -55

drivers/block/virtio_blk.c

··· 722 722 return ret; 723 723 } 724 724 725 - static void virtblk_revalidate_zones(struct virtio_blk *vblk) 726 - { 727 - u8 model; 728 - 729 - virtio_cread(vblk->vdev, struct virtio_blk_config, 730 - zoned.model, &model); 731 - switch (model) { 732 - default: 733 - dev_err(&vblk->vdev->dev, "unknown zone model %d\n", model); 734 - fallthrough; 735 - case VIRTIO_BLK_Z_NONE: 736 - case VIRTIO_BLK_Z_HA: 737 - disk_set_zoned(vblk->disk, BLK_ZONED_NONE); 738 - return; 739 - case VIRTIO_BLK_Z_HM: 740 - WARN_ON_ONCE(!vblk->zone_sectors); 741 - if (!blk_revalidate_disk_zones(vblk->disk, NULL)) 742 - set_capacity_and_notify(vblk->disk, 0); 743 - } 744 - } 745 - 746 725 static int virtblk_probe_zoned_device(struct virtio_device *vdev, 747 726 struct virtio_blk *vblk, 748 727 struct request_queue *q) 749 728 { 750 729 u32 v, wg; 751 - u8 model; 752 - 753 - virtio_cread(vdev, struct virtio_blk_config, 754 - zoned.model, &model); 755 - 756 - switch (model) { 757 - case VIRTIO_BLK_Z_NONE: 758 - case VIRTIO_BLK_Z_HA: 759 - /* Present the host-aware device as non-zoned */ 760 - return 0; 761 - case VIRTIO_BLK_Z_HM: 762 - break; 763 - default: 764 - dev_err(&vdev->dev, "unsupported zone model %d\n", model); 765 - return -EINVAL; 766 - } 767 730 768 731 dev_dbg(&vdev->dev, "probing host-managed zoned device\n"); 769 732 770 - disk_set_zoned(vblk->disk, BLK_ZONED_HM); 733 + disk_set_zoned(vblk->disk); 771 734 blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); 772 735 773 736 virtio_cread(vdev, struct virtio_blk_config, ··· 802 839 */ 803 840 #define virtblk_report_zones NULL 804 841 805 - static inline void virtblk_revalidate_zones(struct virtio_blk *vblk) 806 - { 807 - } 808 - 809 842 static inline int virtblk_probe_zoned_device(struct virtio_device *vdev, 810 843 struct virtio_blk *vblk, struct request_queue *q) 811 844 { 812 - u8 model; 813 - 814 - virtio_cread(vdev, struct virtio_blk_config, zoned.model, &model); 815 - if (model == VIRTIO_BLK_Z_HM) { 816 - dev_err(&vdev->dev, 817 - "virtio_blk: zoned devices are not supported"); 818 - return -EOPNOTSUPP; 819 - } 820 - 821 - return 0; 845 + dev_err(&vdev->dev, 846 + "virtio_blk: zoned devices are not supported"); 847 + return -EOPNOTSUPP; 822 848 } 823 849 #endif /* CONFIG_BLK_DEV_ZONED */ 824 850 ··· 957 1005 struct virtio_blk *vblk = 958 1006 container_of(work, struct virtio_blk, config_work); 959 1007 960 - virtblk_revalidate_zones(vblk); 961 1008 virtblk_update_capacity(vblk, true); 962 1009 } 963 1010 ··· 1521 1570 * placed after the virtio_device_ready() call above. 1522 1571 */ 1523 1572 if (virtio_has_feature(vdev, VIRTIO_BLK_F_ZONED)) { 1524 - err = virtblk_probe_zoned_device(vdev, vblk, q); 1525 - if (err) 1573 + u8 model; 1574 + 1575 + virtio_cread(vdev, struct virtio_blk_config, zoned.model, 1576 + &model); 1577 + switch (model) { 1578 + case VIRTIO_BLK_Z_NONE: 1579 + case VIRTIO_BLK_Z_HA: 1580 + /* Present the host-aware device as non-zoned */ 1581 + break; 1582 + case VIRTIO_BLK_Z_HM: 1583 + err = virtblk_probe_zoned_device(vdev, vblk, q); 1584 + if (err) 1585 + goto out_cleanup_disk; 1586 + break; 1587 + default: 1588 + dev_err(&vdev->dev, "unsupported zone model %d\n", 1589 + model); 1590 + err = -EINVAL; 1526 1591 goto out_cleanup_disk; 1592 + } 1527 1593 } 1528 1594 1529 1595 err = device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups);

+1 -1

drivers/block/xen-blkback/common.h

··· 132 132 struct blkif_x86_64_request_rw { 133 133 uint8_t nr_segments; /* number of segments */ 134 134 blkif_vdev_t handle; /* only for read/write requests */ 135 - uint32_t _pad1; /* offsetof(blkif_reqest..,u.rw.id)==8 */ 135 + uint32_t _pad1; /* offsetof(blkif_request..,u.rw.id)==8 */ 136 136 uint64_t id; 137 137 blkif_sector_t sector_number;/* start sector idx on disk (r/w only) */ 138 138 struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];

-1

drivers/block/zram/zram_drv.c

··· 2226 2226 ZRAM_LOGICAL_BLOCK_SIZE); 2227 2227 blk_queue_io_min(zram->disk->queue, PAGE_SIZE); 2228 2228 blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); 2229 - zram->disk->queue->limits.discard_granularity = PAGE_SIZE; 2230 2229 blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX); 2231 2230 2232 2231 /*

-34

drivers/md/Kconfig

··· 61 61 various kernel APIs and can only work with files on a file system not 62 62 actually sitting on the MD device. 63 63 64 - config MD_LINEAR 65 - tristate "Linear (append) mode (deprecated)" 66 - depends on BLK_DEV_MD 67 - help 68 - If you say Y here, then your multiple devices driver will be able to 69 - use the so-called linear mode, i.e. it will combine the hard disk 70 - partitions by simply appending one to the other. 71 - 72 - To compile this as a module, choose M here: the module 73 - will be called linear. 74 - 75 - If unsure, say Y. 76 - 77 64 config MD_RAID0 78 65 tristate "RAID-0 (striping) mode" 79 66 depends on BLK_DEV_MD ··· 158 171 will be called raid456. 159 172 160 173 If unsure, say Y. 161 - 162 - config MD_MULTIPATH 163 - tristate "Multipath I/O support (deprecated)" 164 - depends on BLK_DEV_MD 165 - help 166 - MD_MULTIPATH provides a simple multi-path personality for use 167 - the MD framework. It is not under active development. New 168 - projects should consider using DM_MULTIPATH which has more 169 - features and more testing. 170 - 171 - If unsure, say N. 172 - 173 - config MD_FAULTY 174 - tristate "Faulty test module for MD (deprecated)" 175 - depends on BLK_DEV_MD 176 - help 177 - The "faulty" module allows for a block device that occasionally returns 178 - read or write errors. It is useful for testing. 179 - 180 - In unsure, say N. 181 - 182 174 183 175 config MD_CLUSTER 184 176 tristate "Cluster Support for MD"

+2 -8

drivers/md/Makefile

··· 29 29 30 30 md-mod-y += md.o md-bitmap.o 31 31 raid456-y += raid5.o raid5-cache.o raid5-ppl.o 32 - linear-y += md-linear.o 33 - multipath-y += md-multipath.o 34 - faulty-y += md-faulty.o 35 32 36 33 # Note: link order is important. All raid personalities 37 - # and must come before md.o, as they each initialise 38 - # themselves, and md.o may use the personalities when it 34 + # and must come before md.o, as they each initialise 35 + # themselves, and md.o may use the personalities when it 39 36 # auto-initialised. 40 37 41 - obj-$(CONFIG_MD_LINEAR) += linear.o 42 38 obj-$(CONFIG_MD_RAID0) += raid0.o 43 39 obj-$(CONFIG_MD_RAID1) += raid1.o 44 40 obj-$(CONFIG_MD_RAID10) += raid10.o 45 41 obj-$(CONFIG_MD_RAID456) += raid456.o 46 - obj-$(CONFIG_MD_MULTIPATH) += multipath.o 47 - obj-$(CONFIG_MD_FAULTY) += faulty.o 48 42 obj-$(CONFIG_MD_CLUSTER) += md-cluster.o 49 43 obj-$(CONFIG_BCACHE) += bcache/ 50 44 obj-$(CONFIG_BLK_DEV_MD) += md-mod.o

-1

drivers/md/bcache/super.c

··· 954 954 q->limits.max_segment_size = UINT_MAX; 955 955 q->limits.max_segments = BIO_MAX_VECS; 956 956 blk_queue_max_discard_sectors(q, UINT_MAX); 957 - q->limits.discard_granularity = 512; 958 957 q->limits.io_min = block_size; 959 958 q->limits.logical_block_size = block_size; 960 959 q->limits.physical_block_size = block_size;

+1 -1

drivers/md/dm-kcopyd.c

··· 807 807 */ 808 808 if (!(job->flags & BIT(DM_KCOPYD_WRITE_SEQ))) { 809 809 for (i = 0; i < job->num_dests; i++) { 810 - if (bdev_zoned_model(dests[i].bdev) == BLK_ZONED_HM) { 810 + if (bdev_is_zoned(dests[i].bdev)) { 811 811 job->flags |= BIT(DM_KCOPYD_WRITE_SEQ); 812 812 break; 813 813 }

+20 -25

drivers/md/dm-table.c

··· 1579 1579 return true; 1580 1580 } 1581 1581 1582 - static int device_not_zoned_model(struct dm_target *ti, struct dm_dev *dev, 1583 - sector_t start, sector_t len, void *data) 1582 + static int device_not_zoned(struct dm_target *ti, struct dm_dev *dev, 1583 + sector_t start, sector_t len, void *data) 1584 1584 { 1585 - struct request_queue *q = bdev_get_queue(dev->bdev); 1586 - enum blk_zoned_model *zoned_model = data; 1585 + bool *zoned = data; 1587 1586 1588 - return blk_queue_zoned_model(q) != *zoned_model; 1587 + return bdev_is_zoned(dev->bdev) != *zoned; 1589 1588 } 1590 1589 1591 1590 static int device_is_zoned_model(struct dm_target *ti, struct dm_dev *dev, 1592 1591 sector_t start, sector_t len, void *data) 1593 1592 { 1594 - struct request_queue *q = bdev_get_queue(dev->bdev); 1595 - 1596 - return blk_queue_zoned_model(q) != BLK_ZONED_NONE; 1593 + return bdev_is_zoned(dev->bdev); 1597 1594 } 1598 1595 1599 1596 /* ··· 1600 1603 * has the DM_TARGET_MIXED_ZONED_MODEL feature set, the devices can have any 1601 1604 * zoned model with all zoned devices having the same zone size. 1602 1605 */ 1603 - static bool dm_table_supports_zoned_model(struct dm_table *t, 1604 - enum blk_zoned_model zoned_model) 1606 + static bool dm_table_supports_zoned(struct dm_table *t, bool zoned) 1605 1607 { 1606 1608 for (unsigned int i = 0; i < t->num_targets; i++) { 1607 1609 struct dm_target *ti = dm_table_get_target(t, i); ··· 1619 1623 1620 1624 if (dm_target_supports_zoned_hm(ti->type)) { 1621 1625 if (!ti->type->iterate_devices || 1622 - ti->type->iterate_devices(ti, device_not_zoned_model, 1623 - &zoned_model)) 1626 + ti->type->iterate_devices(ti, device_not_zoned, 1627 + &zoned)) 1624 1628 return false; 1625 1629 } else if (!dm_target_supports_mixed_zoned_model(ti->type)) { 1626 - if (zoned_model == BLK_ZONED_HM) 1630 + if (zoned) 1627 1631 return false; 1628 1632 } 1629 1633 } ··· 1646 1650 * zone sectors, if the destination device is a zoned block device, it shall 1647 1651 * have the specified zone_sectors. 1648 1652 */ 1649 - static int validate_hardware_zoned_model(struct dm_table *t, 1650 - enum blk_zoned_model zoned_model, 1651 - unsigned int zone_sectors) 1653 + static int validate_hardware_zoned(struct dm_table *t, bool zoned, 1654 + unsigned int zone_sectors) 1652 1655 { 1653 - if (zoned_model == BLK_ZONED_NONE) 1656 + if (!zoned) 1654 1657 return 0; 1655 1658 1656 - if (!dm_table_supports_zoned_model(t, zoned_model)) { 1659 + if (!dm_table_supports_zoned(t, zoned)) { 1657 1660 DMERR("%s: zoned model is not consistent across all devices", 1658 1661 dm_device_name(t->md)); 1659 1662 return -EINVAL; ··· 1678 1683 struct queue_limits *limits) 1679 1684 { 1680 1685 struct queue_limits ti_limits; 1681 - enum blk_zoned_model zoned_model = BLK_ZONED_NONE; 1682 1686 unsigned int zone_sectors = 0; 1687 + bool zoned = false; 1683 1688 1684 1689 blk_set_stacking_limits(limits); 1685 1690 ··· 1701 1706 ti->type->iterate_devices(ti, dm_set_device_limits, 1702 1707 &ti_limits); 1703 1708 1704 - if (zoned_model == BLK_ZONED_NONE && ti_limits.zoned != BLK_ZONED_NONE) { 1709 + if (!zoned && ti_limits.zoned) { 1705 1710 /* 1706 1711 * After stacking all limits, validate all devices 1707 1712 * in table support this zoned model and zone sectors. 1708 1713 */ 1709 - zoned_model = ti_limits.zoned; 1714 + zoned = ti_limits.zoned; 1710 1715 zone_sectors = ti_limits.chunk_sectors; 1711 1716 } 1712 1717 ··· 1739 1744 * Verify that the zoned model and zone sectors, as determined before 1740 1745 * any .io_hints override, are the same across all devices in the table. 1741 1746 * - this is especially relevant if .io_hints is emulating a disk-managed 1742 - * zoned model (aka BLK_ZONED_NONE) on host-managed zoned block devices. 1747 + * zoned model on host-managed zoned block devices. 1743 1748 * BUT... 1744 1749 */ 1745 - if (limits->zoned != BLK_ZONED_NONE) { 1750 + if (limits->zoned) { 1746 1751 /* 1747 1752 * ...IF the above limits stacking determined a zoned model 1748 1753 * validate that all of the table's devices conform to it. 1749 1754 */ 1750 - zoned_model = limits->zoned; 1755 + zoned = limits->zoned; 1751 1756 zone_sectors = limits->chunk_sectors; 1752 1757 } 1753 - if (validate_hardware_zoned_model(t, zoned_model, zone_sectors)) 1758 + if (validate_hardware_zoned(t, zoned, zone_sectors)) 1754 1759 return -EINVAL; 1755 1760 1756 1761 return validate_hardware_logical_block_alignment(t, limits);

+3 -4

drivers/md/dm-zoned-metadata.c

··· 2836 2836 { 2837 2837 struct dmz_dev *dev = &zmd->dev[num]; 2838 2838 2839 - if (bdev_zoned_model(dev->bdev) == BLK_ZONED_NONE) 2839 + if (!bdev_is_zoned(dev->bdev)) 2840 2840 dmz_dev_info(dev, "Regular block device"); 2841 2841 else 2842 - dmz_dev_info(dev, "Host-%s zoned block device", 2843 - bdev_zoned_model(dev->bdev) == BLK_ZONED_HA ? 2844 - "aware" : "managed"); 2842 + dmz_dev_info(dev, "Host-managed zoned block device"); 2843 + 2845 2844 if (zmd->sb_version > 1) { 2846 2845 sector_t sector_offset = 2847 2846 dev->zone_offset << zmd->zone_nr_sectors_shift;

+2 -2

drivers/md/dm-zoned-target.c

··· 702 702 } 703 703 704 704 bdev = ddev->bdev; 705 - if (bdev_zoned_model(bdev) == BLK_ZONED_NONE) { 705 + if (!bdev_is_zoned(bdev)) { 706 706 if (nr_devs == 1) { 707 707 ti->error = "Invalid regular device"; 708 708 goto err; ··· 1010 1010 limits->max_sectors = chunk_sectors; 1011 1011 1012 1012 /* We are exposing a drive-managed zoned block device */ 1013 - limits->zoned = BLK_ZONED_NONE; 1013 + limits->zoned = false; 1014 1014 } 1015 1015 1016 1016 /*

+2 -6

drivers/md/md-autodetect.c

··· 49 49 * instead of just one. -- KTK 50 50 * 18May2000: Added support for persistent-superblock arrays: 51 51 * md=n,0,factor,fault,device-list uses RAID0 for device n 52 - * md=n,-1,factor,fault,device-list uses LINEAR for device n 53 52 * md=n,device-list reads a RAID superblock from the devices 54 53 * elements in device-list are read by name_to_kdev_t so can be 55 54 * a hex number or something like /dev/hda1 /dev/sdb ··· 87 88 md_setup_ents++; 88 89 switch (get_option(&str, &level)) { /* RAID level */ 89 90 case 2: /* could be 0 or -1.. */ 90 - if (level == 0 || level == LEVEL_LINEAR) { 91 + if (level == 0) { 91 92 if (get_option(&str, &factor) != 2 || /* Chunk Size */ 92 93 get_option(&str, &fault) != 2) { 93 94 printk(KERN_WARNING "md: Too few arguments supplied to md=.\n"); ··· 95 96 } 96 97 md_setup_args[ent].level = level; 97 98 md_setup_args[ent].chunk = 1 << (factor+12); 98 - if (level == LEVEL_LINEAR) 99 - pername = "linear"; 100 - else 101 - pername = "raid0"; 99 + pername = "raid0"; 102 100 break; 103 101 } 104 102 fallthrough;

-365

drivers/md/md-faulty.c

··· 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 - /* 3 - * faulty.c : Multiple Devices driver for Linux 4 - * 5 - * Copyright (C) 2004 Neil Brown 6 - * 7 - * fautly-device-simulator personality for md 8 - */ 9 - 10 - 11 - /* 12 - * The "faulty" personality causes some requests to fail. 13 - * 14 - * Possible failure modes are: 15 - * reads fail "randomly" but succeed on retry 16 - * writes fail "randomly" but succeed on retry 17 - * reads for some address fail and then persist until a write 18 - * reads for some address fail and then persist irrespective of write 19 - * writes for some address fail and persist 20 - * all writes fail 21 - * 22 - * Different modes can be active at a time, but only 23 - * one can be set at array creation. Others can be added later. 24 - * A mode can be one-shot or recurrent with the recurrence being 25 - * once in every N requests. 26 - * The bottom 5 bits of the "layout" indicate the mode. The 27 - * remainder indicate a period, or 0 for one-shot. 28 - * 29 - * There is an implementation limit on the number of concurrently 30 - * persisting-faulty blocks. When a new fault is requested that would 31 - * exceed the limit, it is ignored. 32 - * All current faults can be clear using a layout of "0". 33 - * 34 - * Requests are always sent to the device. If they are to fail, 35 - * we clone the bio and insert a new b_end_io into the chain. 36 - */ 37 - 38 - #define WriteTransient 0 39 - #define ReadTransient 1 40 - #define WritePersistent 2 41 - #define ReadPersistent 3 42 - #define WriteAll 4 /* doesn't go to device */ 43 - #define ReadFixable 5 44 - #define Modes 6 45 - 46 - #define ClearErrors 31 47 - #define ClearFaults 30 48 - 49 - #define AllPersist 100 /* internal use only */ 50 - #define NoPersist 101 51 - 52 - #define ModeMask 0x1f 53 - #define ModeShift 5 54 - 55 - #define MaxFault 50 56 - #include <linux/blkdev.h> 57 - #include <linux/module.h> 58 - #include <linux/raid/md_u.h> 59 - #include <linux/slab.h> 60 - #include "md.h" 61 - #include <linux/seq_file.h> 62 - 63 - 64 - static void faulty_fail(struct bio *bio) 65 - { 66 - struct bio *b = bio->bi_private; 67 - 68 - b->bi_iter.bi_size = bio->bi_iter.bi_size; 69 - b->bi_iter.bi_sector = bio->bi_iter.bi_sector; 70 - 71 - bio_put(bio); 72 - 73 - bio_io_error(b); 74 - } 75 - 76 - struct faulty_conf { 77 - int period[Modes]; 78 - atomic_t counters[Modes]; 79 - sector_t faults[MaxFault]; 80 - int modes[MaxFault]; 81 - int nfaults; 82 - struct md_rdev *rdev; 83 - }; 84 - 85 - static int check_mode(struct faulty_conf *conf, int mode) 86 - { 87 - if (conf->period[mode] == 0 && 88 - atomic_read(&conf->counters[mode]) <= 0) 89 - return 0; /* no failure, no decrement */ 90 - 91 - 92 - if (atomic_dec_and_test(&conf->counters[mode])) { 93 - if (conf->period[mode]) 94 - atomic_set(&conf->counters[mode], conf->period[mode]); 95 - return 1; 96 - } 97 - return 0; 98 - } 99 - 100 - static int check_sector(struct faulty_conf *conf, sector_t start, sector_t end, int dir) 101 - { 102 - /* If we find a ReadFixable sector, we fix it ... */ 103 - int i; 104 - for (i=0; i<conf->nfaults; i++) 105 - if (conf->faults[i] >= start && 106 - conf->faults[i] < end) { 107 - /* found it ... */ 108 - switch (conf->modes[i] * 2 + dir) { 109 - case WritePersistent*2+WRITE: return 1; 110 - case ReadPersistent*2+READ: return 1; 111 - case ReadFixable*2+READ: return 1; 112 - case ReadFixable*2+WRITE: 113 - conf->modes[i] = NoPersist; 114 - return 0; 115 - case AllPersist*2+READ: 116 - case AllPersist*2+WRITE: return 1; 117 - default: 118 - return 0; 119 - } 120 - } 121 - return 0; 122 - } 123 - 124 - static void add_sector(struct faulty_conf *conf, sector_t start, int mode) 125 - { 126 - int i; 127 - int n = conf->nfaults; 128 - for (i=0; i<conf->nfaults; i++) 129 - if (conf->faults[i] == start) { 130 - switch(mode) { 131 - case NoPersist: conf->modes[i] = mode; return; 132 - case WritePersistent: 133 - if (conf->modes[i] == ReadPersistent || 134 - conf->modes[i] == ReadFixable) 135 - conf->modes[i] = AllPersist; 136 - else 137 - conf->modes[i] = WritePersistent; 138 - return; 139 - case ReadPersistent: 140 - if (conf->modes[i] == WritePersistent) 141 - conf->modes[i] = AllPersist; 142 - else 143 - conf->modes[i] = ReadPersistent; 144 - return; 145 - case ReadFixable: 146 - if (conf->modes[i] == WritePersistent || 147 - conf->modes[i] == ReadPersistent) 148 - conf->modes[i] = AllPersist; 149 - else 150 - conf->modes[i] = ReadFixable; 151 - return; 152 - } 153 - } else if (conf->modes[i] == NoPersist) 154 - n = i; 155 - 156 - if (n >= MaxFault) 157 - return; 158 - conf->faults[n] = start; 159 - conf->modes[n] = mode; 160 - if (conf->nfaults == n) 161 - conf->nfaults = n+1; 162 - } 163 - 164 - static bool faulty_make_request(struct mddev *mddev, struct bio *bio) 165 - { 166 - struct faulty_conf *conf = mddev->private; 167 - int failit = 0; 168 - 169 - if (bio_data_dir(bio) == WRITE) { 170 - /* write request */ 171 - if (atomic_read(&conf->counters[WriteAll])) { 172 - /* special case - don't decrement, don't submit_bio_noacct, 173 - * just fail immediately 174 - */ 175 - bio_io_error(bio); 176 - return true; 177 - } 178 - 179 - if (check_sector(conf, bio->bi_iter.bi_sector, 180 - bio_end_sector(bio), WRITE)) 181 - failit = 1; 182 - if (check_mode(conf, WritePersistent)) { 183 - add_sector(conf, bio->bi_iter.bi_sector, 184 - WritePersistent); 185 - failit = 1; 186 - } 187 - if (check_mode(conf, WriteTransient)) 188 - failit = 1; 189 - } else { 190 - /* read request */ 191 - if (check_sector(conf, bio->bi_iter.bi_sector, 192 - bio_end_sector(bio), READ)) 193 - failit = 1; 194 - if (check_mode(conf, ReadTransient)) 195 - failit = 1; 196 - if (check_mode(conf, ReadPersistent)) { 197 - add_sector(conf, bio->bi_iter.bi_sector, 198 - ReadPersistent); 199 - failit = 1; 200 - } 201 - if (check_mode(conf, ReadFixable)) { 202 - add_sector(conf, bio->bi_iter.bi_sector, 203 - ReadFixable); 204 - failit = 1; 205 - } 206 - } 207 - 208 - md_account_bio(mddev, &bio); 209 - if (failit) { 210 - struct bio *b = bio_alloc_clone(conf->rdev->bdev, bio, GFP_NOIO, 211 - &mddev->bio_set); 212 - 213 - b->bi_private = bio; 214 - b->bi_end_io = faulty_fail; 215 - bio = b; 216 - } else 217 - bio_set_dev(bio, conf->rdev->bdev); 218 - 219 - submit_bio_noacct(bio); 220 - return true; 221 - } 222 - 223 - static void faulty_status(struct seq_file *seq, struct mddev *mddev) 224 - { 225 - struct faulty_conf *conf = mddev->private; 226 - int n; 227 - 228 - if ((n=atomic_read(&conf->counters[WriteTransient])) != 0) 229 - seq_printf(seq, " WriteTransient=%d(%d)", 230 - n, conf->period[WriteTransient]); 231 - 232 - if ((n=atomic_read(&conf->counters[ReadTransient])) != 0) 233 - seq_printf(seq, " ReadTransient=%d(%d)", 234 - n, conf->period[ReadTransient]); 235 - 236 - if ((n=atomic_read(&conf->counters[WritePersistent])) != 0) 237 - seq_printf(seq, " WritePersistent=%d(%d)", 238 - n, conf->period[WritePersistent]); 239 - 240 - if ((n=atomic_read(&conf->counters[ReadPersistent])) != 0) 241 - seq_printf(seq, " ReadPersistent=%d(%d)", 242 - n, conf->period[ReadPersistent]); 243 - 244 - 245 - if ((n=atomic_read(&conf->counters[ReadFixable])) != 0) 246 - seq_printf(seq, " ReadFixable=%d(%d)", 247 - n, conf->period[ReadFixable]); 248 - 249 - if ((n=atomic_read(&conf->counters[WriteAll])) != 0) 250 - seq_printf(seq, " WriteAll"); 251 - 252 - seq_printf(seq, " nfaults=%d", conf->nfaults); 253 - } 254 - 255 - 256 - static int faulty_reshape(struct mddev *mddev) 257 - { 258 - int mode = mddev->new_layout & ModeMask; 259 - int count = mddev->new_layout >> ModeShift; 260 - struct faulty_conf *conf = mddev->private; 261 - 262 - if (mddev->new_layout < 0) 263 - return 0; 264 - 265 - /* new layout */ 266 - if (mode == ClearFaults) 267 - conf->nfaults = 0; 268 - else if (mode == ClearErrors) { 269 - int i; 270 - for (i=0 ; i < Modes ; i++) { 271 - conf->period[i] = 0; 272 - atomic_set(&conf->counters[i], 0); 273 - } 274 - } else if (mode < Modes) { 275 - conf->period[mode] = count; 276 - if (!count) count++; 277 - atomic_set(&conf->counters[mode], count); 278 - } else 279 - return -EINVAL; 280 - mddev->new_layout = -1; 281 - mddev->layout = -1; /* makes sure further changes come through */ 282 - return 0; 283 - } 284 - 285 - static sector_t faulty_size(struct mddev *mddev, sector_t sectors, int raid_disks) 286 - { 287 - WARN_ONCE(raid_disks, 288 - "%s does not support generic reshape\n", __func__); 289 - 290 - if (sectors == 0) 291 - return mddev->dev_sectors; 292 - 293 - return sectors; 294 - } 295 - 296 - static int faulty_run(struct mddev *mddev) 297 - { 298 - struct md_rdev *rdev; 299 - int i; 300 - struct faulty_conf *conf; 301 - 302 - if (md_check_no_bitmap(mddev)) 303 - return -EINVAL; 304 - 305 - conf = kmalloc(sizeof(*conf), GFP_KERNEL); 306 - if (!conf) 307 - return -ENOMEM; 308 - 309 - for (i=0; i<Modes; i++) { 310 - atomic_set(&conf->counters[i], 0); 311 - conf->period[i] = 0; 312 - } 313 - conf->nfaults = 0; 314 - 315 - rdev_for_each(rdev, mddev) { 316 - conf->rdev = rdev; 317 - disk_stack_limits(mddev->gendisk, rdev->bdev, 318 - rdev->data_offset << 9); 319 - } 320 - 321 - md_set_array_sectors(mddev, faulty_size(mddev, 0, 0)); 322 - mddev->private = conf; 323 - 324 - faulty_reshape(mddev); 325 - 326 - return 0; 327 - } 328 - 329 - static void faulty_free(struct mddev *mddev, void *priv) 330 - { 331 - struct faulty_conf *conf = priv; 332 - 333 - kfree(conf); 334 - } 335 - 336 - static struct md_personality faulty_personality = 337 - { 338 - .name = "faulty", 339 - .level = LEVEL_FAULTY, 340 - .owner = THIS_MODULE, 341 - .make_request = faulty_make_request, 342 - .run = faulty_run, 343 - .free = faulty_free, 344 - .status = faulty_status, 345 - .check_reshape = faulty_reshape, 346 - .size = faulty_size, 347 - }; 348 - 349 - static int __init raid_init(void) 350 - { 351 - return register_md_personality(&faulty_personality); 352 - } 353 - 354 - static void raid_exit(void) 355 - { 356 - unregister_md_personality(&faulty_personality); 357 - } 358 - 359 - module_init(raid_init); 360 - module_exit(raid_exit); 361 - MODULE_LICENSE("GPL"); 362 - MODULE_DESCRIPTION("Fault injection personality for MD (deprecated)"); 363 - MODULE_ALIAS("md-personality-10"); /* faulty */ 364 - MODULE_ALIAS("md-faulty"); 365 - MODULE_ALIAS("md-level--5");

-318

drivers/md/md-linear.c

··· 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 - /* 3 - linear.c : Multiple Devices driver for Linux 4 - Copyright (C) 1994-96 Marc ZYNGIER 5 - <zyngier@ufr-info-p7.ibp.fr> or 6 - <maz@gloups.fdn.fr> 7 - 8 - Linear mode management functions. 9 - 10 - */ 11 - 12 - #include <linux/blkdev.h> 13 - #include <linux/raid/md_u.h> 14 - #include <linux/seq_file.h> 15 - #include <linux/module.h> 16 - #include <linux/slab.h> 17 - #include <trace/events/block.h> 18 - #include "md.h" 19 - #include "md-linear.h" 20 - 21 - /* 22 - * find which device holds a particular offset 23 - */ 24 - static inline struct dev_info *which_dev(struct mddev *mddev, sector_t sector) 25 - { 26 - int lo, mid, hi; 27 - struct linear_conf *conf; 28 - 29 - lo = 0; 30 - hi = mddev->raid_disks - 1; 31 - conf = mddev->private; 32 - 33 - /* 34 - * Binary Search 35 - */ 36 - 37 - while (hi > lo) { 38 - 39 - mid = (hi + lo) / 2; 40 - if (sector < conf->disks[mid].end_sector) 41 - hi = mid; 42 - else 43 - lo = mid + 1; 44 - } 45 - 46 - return conf->disks + lo; 47 - } 48 - 49 - static sector_t linear_size(struct mddev *mddev, sector_t sectors, int raid_disks) 50 - { 51 - struct linear_conf *conf; 52 - sector_t array_sectors; 53 - 54 - conf = mddev->private; 55 - WARN_ONCE(sectors || raid_disks, 56 - "%s does not support generic reshape\n", __func__); 57 - array_sectors = conf->array_sectors; 58 - 59 - return array_sectors; 60 - } 61 - 62 - static struct linear_conf *linear_conf(struct mddev *mddev, int raid_disks) 63 - { 64 - struct linear_conf *conf; 65 - struct md_rdev *rdev; 66 - int i, cnt; 67 - 68 - conf = kzalloc(struct_size(conf, disks, raid_disks), GFP_KERNEL); 69 - if (!conf) 70 - return NULL; 71 - 72 - /* 73 - * conf->raid_disks is copy of mddev->raid_disks. The reason to 74 - * keep a copy of mddev->raid_disks in struct linear_conf is, 75 - * mddev->raid_disks may not be consistent with pointers number of 76 - * conf->disks[] when it is updated in linear_add() and used to 77 - * iterate old conf->disks[] earray in linear_congested(). 78 - * Here conf->raid_disks is always consitent with number of 79 - * pointers in conf->disks[] array, and mddev->private is updated 80 - * with rcu_assign_pointer() in linear_addr(), such race can be 81 - * avoided. 82 - */ 83 - conf->raid_disks = raid_disks; 84 - 85 - cnt = 0; 86 - conf->array_sectors = 0; 87 - 88 - rdev_for_each(rdev, mddev) { 89 - int j = rdev->raid_disk; 90 - struct dev_info *disk = conf->disks + j; 91 - sector_t sectors; 92 - 93 - if (j < 0 || j >= raid_disks || disk->rdev) { 94 - pr_warn("md/linear:%s: disk numbering problem. Aborting!\n", 95 - mdname(mddev)); 96 - goto out; 97 - } 98 - 99 - disk->rdev = rdev; 100 - if (mddev->chunk_sectors) { 101 - sectors = rdev->sectors; 102 - sector_div(sectors, mddev->chunk_sectors); 103 - rdev->sectors = sectors * mddev->chunk_sectors; 104 - } 105 - 106 - disk_stack_limits(mddev->gendisk, rdev->bdev, 107 - rdev->data_offset << 9); 108 - 109 - conf->array_sectors += rdev->sectors; 110 - cnt++; 111 - } 112 - if (cnt != raid_disks) { 113 - pr_warn("md/linear:%s: not enough drives present. Aborting!\n", 114 - mdname(mddev)); 115 - goto out; 116 - } 117 - 118 - /* 119 - * Here we calculate the device offsets. 120 - */ 121 - conf->disks[0].end_sector = conf->disks[0].rdev->sectors; 122 - 123 - for (i = 1; i < raid_disks; i++) 124 - conf->disks[i].end_sector = 125 - conf->disks[i-1].end_sector + 126 - conf->disks[i].rdev->sectors; 127 - 128 - return conf; 129 - 130 - out: 131 - kfree(conf); 132 - return NULL; 133 - } 134 - 135 - static int linear_run (struct mddev *mddev) 136 - { 137 - struct linear_conf *conf; 138 - int ret; 139 - 140 - if (md_check_no_bitmap(mddev)) 141 - return -EINVAL; 142 - conf = linear_conf(mddev, mddev->raid_disks); 143 - 144 - if (!conf) 145 - return 1; 146 - mddev->private = conf; 147 - md_set_array_sectors(mddev, linear_size(mddev, 0, 0)); 148 - 149 - ret = md_integrity_register(mddev); 150 - if (ret) { 151 - kfree(conf); 152 - mddev->private = NULL; 153 - } 154 - return ret; 155 - } 156 - 157 - static int linear_add(struct mddev *mddev, struct md_rdev *rdev) 158 - { 159 - /* Adding a drive to a linear array allows the array to grow. 160 - * It is permitted if the new drive has a matching superblock 161 - * already on it, with raid_disk equal to raid_disks. 162 - * It is achieved by creating a new linear_private_data structure 163 - * and swapping it in in-place of the current one. 164 - * The current one is never freed until the array is stopped. 165 - * This avoids races. 166 - */ 167 - struct linear_conf *newconf, *oldconf; 168 - 169 - if (rdev->saved_raid_disk != mddev->raid_disks) 170 - return -EINVAL; 171 - 172 - rdev->raid_disk = rdev->saved_raid_disk; 173 - rdev->saved_raid_disk = -1; 174 - 175 - newconf = linear_conf(mddev,mddev->raid_disks+1); 176 - 177 - if (!newconf) 178 - return -ENOMEM; 179 - 180 - /* newconf->raid_disks already keeps a copy of * the increased 181 - * value of mddev->raid_disks, WARN_ONCE() is just used to make 182 - * sure of this. It is possible that oldconf is still referenced 183 - * in linear_congested(), therefore kfree_rcu() is used to free 184 - * oldconf until no one uses it anymore. 185 - */ 186 - oldconf = rcu_dereference_protected(mddev->private, 187 - lockdep_is_held(&mddev->reconfig_mutex)); 188 - mddev->raid_disks++; 189 - WARN_ONCE(mddev->raid_disks != newconf->raid_disks, 190 - "copied raid_disks doesn't match mddev->raid_disks"); 191 - rcu_assign_pointer(mddev->private, newconf); 192 - md_set_array_sectors(mddev, linear_size(mddev, 0, 0)); 193 - set_capacity_and_notify(mddev->gendisk, mddev->array_sectors); 194 - kfree_rcu(oldconf, rcu); 195 - return 0; 196 - } 197 - 198 - static void linear_free(struct mddev *mddev, void *priv) 199 - { 200 - struct linear_conf *conf = priv; 201 - 202 - kfree(conf); 203 - } 204 - 205 - static bool linear_make_request(struct mddev *mddev, struct bio *bio) 206 - { 207 - struct dev_info *tmp_dev; 208 - sector_t start_sector, end_sector, data_offset; 209 - sector_t bio_sector = bio->bi_iter.bi_sector; 210 - 211 - if (unlikely(bio->bi_opf & REQ_PREFLUSH) 212 - && md_flush_request(mddev, bio)) 213 - return true; 214 - 215 - tmp_dev = which_dev(mddev, bio_sector); 216 - start_sector = tmp_dev->end_sector - tmp_dev->rdev->sectors; 217 - end_sector = tmp_dev->end_sector; 218 - data_offset = tmp_dev->rdev->data_offset; 219 - 220 - if (unlikely(bio_sector >= end_sector || 221 - bio_sector < start_sector)) 222 - goto out_of_bounds; 223 - 224 - if (unlikely(is_rdev_broken(tmp_dev->rdev))) { 225 - md_error(mddev, tmp_dev->rdev); 226 - bio_io_error(bio); 227 - return true; 228 - } 229 - 230 - if (unlikely(bio_end_sector(bio) > end_sector)) { 231 - /* This bio crosses a device boundary, so we have to split it */ 232 - struct bio *split = bio_split(bio, end_sector - bio_sector, 233 - GFP_NOIO, &mddev->bio_set); 234 - bio_chain(split, bio); 235 - submit_bio_noacct(bio); 236 - bio = split; 237 - } 238 - 239 - md_account_bio(mddev, &bio); 240 - bio_set_dev(bio, tmp_dev->rdev->bdev); 241 - bio->bi_iter.bi_sector = bio->bi_iter.bi_sector - 242 - start_sector + data_offset; 243 - 244 - if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && 245 - !bdev_max_discard_sectors(bio->bi_bdev))) { 246 - /* Just ignore it */ 247 - bio_endio(bio); 248 - } else { 249 - if (mddev->gendisk) 250 - trace_block_bio_remap(bio, disk_devt(mddev->gendisk), 251 - bio_sector); 252 - mddev_check_write_zeroes(mddev, bio); 253 - submit_bio_noacct(bio); 254 - } 255 - return true; 256 - 257 - out_of_bounds: 258 - pr_err("md/linear:%s: make_request: Sector %llu out of bounds on dev %pg: %llu sectors, offset %llu\n", 259 - mdname(mddev), 260 - (unsigned long long)bio->bi_iter.bi_sector, 261 - tmp_dev->rdev->bdev, 262 - (unsigned long long)tmp_dev->rdev->sectors, 263 - (unsigned long long)start_sector); 264 - bio_io_error(bio); 265 - return true; 266 - } 267 - 268 - static void linear_status (struct seq_file *seq, struct mddev *mddev) 269 - { 270 - seq_printf(seq, " %dk rounding", mddev->chunk_sectors / 2); 271 - } 272 - 273 - static void linear_error(struct mddev *mddev, struct md_rdev *rdev) 274 - { 275 - if (!test_and_set_bit(MD_BROKEN, &mddev->flags)) { 276 - char *md_name = mdname(mddev); 277 - 278 - pr_crit("md/linear%s: Disk failure on %pg detected, failing array.\n", 279 - md_name, rdev->bdev); 280 - } 281 - } 282 - 283 - static void linear_quiesce(struct mddev *mddev, int state) 284 - { 285 - } 286 - 287 - static struct md_personality linear_personality = 288 - { 289 - .name = "linear", 290 - .level = LEVEL_LINEAR, 291 - .owner = THIS_MODULE, 292 - .make_request = linear_make_request, 293 - .run = linear_run, 294 - .free = linear_free, 295 - .status = linear_status, 296 - .hot_add_disk = linear_add, 297 - .size = linear_size, 298 - .quiesce = linear_quiesce, 299 - .error_handler = linear_error, 300 - }; 301 - 302 - static int __init linear_init (void) 303 - { 304 - return register_md_personality (&linear_personality); 305 - } 306 - 307 - static void linear_exit (void) 308 - { 309 - unregister_md_personality (&linear_personality); 310 - } 311 - 312 - module_init(linear_init); 313 - module_exit(linear_exit); 314 - MODULE_LICENSE("GPL"); 315 - MODULE_DESCRIPTION("Linear device concatenation personality for MD (deprecated)"); 316 - MODULE_ALIAS("md-personality-1"); /* LINEAR - deprecated*/ 317 - MODULE_ALIAS("md-linear"); 318 - MODULE_ALIAS("md-level--1");

-471

drivers/md/md-multipath.c

··· 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 - /* 3 - * multipath.c : Multiple Devices driver for Linux 4 - * 5 - * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat 6 - * 7 - * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman 8 - * 9 - * MULTIPATH management functions. 10 - * 11 - * derived from raid1.c. 12 - */ 13 - 14 - #include <linux/blkdev.h> 15 - #include <linux/module.h> 16 - #include <linux/raid/md_u.h> 17 - #include <linux/seq_file.h> 18 - #include <linux/slab.h> 19 - #include "md.h" 20 - #include "md-multipath.h" 21 - 22 - #define MAX_WORK_PER_DISK 128 23 - 24 - #define NR_RESERVED_BUFS 32 25 - 26 - static int multipath_map (struct mpconf *conf) 27 - { 28 - int i, disks = conf->raid_disks; 29 - 30 - /* 31 - * Later we do read balancing on the read side 32 - * now we use the first available disk. 33 - */ 34 - 35 - rcu_read_lock(); 36 - for (i = 0; i < disks; i++) { 37 - struct md_rdev *rdev = rcu_dereference(conf->multipaths[i].rdev); 38 - if (rdev && test_bit(In_sync, &rdev->flags) && 39 - !test_bit(Faulty, &rdev->flags)) { 40 - atomic_inc(&rdev->nr_pending); 41 - rcu_read_unlock(); 42 - return i; 43 - } 44 - } 45 - rcu_read_unlock(); 46 - 47 - pr_crit_ratelimited("multipath_map(): no more operational IO paths?\n"); 48 - return (-1); 49 - } 50 - 51 - static void multipath_reschedule_retry (struct multipath_bh *mp_bh) 52 - { 53 - unsigned long flags; 54 - struct mddev *mddev = mp_bh->mddev; 55 - struct mpconf *conf = mddev->private; 56 - 57 - spin_lock_irqsave(&conf->device_lock, flags); 58 - list_add(&mp_bh->retry_list, &conf->retry_list); 59 - spin_unlock_irqrestore(&conf->device_lock, flags); 60 - md_wakeup_thread(mddev->thread); 61 - } 62 - 63 - /* 64 - * multipath_end_bh_io() is called when we have finished servicing a multipathed 65 - * operation and are ready to return a success/failure code to the buffer 66 - * cache layer. 67 - */ 68 - static void multipath_end_bh_io(struct multipath_bh *mp_bh, blk_status_t status) 69 - { 70 - struct bio *bio = mp_bh->master_bio; 71 - struct mpconf *conf = mp_bh->mddev->private; 72 - 73 - bio->bi_status = status; 74 - bio_endio(bio); 75 - mempool_free(mp_bh, &conf->pool); 76 - } 77 - 78 - static void multipath_end_request(struct bio *bio) 79 - { 80 - struct multipath_bh *mp_bh = bio->bi_private; 81 - struct mpconf *conf = mp_bh->mddev->private; 82 - struct md_rdev *rdev = conf->multipaths[mp_bh->path].rdev; 83 - 84 - if (!bio->bi_status) 85 - multipath_end_bh_io(mp_bh, 0); 86 - else if (!(bio->bi_opf & REQ_RAHEAD)) { 87 - /* 88 - * oops, IO error: 89 - */ 90 - md_error (mp_bh->mddev, rdev); 91 - pr_info("multipath: %pg: rescheduling sector %llu\n", 92 - rdev->bdev, 93 - (unsigned long long)bio->bi_iter.bi_sector); 94 - multipath_reschedule_retry(mp_bh); 95 - } else 96 - multipath_end_bh_io(mp_bh, bio->bi_status); 97 - rdev_dec_pending(rdev, conf->mddev); 98 - } 99 - 100 - static bool multipath_make_request(struct mddev *mddev, struct bio * bio) 101 - { 102 - struct mpconf *conf = mddev->private; 103 - struct multipath_bh * mp_bh; 104 - struct multipath_info *multipath; 105 - 106 - if (unlikely(bio->bi_opf & REQ_PREFLUSH) 107 - && md_flush_request(mddev, bio)) 108 - return true; 109 - 110 - md_account_bio(mddev, &bio); 111 - mp_bh = mempool_alloc(&conf->pool, GFP_NOIO); 112 - 113 - mp_bh->master_bio = bio; 114 - mp_bh->mddev = mddev; 115 - 116 - mp_bh->path = multipath_map(conf); 117 - if (mp_bh->path < 0) { 118 - bio_io_error(bio); 119 - mempool_free(mp_bh, &conf->pool); 120 - return true; 121 - } 122 - multipath = conf->multipaths + mp_bh->path; 123 - 124 - bio_init_clone(multipath->rdev->bdev, &mp_bh->bio, bio, GFP_NOIO); 125 - 126 - mp_bh->bio.bi_iter.bi_sector += multipath->rdev->data_offset; 127 - mp_bh->bio.bi_opf |= REQ_FAILFAST_TRANSPORT; 128 - mp_bh->bio.bi_end_io = multipath_end_request; 129 - mp_bh->bio.bi_private = mp_bh; 130 - mddev_check_write_zeroes(mddev, &mp_bh->bio); 131 - submit_bio_noacct(&mp_bh->bio); 132 - return true; 133 - } 134 - 135 - static void multipath_status(struct seq_file *seq, struct mddev *mddev) 136 - { 137 - struct mpconf *conf = mddev->private; 138 - int i; 139 - 140 - seq_printf (seq, " [%d/%d] [", conf->raid_disks, 141 - conf->raid_disks - mddev->degraded); 142 - rcu_read_lock(); 143 - for (i = 0; i < conf->raid_disks; i++) { 144 - struct md_rdev *rdev = rcu_dereference(conf->multipaths[i].rdev); 145 - seq_printf (seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 146 - } 147 - rcu_read_unlock(); 148 - seq_putc(seq, ']'); 149 - } 150 - 151 - /* 152 - * Careful, this can execute in IRQ contexts as well! 153 - */ 154 - static void multipath_error (struct mddev *mddev, struct md_rdev *rdev) 155 - { 156 - struct mpconf *conf = mddev->private; 157 - 158 - if (conf->raid_disks - mddev->degraded <= 1) { 159 - /* 160 - * Uh oh, we can do nothing if this is our last path, but 161 - * first check if this is a queued request for a device 162 - * which has just failed. 163 - */ 164 - pr_warn("multipath: only one IO path left and IO error.\n"); 165 - /* leave it active... it's all we have */ 166 - return; 167 - } 168 - /* 169 - * Mark disk as unusable 170 - */ 171 - if (test_and_clear_bit(In_sync, &rdev->flags)) { 172 - unsigned long flags; 173 - spin_lock_irqsave(&conf->device_lock, flags); 174 - mddev->degraded++; 175 - spin_unlock_irqrestore(&conf->device_lock, flags); 176 - } 177 - set_bit(Faulty, &rdev->flags); 178 - set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 179 - pr_err("multipath: IO failure on %pg, disabling IO path.\n" 180 - "multipath: Operation continuing on %d IO paths.\n", 181 - rdev->bdev, 182 - conf->raid_disks - mddev->degraded); 183 - } 184 - 185 - static void print_multipath_conf (struct mpconf *conf) 186 - { 187 - int i; 188 - struct multipath_info *tmp; 189 - 190 - pr_debug("MULTIPATH conf printout:\n"); 191 - if (!conf) { 192 - pr_debug("(conf==NULL)\n"); 193 - return; 194 - } 195 - pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded, 196 - conf->raid_disks); 197 - 198 - for (i = 0; i < conf->raid_disks; i++) { 199 - tmp = conf->multipaths + i; 200 - if (tmp->rdev) 201 - pr_debug(" disk%d, o:%d, dev:%pg\n", 202 - i,!test_bit(Faulty, &tmp->rdev->flags), 203 - tmp->rdev->bdev); 204 - } 205 - } 206 - 207 - static int multipath_add_disk(struct mddev *mddev, struct md_rdev *rdev) 208 - { 209 - struct mpconf *conf = mddev->private; 210 - int err = -EEXIST; 211 - int path; 212 - struct multipath_info *p; 213 - int first = 0; 214 - int last = mddev->raid_disks - 1; 215 - 216 - if (rdev->raid_disk >= 0) 217 - first = last = rdev->raid_disk; 218 - 219 - print_multipath_conf(conf); 220 - 221 - for (path = first; path <= last; path++) 222 - if ((p=conf->multipaths+path)->rdev == NULL) { 223 - disk_stack_limits(mddev->gendisk, rdev->bdev, 224 - rdev->data_offset << 9); 225 - 226 - err = md_integrity_add_rdev(rdev, mddev); 227 - if (err) 228 - break; 229 - spin_lock_irq(&conf->device_lock); 230 - mddev->degraded--; 231 - rdev->raid_disk = path; 232 - set_bit(In_sync, &rdev->flags); 233 - spin_unlock_irq(&conf->device_lock); 234 - rcu_assign_pointer(p->rdev, rdev); 235 - err = 0; 236 - break; 237 - } 238 - 239 - print_multipath_conf(conf); 240 - 241 - return err; 242 - } 243 - 244 - static int multipath_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 245 - { 246 - struct mpconf *conf = mddev->private; 247 - int err = 0; 248 - int number = rdev->raid_disk; 249 - struct multipath_info *p = conf->multipaths + number; 250 - 251 - print_multipath_conf(conf); 252 - 253 - if (rdev == p->rdev) { 254 - if (test_bit(In_sync, &rdev->flags) || 255 - atomic_read(&rdev->nr_pending)) { 256 - pr_warn("hot-remove-disk, slot %d is identified but is still operational!\n", number); 257 - err = -EBUSY; 258 - goto abort; 259 - } 260 - p->rdev = NULL; 261 - if (!test_bit(RemoveSynchronized, &rdev->flags)) { 262 - synchronize_rcu(); 263 - if (atomic_read(&rdev->nr_pending)) { 264 - /* lost the race, try later */ 265 - err = -EBUSY; 266 - p->rdev = rdev; 267 - goto abort; 268 - } 269 - } 270 - err = md_integrity_register(mddev); 271 - } 272 - abort: 273 - 274 - print_multipath_conf(conf); 275 - return err; 276 - } 277 - 278 - /* 279 - * This is a kernel thread which: 280 - * 281 - * 1. Retries failed read operations on working multipaths. 282 - * 2. Updates the raid superblock when problems encounter. 283 - * 3. Performs writes following reads for array syncronising. 284 - */ 285 - 286 - static void multipathd(struct md_thread *thread) 287 - { 288 - struct mddev *mddev = thread->mddev; 289 - struct multipath_bh *mp_bh; 290 - struct bio *bio; 291 - unsigned long flags; 292 - struct mpconf *conf = mddev->private; 293 - struct list_head *head = &conf->retry_list; 294 - 295 - md_check_recovery(mddev); 296 - for (;;) { 297 - spin_lock_irqsave(&conf->device_lock, flags); 298 - if (list_empty(head)) 299 - break; 300 - mp_bh = list_entry(head->prev, struct multipath_bh, retry_list); 301 - list_del(head->prev); 302 - spin_unlock_irqrestore(&conf->device_lock, flags); 303 - 304 - bio = &mp_bh->bio; 305 - bio->bi_iter.bi_sector = mp_bh->master_bio->bi_iter.bi_sector; 306 - 307 - if ((mp_bh->path = multipath_map (conf))<0) { 308 - pr_err("multipath: %pg: unrecoverable IO read error for block %llu\n", 309 - bio->bi_bdev, 310 - (unsigned long long)bio->bi_iter.bi_sector); 311 - multipath_end_bh_io(mp_bh, BLK_STS_IOERR); 312 - } else { 313 - pr_err("multipath: %pg: redirecting sector %llu to another IO path\n", 314 - bio->bi_bdev, 315 - (unsigned long long)bio->bi_iter.bi_sector); 316 - *bio = *(mp_bh->master_bio); 317 - bio->bi_iter.bi_sector += 318 - conf->multipaths[mp_bh->path].rdev->data_offset; 319 - bio_set_dev(bio, conf->multipaths[mp_bh->path].rdev->bdev); 320 - bio->bi_opf |= REQ_FAILFAST_TRANSPORT; 321 - bio->bi_end_io = multipath_end_request; 322 - bio->bi_private = mp_bh; 323 - submit_bio_noacct(bio); 324 - } 325 - } 326 - spin_unlock_irqrestore(&conf->device_lock, flags); 327 - } 328 - 329 - static sector_t multipath_size(struct mddev *mddev, sector_t sectors, int raid_disks) 330 - { 331 - WARN_ONCE(sectors || raid_disks, 332 - "%s does not support generic reshape\n", __func__); 333 - 334 - return mddev->dev_sectors; 335 - } 336 - 337 - static int multipath_run (struct mddev *mddev) 338 - { 339 - struct mpconf *conf; 340 - int disk_idx; 341 - struct multipath_info *disk; 342 - struct md_rdev *rdev; 343 - int working_disks; 344 - int ret; 345 - 346 - if (md_check_no_bitmap(mddev)) 347 - return -EINVAL; 348 - 349 - if (mddev->level != LEVEL_MULTIPATH) { 350 - pr_warn("multipath: %s: raid level not set to multipath IO (%d)\n", 351 - mdname(mddev), mddev->level); 352 - goto out; 353 - } 354 - /* 355 - * copy the already verified devices into our private MULTIPATH 356 - * bookkeeping area. [whatever we allocate in multipath_run(), 357 - * should be freed in multipath_free()] 358 - */ 359 - 360 - conf = kzalloc(sizeof(struct mpconf), GFP_KERNEL); 361 - mddev->private = conf; 362 - if (!conf) 363 - goto out; 364 - 365 - conf->multipaths = kcalloc(mddev->raid_disks, 366 - sizeof(struct multipath_info), 367 - GFP_KERNEL); 368 - if (!conf->multipaths) 369 - goto out_free_conf; 370 - 371 - working_disks = 0; 372 - rdev_for_each(rdev, mddev) { 373 - disk_idx = rdev->raid_disk; 374 - if (disk_idx < 0 || 375 - disk_idx >= mddev->raid_disks) 376 - continue; 377 - 378 - disk = conf->multipaths + disk_idx; 379 - disk->rdev = rdev; 380 - disk_stack_limits(mddev->gendisk, rdev->bdev, 381 - rdev->data_offset << 9); 382 - 383 - if (!test_bit(Faulty, &rdev->flags)) 384 - working_disks++; 385 - } 386 - 387 - conf->raid_disks = mddev->raid_disks; 388 - conf->mddev = mddev; 389 - spin_lock_init(&conf->device_lock); 390 - INIT_LIST_HEAD(&conf->retry_list); 391 - 392 - if (!working_disks) { 393 - pr_warn("multipath: no operational IO paths for %s\n", 394 - mdname(mddev)); 395 - goto out_free_conf; 396 - } 397 - mddev->degraded = conf->raid_disks - working_disks; 398 - 399 - ret = mempool_init_kmalloc_pool(&conf->pool, NR_RESERVED_BUFS, 400 - sizeof(struct multipath_bh)); 401 - if (ret) 402 - goto out_free_conf; 403 - 404 - rcu_assign_pointer(mddev->thread, 405 - md_register_thread(multipathd, mddev, "multipath")); 406 - if (!mddev->thread) 407 - goto out_free_conf; 408 - 409 - pr_info("multipath: array %s active with %d out of %d IO paths\n", 410 - mdname(mddev), conf->raid_disks - mddev->degraded, 411 - mddev->raid_disks); 412 - /* 413 - * Ok, everything is just fine now 414 - */ 415 - md_set_array_sectors(mddev, multipath_size(mddev, 0, 0)); 416 - 417 - if (md_integrity_register(mddev)) 418 - goto out_free_conf; 419 - 420 - return 0; 421 - 422 - out_free_conf: 423 - mempool_exit(&conf->pool); 424 - kfree(conf->multipaths); 425 - kfree(conf); 426 - mddev->private = NULL; 427 - out: 428 - return -EIO; 429 - } 430 - 431 - static void multipath_free(struct mddev *mddev, void *priv) 432 - { 433 - struct mpconf *conf = priv; 434 - 435 - mempool_exit(&conf->pool); 436 - kfree(conf->multipaths); 437 - kfree(conf); 438 - } 439 - 440 - static struct md_personality multipath_personality = 441 - { 442 - .name = "multipath", 443 - .level = LEVEL_MULTIPATH, 444 - .owner = THIS_MODULE, 445 - .make_request = multipath_make_request, 446 - .run = multipath_run, 447 - .free = multipath_free, 448 - .status = multipath_status, 449 - .error_handler = multipath_error, 450 - .hot_add_disk = multipath_add_disk, 451 - .hot_remove_disk= multipath_remove_disk, 452 - .size = multipath_size, 453 - }; 454 - 455 - static int __init multipath_init (void) 456 - { 457 - return register_md_personality (&multipath_personality); 458 - } 459 - 460 - static void __exit multipath_exit (void) 461 - { 462 - unregister_md_personality (&multipath_personality); 463 - } 464 - 465 - module_init(multipath_init); 466 - module_exit(multipath_exit); 467 - MODULE_LICENSE("GPL"); 468 - MODULE_DESCRIPTION("simple multi-path personality for MD (deprecated)"); 469 - MODULE_ALIAS("md-personality-7"); /* MULTIPATH */ 470 - MODULE_ALIAS("md-multipath"); 471 - MODULE_ALIAS("md-level--4");

+150 -155

drivers/md/md.c

··· 543 543 rdev_dec_pending(rdev, mddev); 544 544 545 545 if (atomic_dec_and_test(&mddev->flush_pending)) { 546 + /* The pair is percpu_ref_get() from md_flush_request() */ 547 + percpu_ref_put(&mddev->active_io); 548 + 546 549 /* The pre-request flush has finished */ 547 550 queue_work(md_wq, &mddev->flush_work); 548 551 } ··· 565 562 rdev_for_each_rcu(rdev, mddev) 566 563 if (rdev->raid_disk >= 0 && 567 564 !test_bit(Faulty, &rdev->flags)) { 568 - /* Take two references, one is dropped 569 - * when request finishes, one after 570 - * we reclaim rcu_read_lock 571 - */ 572 565 struct bio *bi; 573 - atomic_inc(&rdev->nr_pending); 566 + 574 567 atomic_inc(&rdev->nr_pending); 575 568 rcu_read_unlock(); 576 569 bi = bio_alloc_bioset(rdev->bdev, 0, ··· 577 578 atomic_inc(&mddev->flush_pending); 578 579 submit_bio(bi); 579 580 rcu_read_lock(); 580 - rdev_dec_pending(rdev, mddev); 581 581 } 582 582 rcu_read_unlock(); 583 583 if (atomic_dec_and_test(&mddev->flush_pending)) ··· 629 631 /* new request after previous flush is completed */ 630 632 if (ktime_after(req_start, mddev->prev_flush_start)) { 631 633 WARN_ON(mddev->flush_bio); 634 + /* 635 + * Grab a reference to make sure mddev_suspend() will wait for 636 + * this flush to be done. 637 + * 638 + * md_flush_reqeust() is called under md_handle_request() and 639 + * 'active_io' is already grabbed, hence percpu_ref_is_zero() 640 + * won't pass, percpu_ref_tryget_live() can't be used because 641 + * percpu_ref_kill() can be called by mddev_suspend() 642 + * concurrently. 643 + */ 644 + WARN_ON(percpu_ref_is_zero(&mddev->active_io)); 645 + percpu_ref_get(&mddev->active_io); 632 646 mddev->flush_bio = bio; 633 647 bio = NULL; 634 648 } ··· 1037 1027 return; 1038 1028 1039 1029 bio = bio_alloc_bioset(rdev->meta_bdev ? rdev->meta_bdev : rdev->bdev, 1040 - 1, 1041 - REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH | REQ_FUA, 1042 - GFP_NOIO, &mddev->sync_set); 1030 + 1, 1031 + REQ_OP_WRITE | REQ_SYNC | REQ_IDLE | REQ_META 1032 + | REQ_PREFLUSH | REQ_FUA, 1033 + GFP_NOIO, &mddev->sync_set); 1043 1034 1044 1035 atomic_inc(&rdev->nr_pending); 1045 1036 ··· 1220 1209 struct md_rdev *refdev, 1221 1210 int minor_version); 1222 1211 int (*validate_super)(struct mddev *mddev, 1212 + struct md_rdev *freshest, 1223 1213 struct md_rdev *rdev); 1224 1214 void (*sync_super)(struct mddev *mddev, 1225 1215 struct md_rdev *rdev); ··· 1301 1289 rdev->sb_size = MD_SB_BYTES; 1302 1290 rdev->badblocks.shift = -1; 1303 1291 1304 - if (sb->level == LEVEL_MULTIPATH) 1305 - rdev->desc_nr = -1; 1306 - else 1307 - rdev->desc_nr = sb->this_disk.number; 1292 + rdev->desc_nr = sb->this_disk.number; 1308 1293 1309 - /* not spare disk, or LEVEL_MULTIPATH */ 1310 - if (sb->level == LEVEL_MULTIPATH || 1311 - (rdev->desc_nr >= 0 && 1312 - rdev->desc_nr < MD_SB_DISKS && 1313 - sb->disks[rdev->desc_nr].state & 1314 - ((1<<MD_DISK_SYNC) | (1 << MD_DISK_ACTIVE)))) 1294 + /* not spare disk */ 1295 + if (rdev->desc_nr >= 0 && rdev->desc_nr < MD_SB_DISKS && 1296 + sb->disks[rdev->desc_nr].state & ((1<<MD_DISK_SYNC) | (1 << MD_DISK_ACTIVE))) 1315 1297 spare_disk = false; 1316 1298 1317 1299 if (!refdev) { ··· 1352 1346 1353 1347 /* 1354 1348 * validate_super for 0.90.0 1349 + * note: we are not using "freshest" for 0.9 superblock 1355 1350 */ 1356 - static int super_90_validate(struct mddev *mddev, struct md_rdev *rdev) 1351 + static int super_90_validate(struct mddev *mddev, struct md_rdev *freshest, struct md_rdev *rdev) 1357 1352 { 1358 1353 mdp_disk_t *desc; 1359 1354 mdp_super_t *sb = page_address(rdev->sb_page); ··· 1452 1445 return 0; 1453 1446 } 1454 1447 1455 - if (mddev->level != LEVEL_MULTIPATH) { 1456 - desc = sb->disks + rdev->desc_nr; 1448 + desc = sb->disks + rdev->desc_nr; 1457 1449 1458 - if (desc->state & (1<<MD_DISK_FAULTY)) 1459 - set_bit(Faulty, &rdev->flags); 1460 - else if (desc->state & (1<<MD_DISK_SYNC) /* && 1461 - desc->raid_disk < mddev->raid_disks */) { 1462 - set_bit(In_sync, &rdev->flags); 1463 - rdev->raid_disk = desc->raid_disk; 1464 - rdev->saved_raid_disk = desc->raid_disk; 1465 - } else if (desc->state & (1<<MD_DISK_ACTIVE)) { 1466 - /* active but not in sync implies recovery up to 1467 - * reshape position. We don't know exactly where 1468 - * that is, so set to zero for now */ 1469 - if (mddev->minor_version >= 91) { 1470 - rdev->recovery_offset = 0; 1471 - rdev->raid_disk = desc->raid_disk; 1472 - } 1473 - } 1474 - if (desc->state & (1<<MD_DISK_WRITEMOSTLY)) 1475 - set_bit(WriteMostly, &rdev->flags); 1476 - if (desc->state & (1<<MD_DISK_FAILFAST)) 1477 - set_bit(FailFast, &rdev->flags); 1478 - } else /* MULTIPATH are always insync */ 1450 + if (desc->state & (1<<MD_DISK_FAULTY)) 1451 + set_bit(Faulty, &rdev->flags); 1452 + else if (desc->state & (1<<MD_DISK_SYNC)) { 1479 1453 set_bit(In_sync, &rdev->flags); 1454 + rdev->raid_disk = desc->raid_disk; 1455 + rdev->saved_raid_disk = desc->raid_disk; 1456 + } else if (desc->state & (1<<MD_DISK_ACTIVE)) { 1457 + /* active but not in sync implies recovery up to 1458 + * reshape position. We don't know exactly where 1459 + * that is, so set to zero for now 1460 + */ 1461 + if (mddev->minor_version >= 91) { 1462 + rdev->recovery_offset = 0; 1463 + rdev->raid_disk = desc->raid_disk; 1464 + } 1465 + } 1466 + if (desc->state & (1<<MD_DISK_WRITEMOSTLY)) 1467 + set_bit(WriteMostly, &rdev->flags); 1468 + if (desc->state & (1<<MD_DISK_FAILFAST)) 1469 + set_bit(FailFast, &rdev->flags); 1480 1470 return 0; 1481 1471 } 1482 1472 ··· 1763 1759 && rdev->new_data_offset < sb_start + (rdev->sb_size/512)) 1764 1760 return -EINVAL; 1765 1761 1766 - if (sb->level == cpu_to_le32(LEVEL_MULTIPATH)) 1767 - rdev->desc_nr = -1; 1768 - else 1769 - rdev->desc_nr = le32_to_cpu(sb->dev_number); 1762 + rdev->desc_nr = le32_to_cpu(sb->dev_number); 1770 1763 1771 1764 if (!rdev->bb_page) { 1772 1765 rdev->bb_page = alloc_page(GFP_KERNEL); ··· 1816 1815 sb->level != 0) 1817 1816 return -EINVAL; 1818 1817 1819 - /* not spare disk, or LEVEL_MULTIPATH */ 1820 - if (sb->level == cpu_to_le32(LEVEL_MULTIPATH) || 1821 - (rdev->desc_nr >= 0 && 1822 - rdev->desc_nr < le32_to_cpu(sb->max_dev) && 1823 - (le16_to_cpu(sb->dev_roles[rdev->desc_nr]) < MD_DISK_ROLE_MAX || 1824 - le16_to_cpu(sb->dev_roles[rdev->desc_nr]) == MD_DISK_ROLE_JOURNAL))) 1818 + /* not spare disk */ 1819 + if (rdev->desc_nr >= 0 && rdev->desc_nr < le32_to_cpu(sb->max_dev) && 1820 + (le16_to_cpu(sb->dev_roles[rdev->desc_nr]) < MD_DISK_ROLE_MAX || 1821 + le16_to_cpu(sb->dev_roles[rdev->desc_nr]) == MD_DISK_ROLE_JOURNAL)) 1825 1822 spare_disk = false; 1826 1823 1827 1824 if (!refdev) { ··· 1858 1859 return ret; 1859 1860 } 1860 1861 1861 - static int super_1_validate(struct mddev *mddev, struct md_rdev *rdev) 1862 + static int super_1_validate(struct mddev *mddev, struct md_rdev *freshest, struct md_rdev *rdev) 1862 1863 { 1863 1864 struct mdp_superblock_1 *sb = page_address(rdev->sb_page); 1864 1865 __u64 ev1 = le64_to_cpu(sb->events); 1866 + int role; 1865 1867 1866 1868 rdev->raid_disk = -1; 1867 1869 clear_bit(Faulty, &rdev->flags); ··· 1955 1955 } 1956 1956 } else if (mddev->pers == NULL) { 1957 1957 /* Insist of good event counter while assembling, except for 1958 - * spares (which don't need an event count) */ 1959 - ++ev1; 1958 + * spares (which don't need an event count). 1959 + * Similar to mdadm, we allow event counter difference of 1 1960 + * from the freshest device. 1961 + */ 1960 1962 if (rdev->desc_nr >= 0 && 1961 1963 rdev->desc_nr < le32_to_cpu(sb->max_dev) && 1962 1964 (le16_to_cpu(sb->dev_roles[rdev->desc_nr]) < MD_DISK_ROLE_MAX || 1963 1965 le16_to_cpu(sb->dev_roles[rdev->desc_nr]) == MD_DISK_ROLE_JOURNAL)) 1964 - if (ev1 < mddev->events) 1966 + if (ev1 + 1 < mddev->events) 1965 1967 return -EINVAL; 1966 1968 } else if (mddev->bitmap) { 1967 1969 /* If adding to array with a bitmap, then we can accept an ··· 1978 1976 /* just a hot-add of a new device, leave raid_disk at -1 */ 1979 1977 return 0; 1980 1978 } 1981 - if (mddev->level != LEVEL_MULTIPATH) { 1982 - int role; 1983 - if (rdev->desc_nr < 0 || 1984 - rdev->desc_nr >= le32_to_cpu(sb->max_dev)) { 1985 - role = MD_DISK_ROLE_SPARE; 1986 - rdev->desc_nr = -1; 1987 - } else 1988 - role = le16_to_cpu(sb->dev_roles[rdev->desc_nr]); 1989 - switch(role) { 1990 - case MD_DISK_ROLE_SPARE: /* spare */ 1991 - break; 1992 - case MD_DISK_ROLE_FAULTY: /* faulty */ 1993 - set_bit(Faulty, &rdev->flags); 1994 - break; 1995 - case MD_DISK_ROLE_JOURNAL: /* journal device */ 1996 - if (!(le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL)) { 1997 - /* journal device without journal feature */ 1998 - pr_warn("md: journal device provided without journal feature, ignoring the device\n"); 1999 - return -EINVAL; 2000 - } 2001 - set_bit(Journal, &rdev->flags); 2002 - rdev->journal_tail = le64_to_cpu(sb->journal_tail); 2003 - rdev->raid_disk = 0; 2004 - break; 2005 - default: 2006 - rdev->saved_raid_disk = role; 2007 - if ((le32_to_cpu(sb->feature_map) & 2008 - MD_FEATURE_RECOVERY_OFFSET)) { 2009 - rdev->recovery_offset = le64_to_cpu(sb->recovery_offset); 2010 - if (!(le32_to_cpu(sb->feature_map) & 2011 - MD_FEATURE_RECOVERY_BITMAP)) 2012 - rdev->saved_raid_disk = -1; 2013 - } else { 2014 - /* 2015 - * If the array is FROZEN, then the device can't 2016 - * be in_sync with rest of array. 2017 - */ 2018 - if (!test_bit(MD_RECOVERY_FROZEN, 2019 - &mddev->recovery)) 2020 - set_bit(In_sync, &rdev->flags); 2021 - } 2022 - rdev->raid_disk = role; 2023 - break; 1979 + 1980 + if (rdev->desc_nr < 0 || 1981 + rdev->desc_nr >= le32_to_cpu(sb->max_dev)) { 1982 + role = MD_DISK_ROLE_SPARE; 1983 + rdev->desc_nr = -1; 1984 + } else if (mddev->pers == NULL && freshest && ev1 < mddev->events) { 1985 + /* 1986 + * If we are assembling, and our event counter is smaller than the 1987 + * highest event counter, we cannot trust our superblock about the role. 1988 + * It could happen that our rdev was marked as Faulty, and all other 1989 + * superblocks were updated with +1 event counter. 1990 + * Then, before the next superblock update, which typically happens when 1991 + * remove_and_add_spares() removes the device from the array, there was 1992 + * a crash or reboot. 1993 + * If we allow current rdev without consulting the freshest superblock, 1994 + * we could cause data corruption. 1995 + * Note that in this case our event counter is smaller by 1 than the 1996 + * highest, otherwise, this rdev would not be allowed into array; 1997 + * both kernel and mdadm allow event counter difference of 1. 1998 + */ 1999 + struct mdp_superblock_1 *freshest_sb = page_address(freshest->sb_page); 2000 + u32 freshest_max_dev = le32_to_cpu(freshest_sb->max_dev); 2001 + 2002 + if (rdev->desc_nr >= freshest_max_dev) { 2003 + /* this is unexpected, better not proceed */ 2004 + pr_warn("md: %s: rdev[%pg]: desc_nr(%d) >= freshest(%pg)->sb->max_dev(%u)\n", 2005 + mdname(mddev), rdev->bdev, rdev->desc_nr, 2006 + freshest->bdev, freshest_max_dev); 2007 + return -EUCLEAN; 2024 2008 } 2025 - if (sb->devflags & WriteMostly1) 2026 - set_bit(WriteMostly, &rdev->flags); 2027 - if (sb->devflags & FailFast1) 2028 - set_bit(FailFast, &rdev->flags); 2029 - if (le32_to_cpu(sb->feature_map) & MD_FEATURE_REPLACEMENT) 2030 - set_bit(Replacement, &rdev->flags); 2031 - } else /* MULTIPATH are always insync */ 2032 - set_bit(In_sync, &rdev->flags); 2009 + 2010 + role = le16_to_cpu(freshest_sb->dev_roles[rdev->desc_nr]); 2011 + pr_debug("md: %s: rdev[%pg]: role=%d(0x%x) according to freshest %pg\n", 2012 + mdname(mddev), rdev->bdev, role, role, freshest->bdev); 2013 + } else { 2014 + role = le16_to_cpu(sb->dev_roles[rdev->desc_nr]); 2015 + } 2016 + switch (role) { 2017 + case MD_DISK_ROLE_SPARE: /* spare */ 2018 + break; 2019 + case MD_DISK_ROLE_FAULTY: /* faulty */ 2020 + set_bit(Faulty, &rdev->flags); 2021 + break; 2022 + case MD_DISK_ROLE_JOURNAL: /* journal device */ 2023 + if (!(le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL)) { 2024 + /* journal device without journal feature */ 2025 + pr_warn("md: journal device provided without journal feature, ignoring the device\n"); 2026 + return -EINVAL; 2027 + } 2028 + set_bit(Journal, &rdev->flags); 2029 + rdev->journal_tail = le64_to_cpu(sb->journal_tail); 2030 + rdev->raid_disk = 0; 2031 + break; 2032 + default: 2033 + rdev->saved_raid_disk = role; 2034 + if ((le32_to_cpu(sb->feature_map) & 2035 + MD_FEATURE_RECOVERY_OFFSET)) { 2036 + rdev->recovery_offset = le64_to_cpu(sb->recovery_offset); 2037 + if (!(le32_to_cpu(sb->feature_map) & 2038 + MD_FEATURE_RECOVERY_BITMAP)) 2039 + rdev->saved_raid_disk = -1; 2040 + } else { 2041 + /* 2042 + * If the array is FROZEN, then the device can't 2043 + * be in_sync with rest of array. 2044 + */ 2045 + if (!test_bit(MD_RECOVERY_FROZEN, 2046 + &mddev->recovery)) 2047 + set_bit(In_sync, &rdev->flags); 2048 + } 2049 + rdev->raid_disk = role; 2050 + break; 2051 + } 2052 + if (sb->devflags & WriteMostly1) 2053 + set_bit(WriteMostly, &rdev->flags); 2054 + if (sb->devflags & FailFast1) 2055 + set_bit(FailFast, &rdev->flags); 2056 + if (le32_to_cpu(sb->feature_map) & MD_FEATURE_REPLACEMENT) 2057 + set_bit(Replacement, &rdev->flags); 2033 2058 2034 2059 return 0; 2035 2060 } ··· 2874 2845 } else 2875 2846 pr_debug("md: %pg (skipping faulty)\n", 2876 2847 rdev->bdev); 2877 - 2878 - if (mddev->level == LEVEL_MULTIPATH) 2879 - /* only need to write one superblock... */ 2880 - break; 2881 2848 } 2882 2849 if (md_super_wait(mddev) < 0) 2883 2850 goto rewrite; ··· 2915 2890 * and should be added immediately. 2916 2891 */ 2917 2892 super_types[mddev->major_version]. 2918 - validate_super(mddev, rdev); 2893 + validate_super(mddev, NULL/*freshest*/, rdev); 2919 2894 err = mddev->pers->hot_add_disk(mddev, rdev); 2920 2895 if (err) { 2921 2896 md_kick_rdev_from_array(rdev); ··· 3852 3827 } 3853 3828 3854 3829 super_types[mddev->major_version]. 3855 - validate_super(mddev, freshest); 3830 + validate_super(mddev, NULL/*freshest*/, freshest); 3856 3831 3857 3832 i = 0; 3858 3833 rdev_for_each_safe(rdev, tmp, mddev) { ··· 3867 3842 } 3868 3843 if (rdev != freshest) { 3869 3844 if (super_types[mddev->major_version]. 3870 - validate_super(mddev, rdev)) { 3845 + validate_super(mddev, freshest, rdev)) { 3871 3846 pr_warn("md: kicking non-fresh %pg from array!\n", 3872 3847 rdev->bdev); 3873 3848 md_kick_rdev_from_array(rdev); 3874 3849 continue; 3875 3850 } 3876 3851 } 3877 - if (mddev->level == LEVEL_MULTIPATH) { 3878 - rdev->desc_nr = i++; 3879 - rdev->raid_disk = rdev->desc_nr; 3880 - set_bit(In_sync, &rdev->flags); 3881 - } else if (rdev->raid_disk >= 3882 - (mddev->raid_disks - min(0, mddev->delta_disks)) && 3883 - !test_bit(Journal, &rdev->flags)) { 3852 + if (rdev->raid_disk >= (mddev->raid_disks - min(0, mddev->delta_disks)) && 3853 + !test_bit(Journal, &rdev->flags)) { 3884 3854 rdev->raid_disk = -1; 3885 3855 clear_bit(In_sync, &rdev->flags); 3886 3856 } ··· 6853 6833 rdev->saved_raid_disk = rdev->raid_disk; 6854 6834 } else 6855 6835 super_types[mddev->major_version]. 6856 - validate_super(mddev, rdev); 6836 + validate_super(mddev, NULL/*freshest*/, rdev); 6857 6837 if ((info->state & (1<<MD_DISK_SYNC)) && 6858 6838 rdev->raid_disk != info->raid_disk) { 6859 6839 /* This was a hot-add request, but events doesn't ··· 8096 8076 return; 8097 8077 mddev->pers->error_handler(mddev, rdev); 8098 8078 8099 - if (mddev->pers->level == 0 || mddev->pers->level == LEVEL_LINEAR) 8079 + if (mddev->pers->level == 0) 8100 8080 return; 8101 8081 8102 8082 if (mddev->degraded && !test_bit(MD_BROKEN, &mddev->flags)) ··· 9260 9240 struct md_rdev *rdev; 9261 9241 int spares = 0; 9262 9242 int removed = 0; 9263 - bool remove_some = false; 9264 9243 9265 9244 if (this && test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 9266 9245 /* Mustn't remove devices when resync thread is running */ 9267 9246 return 0; 9268 9247 9269 9248 rdev_for_each(rdev, mddev) { 9270 - if ((this == NULL || rdev == this) && 9271 - rdev->raid_disk >= 0 && 9272 - !test_bit(Blocked, &rdev->flags) && 9273 - test_bit(Faulty, &rdev->flags) && 9274 - atomic_read(&rdev->nr_pending)==0) { 9275 - /* Faulty non-Blocked devices with nr_pending == 0 9276 - * never get nr_pending incremented, 9277 - * never get Faulty cleared, and never get Blocked set. 9278 - * So we can synchronize_rcu now rather than once per device 9279 - */ 9280 - remove_some = true; 9281 - set_bit(RemoveSynchronized, &rdev->flags); 9249 + if ((this == NULL || rdev == this) && rdev_removeable(rdev) && 9250 + !mddev->pers->hot_remove_disk(mddev, rdev)) { 9251 + sysfs_unlink_rdev(mddev, rdev); 9252 + rdev->saved_raid_disk = rdev->raid_disk; 9253 + rdev->raid_disk = -1; 9254 + removed++; 9282 9255 } 9283 - } 9284 - 9285 - if (remove_some) 9286 - synchronize_rcu(); 9287 - rdev_for_each(rdev, mddev) { 9288 - if ((this == NULL || rdev == this) && 9289 - (test_bit(RemoveSynchronized, &rdev->flags) || 9290 - rdev_removeable(rdev))) { 9291 - if (mddev->pers->hot_remove_disk( 9292 - mddev, rdev) == 0) { 9293 - sysfs_unlink_rdev(mddev, rdev); 9294 - rdev->saved_raid_disk = rdev->raid_disk; 9295 - rdev->raid_disk = -1; 9296 - removed++; 9297 - } 9298 - } 9299 - if (remove_some && test_bit(RemoveSynchronized, &rdev->flags)) 9300 - clear_bit(RemoveSynchronized, &rdev->flags); 9301 9256 } 9302 9257 9303 9258 if (removed && mddev->kobj.sd)

-5

drivers/md/md.h

··· 190 190 * than other devices in the array 191 191 */ 192 192 ClusterRemove, 193 - RemoveSynchronized, /* synchronize_rcu() was called after 194 - * this device was known to be faulty, 195 - * so it is safe to remove without 196 - * another synchronize_rcu() call. 197 - */ 198 193 ExternalBbl, /* External metadata provides bad 199 194 * block management for a disk 200 195 */

+54

drivers/md/raid1-10.c

··· 173 173 else 174 174 md_bitmap_unplug(bitmap); 175 175 } 176 + 177 + /* 178 + * Used by fix_read_error() to decay the per rdev read_errors. 179 + * We halve the read error count for every hour that has elapsed 180 + * since the last recorded read error. 181 + */ 182 + static inline void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev) 183 + { 184 + long cur_time_mon; 185 + unsigned long hours_since_last; 186 + unsigned int read_errors = atomic_read(&rdev->read_errors); 187 + 188 + cur_time_mon = ktime_get_seconds(); 189 + 190 + if (rdev->last_read_error == 0) { 191 + /* first time we've seen a read error */ 192 + rdev->last_read_error = cur_time_mon; 193 + return; 194 + } 195 + 196 + hours_since_last = (long)(cur_time_mon - 197 + rdev->last_read_error) / 3600; 198 + 199 + rdev->last_read_error = cur_time_mon; 200 + 201 + /* 202 + * if hours_since_last is > the number of bits in read_errors 203 + * just set read errors to 0. We do this to avoid 204 + * overflowing the shift of read_errors by hours_since_last. 205 + */ 206 + if (hours_since_last >= 8 * sizeof(read_errors)) 207 + atomic_set(&rdev->read_errors, 0); 208 + else 209 + atomic_set(&rdev->read_errors, read_errors >> hours_since_last); 210 + } 211 + 212 + static inline bool exceed_read_errors(struct mddev *mddev, struct md_rdev *rdev) 213 + { 214 + int max_read_errors = atomic_read(&mddev->max_corr_read_errors); 215 + int read_errors; 216 + 217 + check_decay_read_errors(mddev, rdev); 218 + read_errors = atomic_inc_return(&rdev->read_errors); 219 + if (read_errors > max_read_errors) { 220 + pr_notice("md/"RAID_1_10_NAME":%s: %pg: Raid device exceeded read_error threshold [cur %d:max %d]\n", 221 + mdname(mddev), rdev->bdev, read_errors, max_read_errors); 222 + pr_notice("md/"RAID_1_10_NAME":%s: %pg: Failing raid device\n", 223 + mdname(mddev), rdev->bdev); 224 + md_error(mddev, rdev); 225 + return true; 226 + } 227 + 228 + return false; 229 + }

+36 -55

drivers/md/raid1.c

··· 49 49 #define raid1_log(md, fmt, args...) \ 50 50 do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid1 " fmt, ##args); } while (0) 51 51 52 + #define RAID_1_10_NAME "raid1" 52 53 #include "raid1-10.c" 53 54 54 55 #define START(node) ((node)->start) ··· 610 609 int choose_first; 611 610 int choose_next_idle; 612 611 613 - rcu_read_lock(); 614 612 /* 615 613 * Check if we can balance. We can balance on the whole 616 614 * device if no resync is going on, or below the resync window. ··· 642 642 unsigned int pending; 643 643 bool nonrot; 644 644 645 - rdev = rcu_dereference(conf->mirrors[disk].rdev); 645 + rdev = conf->mirrors[disk].rdev; 646 646 if (r1_bio->bios[disk] == IO_BLOCKED 647 647 || rdev == NULL 648 648 || test_bit(Faulty, &rdev->flags)) ··· 773 773 } 774 774 775 775 if (best_disk >= 0) { 776 - rdev = rcu_dereference(conf->mirrors[best_disk].rdev); 776 + rdev = conf->mirrors[best_disk].rdev; 777 777 if (!rdev) 778 778 goto retry; 779 779 atomic_inc(&rdev->nr_pending); ··· 784 784 785 785 conf->mirrors[best_disk].next_seq_sect = this_sector + sectors; 786 786 } 787 - rcu_read_unlock(); 788 787 *max_sectors = sectors; 789 788 790 789 return best_disk; ··· 1125 1126 1126 1127 behind_bio = bio_alloc_bioset(NULL, vcnt, 0, GFP_NOIO, 1127 1128 &r1_bio->mddev->bio_set); 1128 - if (!behind_bio) 1129 - return; 1130 1129 1131 1130 /* discard op, we don't support writezero/writesame yet */ 1132 1131 if (!bio_has_data(bio)) { ··· 1232 1235 1233 1236 if (r1bio_existed) { 1234 1237 /* Need to get the block device name carefully */ 1235 - struct md_rdev *rdev; 1236 - rcu_read_lock(); 1237 - rdev = rcu_dereference(conf->mirrors[r1_bio->read_disk].rdev); 1238 + struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev; 1239 + 1238 1240 if (rdev) 1239 1241 snprintf(b, sizeof(b), "%pg", rdev->bdev); 1240 1242 else 1241 1243 strcpy(b, "???"); 1242 - rcu_read_unlock(); 1243 1244 } 1244 1245 1245 1246 /* ··· 1391 1396 1392 1397 disks = conf->raid_disks * 2; 1393 1398 blocked_rdev = NULL; 1394 - rcu_read_lock(); 1395 1399 max_sectors = r1_bio->sectors; 1396 1400 for (i = 0; i < disks; i++) { 1397 - struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 1401 + struct md_rdev *rdev = conf->mirrors[i].rdev; 1398 1402 1399 1403 /* 1400 1404 * The write-behind io is only attempted on drives marked as ··· 1459 1465 } 1460 1466 r1_bio->bios[i] = bio; 1461 1467 } 1462 - rcu_read_unlock(); 1463 1468 1464 1469 if (unlikely(blocked_rdev)) { 1465 1470 /* Wait for this device to become unblocked */ ··· 1610 1617 struct r1conf *conf = mddev->private; 1611 1618 int i; 1612 1619 1620 + lockdep_assert_held(&mddev->lock); 1621 + 1613 1622 seq_printf(seq, " [%d/%d] [", conf->raid_disks, 1614 1623 conf->raid_disks - mddev->degraded); 1615 - rcu_read_lock(); 1616 1624 for (i = 0; i < conf->raid_disks; i++) { 1617 - struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 1625 + struct md_rdev *rdev = READ_ONCE(conf->mirrors[i].rdev); 1626 + 1618 1627 seq_printf(seq, "%s", 1619 1628 rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 1620 1629 } 1621 - rcu_read_unlock(); 1622 1630 seq_printf(seq, "]"); 1623 1631 } 1624 1632 ··· 1685 1691 pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded, 1686 1692 conf->raid_disks); 1687 1693 1688 - rcu_read_lock(); 1694 + lockdep_assert_held(&conf->mddev->reconfig_mutex); 1689 1695 for (i = 0; i < conf->raid_disks; i++) { 1690 - struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 1696 + struct md_rdev *rdev = conf->mirrors[i].rdev; 1691 1697 if (rdev) 1692 1698 pr_debug(" disk %d, wo:%d, o:%d, dev:%pg\n", 1693 1699 i, !test_bit(In_sync, &rdev->flags), 1694 1700 !test_bit(Faulty, &rdev->flags), 1695 1701 rdev->bdev); 1696 1702 } 1697 - rcu_read_unlock(); 1698 1703 } 1699 1704 1700 1705 static void close_sync(struct r1conf *conf) ··· 1803 1810 */ 1804 1811 if (rdev->saved_raid_disk < 0) 1805 1812 conf->fullsync = 1; 1806 - rcu_assign_pointer(p->rdev, rdev); 1813 + WRITE_ONCE(p->rdev, rdev); 1807 1814 break; 1808 1815 } 1809 1816 if (test_bit(WantReplacement, &p->rdev->flags) && ··· 1819 1826 rdev->raid_disk = repl_slot; 1820 1827 err = 0; 1821 1828 conf->fullsync = 1; 1822 - rcu_assign_pointer(p[conf->raid_disks].rdev, rdev); 1829 + WRITE_ONCE(p[conf->raid_disks].rdev, rdev); 1823 1830 } 1824 1831 1825 1832 print_conf(conf); ··· 1855 1862 err = -EBUSY; 1856 1863 goto abort; 1857 1864 } 1858 - p->rdev = NULL; 1859 - if (!test_bit(RemoveSynchronized, &rdev->flags)) { 1860 - synchronize_rcu(); 1861 - if (atomic_read(&rdev->nr_pending)) { 1862 - /* lost the race, try later */ 1863 - err = -EBUSY; 1864 - p->rdev = rdev; 1865 - goto abort; 1866 - } 1867 - } 1865 + WRITE_ONCE(p->rdev, NULL); 1868 1866 if (conf->mirrors[conf->raid_disks + number].rdev) { 1869 1867 /* We just removed a device that is being replaced. 1870 1868 * Move down the replacement. We drain all IO before ··· 1876 1892 goto abort; 1877 1893 } 1878 1894 clear_bit(Replacement, &repl->flags); 1879 - p->rdev = repl; 1895 + WRITE_ONCE(p->rdev, repl); 1880 1896 conf->mirrors[conf->raid_disks + number].rdev = NULL; 1881 1897 unfreeze_array(conf); 1882 1898 } ··· 2256 2272 * 3. Performs writes following reads for array synchronising. 2257 2273 */ 2258 2274 2259 - static void fix_read_error(struct r1conf *conf, int read_disk, 2260 - sector_t sect, int sectors) 2275 + static void fix_read_error(struct r1conf *conf, struct r1bio *r1_bio) 2261 2276 { 2277 + sector_t sect = r1_bio->sector; 2278 + int sectors = r1_bio->sectors; 2279 + int read_disk = r1_bio->read_disk; 2262 2280 struct mddev *mddev = conf->mddev; 2281 + struct md_rdev *rdev = rcu_dereference(conf->mirrors[read_disk].rdev); 2282 + 2283 + if (exceed_read_errors(mddev, rdev)) { 2284 + r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED; 2285 + return; 2286 + } 2287 + 2263 2288 while(sectors) { 2264 2289 int s = sectors; 2265 2290 int d = read_disk; 2266 2291 int success = 0; 2267 2292 int start; 2268 - struct md_rdev *rdev; 2269 2293 2270 2294 if (s > (PAGE_SIZE>>9)) 2271 2295 s = PAGE_SIZE >> 9; ··· 2282 2290 sector_t first_bad; 2283 2291 int bad_sectors; 2284 2292 2285 - rcu_read_lock(); 2286 - rdev = rcu_dereference(conf->mirrors[d].rdev); 2293 + rdev = conf->mirrors[d].rdev; 2287 2294 if (rdev && 2288 2295 (test_bit(In_sync, &rdev->flags) || 2289 2296 (!test_bit(Faulty, &rdev->flags) && ··· 2290 2299 is_badblock(rdev, sect, s, 2291 2300 &first_bad, &bad_sectors) == 0) { 2292 2301 atomic_inc(&rdev->nr_pending); 2293 - rcu_read_unlock(); 2294 2302 if (sync_page_io(rdev, sect, s<<9, 2295 2303 conf->tmppage, REQ_OP_READ, false)) 2296 2304 success = 1; 2297 2305 rdev_dec_pending(rdev, mddev); 2298 2306 if (success) 2299 2307 break; 2300 - } else 2301 - rcu_read_unlock(); 2308 + } 2309 + 2302 2310 d++; 2303 2311 if (d == conf->raid_disks * 2) 2304 2312 d = 0; ··· 2316 2326 if (d==0) 2317 2327 d = conf->raid_disks * 2; 2318 2328 d--; 2319 - rcu_read_lock(); 2320 - rdev = rcu_dereference(conf->mirrors[d].rdev); 2329 + rdev = conf->mirrors[d].rdev; 2321 2330 if (rdev && 2322 2331 !test_bit(Faulty, &rdev->flags)) { 2323 2332 atomic_inc(&rdev->nr_pending); 2324 - rcu_read_unlock(); 2325 2333 r1_sync_page_io(rdev, sect, s, 2326 2334 conf->tmppage, WRITE); 2327 2335 rdev_dec_pending(rdev, mddev); 2328 - } else 2329 - rcu_read_unlock(); 2336 + } 2330 2337 } 2331 2338 d = start; 2332 2339 while (d != read_disk) { 2333 2340 if (d==0) 2334 2341 d = conf->raid_disks * 2; 2335 2342 d--; 2336 - rcu_read_lock(); 2337 - rdev = rcu_dereference(conf->mirrors[d].rdev); 2343 + rdev = conf->mirrors[d].rdev; 2338 2344 if (rdev && 2339 2345 !test_bit(Faulty, &rdev->flags)) { 2340 2346 atomic_inc(&rdev->nr_pending); 2341 - rcu_read_unlock(); 2342 2347 if (r1_sync_page_io(rdev, sect, s, 2343 2348 conf->tmppage, READ)) { 2344 2349 atomic_add(s, &rdev->corrected_errors); ··· 2344 2359 rdev->bdev); 2345 2360 } 2346 2361 rdev_dec_pending(rdev, mddev); 2347 - } else 2348 - rcu_read_unlock(); 2362 + } 2349 2363 } 2350 2364 sectors -= s; 2351 2365 sect += s; ··· 2514 2530 if (mddev->ro == 0 2515 2531 && !test_bit(FailFast, &rdev->flags)) { 2516 2532 freeze_array(conf, 1); 2517 - fix_read_error(conf, r1_bio->read_disk, 2518 - r1_bio->sector, r1_bio->sectors); 2533 + fix_read_error(conf, r1_bio); 2519 2534 unfreeze_array(conf); 2520 2535 } else if (mddev->ro == 0 && test_bit(FailFast, &rdev->flags)) { 2521 2536 md_error(mddev, rdev); ··· 2724 2741 2725 2742 r1_bio = raid1_alloc_init_r1buf(conf); 2726 2743 2727 - rcu_read_lock(); 2728 2744 /* 2729 2745 * If we get a correctably read error during resync or recovery, 2730 2746 * we might want to read from a different device. So we ··· 2744 2762 struct md_rdev *rdev; 2745 2763 bio = r1_bio->bios[i]; 2746 2764 2747 - rdev = rcu_dereference(conf->mirrors[i].rdev); 2765 + rdev = conf->mirrors[i].rdev; 2748 2766 if (rdev == NULL || 2749 2767 test_bit(Faulty, &rdev->flags)) { 2750 2768 if (i < conf->raid_disks) ··· 2802 2820 bio->bi_opf |= MD_FAILFAST; 2803 2821 } 2804 2822 } 2805 - rcu_read_unlock(); 2806 2823 if (disk < 0) 2807 2824 disk = wonly; 2808 2825 r1_bio->read_disk = disk;

+61 -210

drivers/md/raid10.c

··· 19 19 #include <linux/raid/md_p.h> 20 20 #include <trace/events/block.h> 21 21 #include "md.h" 22 + 23 + #define RAID_1_10_NAME "raid10" 22 24 #include "raid10.h" 23 25 #include "raid0.h" 24 26 #include "md-bitmap.h" ··· 745 743 struct geom *geo = &conf->geo; 746 744 747 745 raid10_find_phys(conf, r10_bio); 748 - rcu_read_lock(); 749 746 best_dist_slot = -1; 750 747 min_pending = UINT_MAX; 751 748 best_dist_rdev = NULL; ··· 776 775 if (r10_bio->devs[slot].bio == IO_BLOCKED) 777 776 continue; 778 777 disk = r10_bio->devs[slot].devnum; 779 - rdev = rcu_dereference(conf->mirrors[disk].replacement); 778 + rdev = conf->mirrors[disk].replacement; 780 779 if (rdev == NULL || test_bit(Faulty, &rdev->flags) || 781 780 r10_bio->devs[slot].addr + sectors > 782 - rdev->recovery_offset) { 783 - /* 784 - * Read replacement first to prevent reading both rdev 785 - * and replacement as NULL during replacement replace 786 - * rdev. 787 - */ 788 - smp_mb(); 789 - rdev = rcu_dereference(conf->mirrors[disk].rdev); 790 - } 781 + rdev->recovery_offset) 782 + rdev = conf->mirrors[disk].rdev; 791 783 if (rdev == NULL || 792 784 test_bit(Faulty, &rdev->flags)) 793 785 continue; ··· 870 876 r10_bio->read_slot = slot; 871 877 } else 872 878 rdev = NULL; 873 - rcu_read_unlock(); 874 879 *max_sectors = best_good_sectors; 875 880 876 881 return rdev; ··· 1191 1198 */ 1192 1199 gfp = GFP_NOIO | __GFP_HIGH; 1193 1200 1194 - rcu_read_lock(); 1195 1201 disk = r10_bio->devs[slot].devnum; 1196 - err_rdev = rcu_dereference(conf->mirrors[disk].rdev); 1202 + err_rdev = conf->mirrors[disk].rdev; 1197 1203 if (err_rdev) 1198 1204 snprintf(b, sizeof(b), "%pg", err_rdev->bdev); 1199 1205 else { ··· 1200 1208 /* This never gets dereferenced */ 1201 1209 err_rdev = r10_bio->devs[slot].rdev; 1202 1210 } 1203 - rcu_read_unlock(); 1204 1211 } 1205 1212 1206 1213 if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors)) ··· 1270 1279 int devnum = r10_bio->devs[n_copy].devnum; 1271 1280 struct bio *mbio; 1272 1281 1273 - if (replacement) { 1274 - rdev = conf->mirrors[devnum].replacement; 1275 - if (rdev == NULL) { 1276 - /* Replacement just got moved to main 'rdev' */ 1277 - smp_mb(); 1278 - rdev = conf->mirrors[devnum].rdev; 1279 - } 1280 - } else 1281 - rdev = conf->mirrors[devnum].rdev; 1282 + rdev = replacement ? conf->mirrors[devnum].replacement : 1283 + conf->mirrors[devnum].rdev; 1282 1284 1283 1285 mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO, &mddev->bio_set); 1284 1286 if (replacement) ··· 1305 1321 } 1306 1322 } 1307 1323 1308 - static struct md_rdev *dereference_rdev_and_rrdev(struct raid10_info *mirror, 1309 - struct md_rdev **prrdev) 1310 - { 1311 - struct md_rdev *rdev, *rrdev; 1312 - 1313 - rrdev = rcu_dereference(mirror->replacement); 1314 - /* 1315 - * Read replacement first to prevent reading both rdev and 1316 - * replacement as NULL during replacement replace rdev. 1317 - */ 1318 - smp_mb(); 1319 - rdev = rcu_dereference(mirror->rdev); 1320 - if (rdev == rrdev) 1321 - rrdev = NULL; 1322 - 1323 - *prrdev = rrdev; 1324 - return rdev; 1325 - } 1326 - 1327 1324 static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio) 1328 1325 { 1329 1326 int i; ··· 1313 1348 1314 1349 retry_wait: 1315 1350 blocked_rdev = NULL; 1316 - rcu_read_lock(); 1317 1351 for (i = 0; i < conf->copies; i++) { 1318 1352 struct md_rdev *rdev, *rrdev; 1319 1353 1320 - rdev = dereference_rdev_and_rrdev(&conf->mirrors[i], &rrdev); 1354 + rdev = conf->mirrors[i].rdev; 1355 + rrdev = conf->mirrors[i].replacement; 1321 1356 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 1322 1357 atomic_inc(&rdev->nr_pending); 1323 1358 blocked_rdev = rdev; ··· 1356 1391 } 1357 1392 } 1358 1393 } 1359 - rcu_read_unlock(); 1360 1394 1361 1395 if (unlikely(blocked_rdev)) { 1362 1396 /* Have to wait for this device to get unblocked, then retry */ ··· 1438 1474 1439 1475 wait_blocked_dev(mddev, r10_bio); 1440 1476 1441 - rcu_read_lock(); 1442 1477 max_sectors = r10_bio->sectors; 1443 1478 1444 1479 for (i = 0; i < conf->copies; i++) { 1445 1480 int d = r10_bio->devs[i].devnum; 1446 1481 struct md_rdev *rdev, *rrdev; 1447 1482 1448 - rdev = dereference_rdev_and_rrdev(&conf->mirrors[d], &rrdev); 1483 + rdev = conf->mirrors[d].rdev; 1484 + rrdev = conf->mirrors[d].replacement; 1449 1485 if (rdev && (test_bit(Faulty, &rdev->flags))) 1450 1486 rdev = NULL; 1451 1487 if (rrdev && (test_bit(Faulty, &rrdev->flags))) ··· 1499 1535 atomic_inc(&rrdev->nr_pending); 1500 1536 } 1501 1537 } 1502 - rcu_read_unlock(); 1503 1538 1504 1539 if (max_sectors < r10_bio->sectors) 1505 1540 r10_bio->sectors = max_sectors; ··· 1588 1625 set_bit(R10BIO_Uptodate, &r10_bio->state); 1589 1626 1590 1627 dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 1591 - if (repl) 1592 - rdev = conf->mirrors[dev].replacement; 1593 - if (!rdev) { 1594 - /* 1595 - * raid10_remove_disk uses smp_mb to make sure rdev is set to 1596 - * replacement before setting replacement to NULL. It can read 1597 - * rdev first without barrier protect even replacement is NULL 1598 - */ 1599 - smp_rmb(); 1600 - rdev = conf->mirrors[dev].rdev; 1601 - } 1628 + rdev = repl ? conf->mirrors[dev].replacement : 1629 + conf->mirrors[dev].rdev; 1602 1630 1603 1631 raid_end_discard_bio(r10_bio); 1604 1632 rdev_dec_pending(rdev, conf->mddev); ··· 1739 1785 * inc refcount on their rdev. Record them by setting 1740 1786 * bios[x] to bio 1741 1787 */ 1742 - rcu_read_lock(); 1743 1788 for (disk = 0; disk < geo->raid_disks; disk++) { 1744 1789 struct md_rdev *rdev, *rrdev; 1745 1790 1746 - rdev = dereference_rdev_and_rrdev(&conf->mirrors[disk], &rrdev); 1791 + rdev = conf->mirrors[disk].rdev; 1792 + rrdev = conf->mirrors[disk].replacement; 1747 1793 r10_bio->devs[disk].bio = NULL; 1748 1794 r10_bio->devs[disk].repl_bio = NULL; 1749 1795 ··· 1763 1809 atomic_inc(&rrdev->nr_pending); 1764 1810 } 1765 1811 } 1766 - rcu_read_unlock(); 1767 1812 1768 1813 atomic_set(&r10_bio->remaining, 1); 1769 1814 for (disk = 0; disk < geo->raid_disks; disk++) { ··· 1892 1939 struct r10conf *conf = mddev->private; 1893 1940 int i; 1894 1941 1942 + lockdep_assert_held(&mddev->lock); 1943 + 1895 1944 if (conf->geo.near_copies < conf->geo.raid_disks) 1896 1945 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2); 1897 1946 if (conf->geo.near_copies > 1) ··· 1908 1953 } 1909 1954 seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks, 1910 1955 conf->geo.raid_disks - mddev->degraded); 1911 - rcu_read_lock(); 1912 1956 for (i = 0; i < conf->geo.raid_disks; i++) { 1913 - struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 1957 + struct md_rdev *rdev = READ_ONCE(conf->mirrors[i].rdev); 1958 + 1914 1959 seq_printf(seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 1915 1960 } 1916 - rcu_read_unlock(); 1917 1961 seq_printf(seq, "]"); 1918 1962 } 1919 1963 ··· 1934 1980 ncopies = conf->geo.near_copies; 1935 1981 } 1936 1982 1937 - rcu_read_lock(); 1938 1983 do { 1939 1984 int n = conf->copies; 1940 1985 int cnt = 0; ··· 1941 1988 while (n--) { 1942 1989 struct md_rdev *rdev; 1943 1990 if (this != ignore && 1944 - (rdev = rcu_dereference(conf->mirrors[this].rdev)) && 1991 + (rdev = conf->mirrors[this].rdev) && 1945 1992 test_bit(In_sync, &rdev->flags)) 1946 1993 cnt++; 1947 1994 this = (this+1) % disks; ··· 1952 1999 } while (first != 0); 1953 2000 has_enough = 1; 1954 2001 out: 1955 - rcu_read_unlock(); 1956 2002 return has_enough; 1957 2003 } 1958 2004 ··· 2024 2072 pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded, 2025 2073 conf->geo.raid_disks); 2026 2074 2027 - /* This is only called with ->reconfix_mutex held, so 2028 - * rcu protection of rdev is not needed */ 2075 + lockdep_assert_held(&conf->mddev->reconfig_mutex); 2029 2076 for (i = 0; i < conf->geo.raid_disks; i++) { 2030 2077 rdev = conf->mirrors[i].rdev; 2031 2078 if (rdev) ··· 2141 2190 err = 0; 2142 2191 if (rdev->saved_raid_disk != mirror) 2143 2192 conf->fullsync = 1; 2144 - rcu_assign_pointer(p->rdev, rdev); 2193 + WRITE_ONCE(p->rdev, rdev); 2145 2194 break; 2146 2195 } 2147 2196 ··· 2155 2204 disk_stack_limits(mddev->gendisk, rdev->bdev, 2156 2205 rdev->data_offset << 9); 2157 2206 conf->fullsync = 1; 2158 - rcu_assign_pointer(p->replacement, rdev); 2207 + WRITE_ONCE(p->replacement, rdev); 2159 2208 } 2160 2209 2161 2210 print_conf(conf); ··· 2197 2246 err = -EBUSY; 2198 2247 goto abort; 2199 2248 } 2200 - *rdevp = NULL; 2201 - if (!test_bit(RemoveSynchronized, &rdev->flags)) { 2202 - synchronize_rcu(); 2203 - if (atomic_read(&rdev->nr_pending)) { 2204 - /* lost the race, try later */ 2205 - err = -EBUSY; 2206 - *rdevp = rdev; 2207 - goto abort; 2208 - } 2209 - } 2249 + WRITE_ONCE(*rdevp, NULL); 2210 2250 if (p->replacement) { 2211 2251 /* We must have just cleared 'rdev' */ 2212 - p->rdev = p->replacement; 2252 + WRITE_ONCE(p->rdev, p->replacement); 2213 2253 clear_bit(Replacement, &p->replacement->flags); 2214 - smp_mb(); /* Make sure other CPUs may see both as identical 2215 - * but will never see neither -- if they are careful. 2216 - */ 2217 - p->replacement = NULL; 2254 + WRITE_ONCE(p->replacement, NULL); 2218 2255 } 2219 2256 2220 2257 clear_bit(WantReplacement, &rdev->flags); ··· 2594 2655 } 2595 2656 } 2596 2657 2597 - /* 2598 - * Used by fix_read_error() to decay the per rdev read_errors. 2599 - * We halve the read error count for every hour that has elapsed 2600 - * since the last recorded read error. 2601 - * 2602 - */ 2603 - static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev) 2604 - { 2605 - long cur_time_mon; 2606 - unsigned long hours_since_last; 2607 - unsigned int read_errors = atomic_read(&rdev->read_errors); 2608 - 2609 - cur_time_mon = ktime_get_seconds(); 2610 - 2611 - if (rdev->last_read_error == 0) { 2612 - /* first time we've seen a read error */ 2613 - rdev->last_read_error = cur_time_mon; 2614 - return; 2615 - } 2616 - 2617 - hours_since_last = (long)(cur_time_mon - 2618 - rdev->last_read_error) / 3600; 2619 - 2620 - rdev->last_read_error = cur_time_mon; 2621 - 2622 - /* 2623 - * if hours_since_last is > the number of bits in read_errors 2624 - * just set read errors to 0. We do this to avoid 2625 - * overflowing the shift of read_errors by hours_since_last. 2626 - */ 2627 - if (hours_since_last >= 8 * sizeof(read_errors)) 2628 - atomic_set(&rdev->read_errors, 0); 2629 - else 2630 - atomic_set(&rdev->read_errors, read_errors >> hours_since_last); 2631 - } 2632 - 2633 2658 static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector, 2634 2659 int sectors, struct page *page, enum req_op op) 2635 2660 { ··· 2631 2728 int sect = 0; /* Offset from r10_bio->sector */ 2632 2729 int sectors = r10_bio->sectors, slot = r10_bio->read_slot; 2633 2730 struct md_rdev *rdev; 2634 - int max_read_errors = atomic_read(&mddev->max_corr_read_errors); 2635 2731 int d = r10_bio->devs[slot].devnum; 2636 2732 2637 2733 /* still own a reference to this rdev, so it cannot ··· 2643 2741 more fix_read_error() attempts */ 2644 2742 return; 2645 2743 2646 - check_decay_read_errors(mddev, rdev); 2647 - atomic_inc(&rdev->read_errors); 2648 - if (atomic_read(&rdev->read_errors) > max_read_errors) { 2649 - pr_notice("md/raid10:%s: %pg: Raid device exceeded read_error threshold [cur %d:max %d]\n", 2650 - mdname(mddev), rdev->bdev, 2651 - atomic_read(&rdev->read_errors), max_read_errors); 2652 - pr_notice("md/raid10:%s: %pg: Failing raid device\n", 2653 - mdname(mddev), rdev->bdev); 2654 - md_error(mddev, rdev); 2744 + if (exceed_read_errors(mddev, rdev)) { 2655 2745 r10_bio->devs[slot].bio = IO_BLOCKED; 2656 2746 return; 2657 2747 } ··· 2657 2763 if (s > (PAGE_SIZE>>9)) 2658 2764 s = PAGE_SIZE >> 9; 2659 2765 2660 - rcu_read_lock(); 2661 2766 do { 2662 2767 sector_t first_bad; 2663 2768 int bad_sectors; 2664 2769 2665 2770 d = r10_bio->devs[sl].devnum; 2666 - rdev = rcu_dereference(conf->mirrors[d].rdev); 2771 + rdev = conf->mirrors[d].rdev; 2667 2772 if (rdev && 2668 2773 test_bit(In_sync, &rdev->flags) && 2669 2774 !test_bit(Faulty, &rdev->flags) && 2670 2775 is_badblock(rdev, r10_bio->devs[sl].addr + sect, s, 2671 2776 &first_bad, &bad_sectors) == 0) { 2672 2777 atomic_inc(&rdev->nr_pending); 2673 - rcu_read_unlock(); 2674 2778 success = sync_page_io(rdev, 2675 2779 r10_bio->devs[sl].addr + 2676 2780 sect, ··· 2676 2784 conf->tmppage, 2677 2785 REQ_OP_READ, false); 2678 2786 rdev_dec_pending(rdev, mddev); 2679 - rcu_read_lock(); 2680 2787 if (success) 2681 2788 break; 2682 2789 } ··· 2683 2792 if (sl == conf->copies) 2684 2793 sl = 0; 2685 2794 } while (sl != slot); 2686 - rcu_read_unlock(); 2687 2795 2688 2796 if (!success) { 2689 2797 /* Cannot read from anywhere, just mark the block ··· 2706 2816 2707 2817 start = sl; 2708 2818 /* write it back and re-read */ 2709 - rcu_read_lock(); 2710 2819 while (sl != slot) { 2711 2820 if (sl==0) 2712 2821 sl = conf->copies; 2713 2822 sl--; 2714 2823 d = r10_bio->devs[sl].devnum; 2715 - rdev = rcu_dereference(conf->mirrors[d].rdev); 2824 + rdev = conf->mirrors[d].rdev; 2716 2825 if (!rdev || 2717 2826 test_bit(Faulty, &rdev->flags) || 2718 2827 !test_bit(In_sync, &rdev->flags)) 2719 2828 continue; 2720 2829 2721 2830 atomic_inc(&rdev->nr_pending); 2722 - rcu_read_unlock(); 2723 2831 if (r10_sync_page_io(rdev, 2724 2832 r10_bio->devs[sl].addr + 2725 2833 sect, ··· 2736 2848 rdev->bdev); 2737 2849 } 2738 2850 rdev_dec_pending(rdev, mddev); 2739 - rcu_read_lock(); 2740 2851 } 2741 2852 sl = start; 2742 2853 while (sl != slot) { ··· 2743 2856 sl = conf->copies; 2744 2857 sl--; 2745 2858 d = r10_bio->devs[sl].devnum; 2746 - rdev = rcu_dereference(conf->mirrors[d].rdev); 2859 + rdev = conf->mirrors[d].rdev; 2747 2860 if (!rdev || 2748 2861 test_bit(Faulty, &rdev->flags) || 2749 2862 !test_bit(In_sync, &rdev->flags)) 2750 2863 continue; 2751 2864 2752 2865 atomic_inc(&rdev->nr_pending); 2753 - rcu_read_unlock(); 2754 2866 switch (r10_sync_page_io(rdev, 2755 2867 r10_bio->devs[sl].addr + 2756 2868 sect, ··· 2777 2891 } 2778 2892 2779 2893 rdev_dec_pending(rdev, mddev); 2780 - rcu_read_lock(); 2781 2894 } 2782 - rcu_read_unlock(); 2783 2895 2784 2896 sectors -= s; 2785 2897 sect += s; ··· 3251 3367 /* Completed a full sync so the replacements 3252 3368 * are now fully recovered. 3253 3369 */ 3254 - rcu_read_lock(); 3255 3370 for (i = 0; i < conf->geo.raid_disks; i++) { 3256 3371 struct md_rdev *rdev = 3257 - rcu_dereference(conf->mirrors[i].replacement); 3372 + conf->mirrors[i].replacement; 3373 + 3258 3374 if (rdev) 3259 3375 rdev->recovery_offset = MaxSector; 3260 3376 } 3261 - rcu_read_unlock(); 3262 3377 } 3263 3378 conf->fullsync = 0; 3264 3379 } ··· 3338 3455 struct raid10_info *mirror = &conf->mirrors[i]; 3339 3456 struct md_rdev *mrdev, *mreplace; 3340 3457 3341 - rcu_read_lock(); 3342 - mrdev = rcu_dereference(mirror->rdev); 3343 - mreplace = rcu_dereference(mirror->replacement); 3458 + mrdev = mirror->rdev; 3459 + mreplace = mirror->replacement; 3344 3460 3345 3461 if (mrdev && (test_bit(Faulty, &mrdev->flags) || 3346 3462 test_bit(In_sync, &mrdev->flags))) ··· 3347 3465 if (mreplace && test_bit(Faulty, &mreplace->flags)) 3348 3466 mreplace = NULL; 3349 3467 3350 - if (!mrdev && !mreplace) { 3351 - rcu_read_unlock(); 3468 + if (!mrdev && !mreplace) 3352 3469 continue; 3353 - } 3354 3470 3355 3471 still_degraded = 0; 3356 3472 /* want to reconstruct this device */ 3357 3473 rb2 = r10_bio; 3358 3474 sect = raid10_find_virt(conf, sector_nr, i); 3359 - if (sect >= mddev->resync_max_sectors) { 3475 + if (sect >= mddev->resync_max_sectors) 3360 3476 /* last stripe is not complete - don't 3361 3477 * try to recover this sector. 3362 3478 */ 3363 - rcu_read_unlock(); 3364 3479 continue; 3365 - } 3366 3480 /* Unless we are doing a full sync, or a replacement 3367 3481 * we only need to recover the block if it is set in 3368 3482 * the bitmap ··· 3374 3496 * that there will never be anything to do here 3375 3497 */ 3376 3498 chunks_skipped = -1; 3377 - rcu_read_unlock(); 3378 3499 continue; 3379 3500 } 3380 3501 if (mrdev) 3381 3502 atomic_inc(&mrdev->nr_pending); 3382 3503 if (mreplace) 3383 3504 atomic_inc(&mreplace->nr_pending); 3384 - rcu_read_unlock(); 3385 3505 3386 3506 r10_bio = raid10_alloc_init_r10buf(conf); 3387 3507 r10_bio->state = 0; ··· 3398 3522 /* Need to check if the array will still be 3399 3523 * degraded 3400 3524 */ 3401 - rcu_read_lock(); 3402 3525 for (j = 0; j < conf->geo.raid_disks; j++) { 3403 - struct md_rdev *rdev = rcu_dereference( 3404 - conf->mirrors[j].rdev); 3526 + struct md_rdev *rdev = conf->mirrors[j].rdev; 3527 + 3405 3528 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3406 3529 still_degraded = 1; 3407 3530 break; ··· 3415 3540 int k; 3416 3541 int d = r10_bio->devs[j].devnum; 3417 3542 sector_t from_addr, to_addr; 3418 - struct md_rdev *rdev = 3419 - rcu_dereference(conf->mirrors[d].rdev); 3543 + struct md_rdev *rdev = conf->mirrors[d].rdev; 3420 3544 sector_t sector, first_bad; 3421 3545 int bad_sectors; 3422 3546 if (!rdev || ··· 3494 3620 atomic_inc(&r10_bio->remaining); 3495 3621 break; 3496 3622 } 3497 - rcu_read_unlock(); 3498 3623 if (j == conf->copies) { 3499 3624 /* Cannot recover, so abort the recovery or 3500 3625 * record a bad block */ ··· 3620 3747 3621 3748 bio = r10_bio->devs[i].bio; 3622 3749 bio->bi_status = BLK_STS_IOERR; 3623 - rcu_read_lock(); 3624 - rdev = rcu_dereference(conf->mirrors[d].rdev); 3625 - if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3626 - rcu_read_unlock(); 3750 + rdev = conf->mirrors[d].rdev; 3751 + if (rdev == NULL || test_bit(Faulty, &rdev->flags)) 3627 3752 continue; 3628 - } 3753 + 3629 3754 sector = r10_bio->devs[i].addr; 3630 3755 if (is_badblock(rdev, sector, max_sync, 3631 3756 &first_bad, &bad_sectors)) { ··· 3633 3762 bad_sectors -= (sector - first_bad); 3634 3763 if (max_sync > bad_sectors) 3635 3764 max_sync = bad_sectors; 3636 - rcu_read_unlock(); 3637 3765 continue; 3638 3766 } 3639 3767 } ··· 3648 3778 bio_set_dev(bio, rdev->bdev); 3649 3779 count++; 3650 3780 3651 - rdev = rcu_dereference(conf->mirrors[d].replacement); 3652 - if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3653 - rcu_read_unlock(); 3781 + rdev = conf->mirrors[d].replacement; 3782 + if (rdev == NULL || test_bit(Faulty, &rdev->flags)) 3654 3783 continue; 3655 - } 3784 + 3656 3785 atomic_inc(&rdev->nr_pending); 3657 3786 3658 3787 /* Need to set up for writing to the replacement */ ··· 3668 3799 bio->bi_iter.bi_sector = sector + rdev->data_offset; 3669 3800 bio_set_dev(bio, rdev->bdev); 3670 3801 count++; 3671 - rcu_read_unlock(); 3672 3802 } 3673 3803 3674 3804 if (count < 2) { ··· 4377 4509 int degraded, degraded2; 4378 4510 int i; 4379 4511 4380 - rcu_read_lock(); 4381 4512 degraded = 0; 4382 4513 /* 'prev' section first */ 4383 4514 for (i = 0; i < conf->prev.raid_disks; i++) { 4384 - struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 4515 + struct md_rdev *rdev = conf->mirrors[i].rdev; 4516 + 4385 4517 if (!rdev || test_bit(Faulty, &rdev->flags)) 4386 4518 degraded++; 4387 4519 else if (!test_bit(In_sync, &rdev->flags)) ··· 4391 4523 */ 4392 4524 degraded++; 4393 4525 } 4394 - rcu_read_unlock(); 4395 4526 if (conf->geo.raid_disks == conf->prev.raid_disks) 4396 4527 return degraded; 4397 - rcu_read_lock(); 4398 4528 degraded2 = 0; 4399 4529 for (i = 0; i < conf->geo.raid_disks; i++) { 4400 - struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 4530 + struct md_rdev *rdev = conf->mirrors[i].rdev; 4531 + 4401 4532 if (!rdev || test_bit(Faulty, &rdev->flags)) 4402 4533 degraded2++; 4403 4534 else if (!test_bit(In_sync, &rdev->flags)) { ··· 4409 4542 degraded2++; 4410 4543 } 4411 4544 } 4412 - rcu_read_unlock(); 4413 4545 if (degraded2 > degraded) 4414 4546 return degraded2; 4415 4547 return degraded; ··· 4840 4974 blist = read_bio; 4841 4975 read_bio->bi_next = NULL; 4842 4976 4843 - rcu_read_lock(); 4844 4977 for (s = 0; s < conf->copies*2; s++) { 4845 4978 struct bio *b; 4846 4979 int d = r10_bio->devs[s/2].devnum; 4847 4980 struct md_rdev *rdev2; 4848 4981 if (s&1) { 4849 - rdev2 = rcu_dereference(conf->mirrors[d].replacement); 4982 + rdev2 = conf->mirrors[d].replacement; 4850 4983 b = r10_bio->devs[s/2].repl_bio; 4851 4984 } else { 4852 - rdev2 = rcu_dereference(conf->mirrors[d].rdev); 4985 + rdev2 = conf->mirrors[d].rdev; 4853 4986 b = r10_bio->devs[s/2].bio; 4854 4987 } 4855 4988 if (!rdev2 || test_bit(Faulty, &rdev2->flags)) ··· 4882 5017 sector_nr += len >> 9; 4883 5018 nr_sectors += len >> 9; 4884 5019 } 4885 - rcu_read_unlock(); 4886 5020 r10_bio->sectors = nr_sectors; 4887 5021 4888 5022 /* Now submit the read */ ··· 4934 5070 struct bio *b; 4935 5071 int d = r10_bio->devs[s/2].devnum; 4936 5072 struct md_rdev *rdev; 4937 - rcu_read_lock(); 4938 5073 if (s&1) { 4939 - rdev = rcu_dereference(conf->mirrors[d].replacement); 5074 + rdev = conf->mirrors[d].replacement; 4940 5075 b = r10_bio->devs[s/2].repl_bio; 4941 5076 } else { 4942 - rdev = rcu_dereference(conf->mirrors[d].rdev); 5077 + rdev = conf->mirrors[d].rdev; 4943 5078 b = r10_bio->devs[s/2].bio; 4944 5079 } 4945 - if (!rdev || test_bit(Faulty, &rdev->flags)) { 4946 - rcu_read_unlock(); 5080 + if (!rdev || test_bit(Faulty, &rdev->flags)) 4947 5081 continue; 4948 - } 5082 + 4949 5083 atomic_inc(&rdev->nr_pending); 4950 - rcu_read_unlock(); 4951 5084 md_sync_acct_bio(b, r10_bio->sectors); 4952 5085 atomic_inc(&r10_bio->remaining); 4953 5086 b->bi_next = NULL; ··· 5015 5154 if (s > (PAGE_SIZE >> 9)) 5016 5155 s = PAGE_SIZE >> 9; 5017 5156 5018 - rcu_read_lock(); 5019 5157 while (!success) { 5020 5158 int d = r10b->devs[slot].devnum; 5021 - struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 5159 + struct md_rdev *rdev = conf->mirrors[d].rdev; 5022 5160 sector_t addr; 5023 5161 if (rdev == NULL || 5024 5162 test_bit(Faulty, &rdev->flags) || ··· 5026 5166 5027 5167 addr = r10b->devs[slot].addr + idx * PAGE_SIZE; 5028 5168 atomic_inc(&rdev->nr_pending); 5029 - rcu_read_unlock(); 5030 5169 success = sync_page_io(rdev, 5031 5170 addr, 5032 5171 s << 9, 5033 5172 pages[idx], 5034 5173 REQ_OP_READ, false); 5035 5174 rdev_dec_pending(rdev, mddev); 5036 - rcu_read_lock(); 5037 5175 if (success) 5038 5176 break; 5039 5177 failed: ··· 5041 5183 if (slot == first_slot) 5042 5184 break; 5043 5185 } 5044 - rcu_read_unlock(); 5045 5186 if (!success) { 5046 5187 /* couldn't read this block, must give up */ 5047 5188 set_bit(MD_RECOVERY_INTR, ··· 5066 5209 struct md_rdev *rdev = NULL; 5067 5210 5068 5211 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 5069 - if (repl) 5070 - rdev = conf->mirrors[d].replacement; 5071 - if (!rdev) { 5072 - smp_mb(); 5073 - rdev = conf->mirrors[d].rdev; 5074 - } 5212 + rdev = repl ? conf->mirrors[d].replacement : 5213 + conf->mirrors[d].rdev; 5075 5214 5076 5215 if (bio->bi_status) { 5077 5216 /* FIXME should record badblock */ ··· 5102 5249 mddev->resync_max_sectors = mddev->array_sectors; 5103 5250 } else { 5104 5251 int d; 5105 - rcu_read_lock(); 5106 5252 for (d = conf->geo.raid_disks ; 5107 5253 d < conf->geo.raid_disks - mddev->delta_disks; 5108 5254 d++) { 5109 - struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 5255 + struct md_rdev *rdev = conf->mirrors[d].rdev; 5110 5256 if (rdev) 5111 5257 clear_bit(In_sync, &rdev->flags); 5112 - rdev = rcu_dereference(conf->mirrors[d].replacement); 5258 + rdev = conf->mirrors[d].replacement; 5113 5259 if (rdev) 5114 5260 clear_bit(In_sync, &rdev->flags); 5115 5261 } 5116 - rcu_read_unlock(); 5117 5262 } 5118 5263 mddev->layout = mddev->new_layout; 5119 5264 mddev->chunk_sectors = 1 << conf->geo.chunk_shift;

+2 -9

drivers/md/raid5-cache.c

··· 1890 1890 continue; 1891 1891 1892 1892 /* in case device is broken */ 1893 - rcu_read_lock(); 1894 - rdev = rcu_dereference(conf->disks[disk_index].rdev); 1893 + rdev = conf->disks[disk_index].rdev; 1895 1894 if (rdev) { 1896 1895 atomic_inc(&rdev->nr_pending); 1897 - rcu_read_unlock(); 1898 1896 sync_page_io(rdev, sh->sector, PAGE_SIZE, 1899 1897 sh->dev[disk_index].page, REQ_OP_WRITE, 1900 1898 false); 1901 1899 rdev_dec_pending(rdev, rdev->mddev); 1902 - rcu_read_lock(); 1903 1900 } 1904 - rrdev = rcu_dereference(conf->disks[disk_index].replacement); 1901 + rrdev = conf->disks[disk_index].replacement; 1905 1902 if (rrdev) { 1906 1903 atomic_inc(&rrdev->nr_pending); 1907 - rcu_read_unlock(); 1908 1904 sync_page_io(rrdev, sh->sector, PAGE_SIZE, 1909 1905 sh->dev[disk_index].page, REQ_OP_WRITE, 1910 1906 false); 1911 1907 rdev_dec_pending(rrdev, rrdev->mddev); 1912 - rcu_read_lock(); 1913 1908 } 1914 - rcu_read_unlock(); 1915 1909 } 1916 1910 ctx->data_parity_stripes++; 1917 1911 out: ··· 2942 2948 if (!log) 2943 2949 return false; 2944 2950 2945 - WARN_ON_ONCE(!rcu_read_lock_held()); 2946 2951 tree_index = r5c_tree_index(conf, sect); 2947 2952 slot = radix_tree_lookup(&log->big_stripe_tree, tree_index); 2948 2953 return slot != NULL;

+4 -12

drivers/md/raid5-ppl.c

··· 620 620 struct md_rdev *rdev; 621 621 struct block_device *bdev = NULL; 622 622 623 - rcu_read_lock(); 624 - rdev = rcu_dereference(conf->disks[i].rdev); 623 + rdev = conf->disks[i].rdev; 625 624 if (rdev && !test_bit(Faulty, &rdev->flags)) 626 625 bdev = rdev->bdev; 627 - rcu_read_unlock(); 628 626 629 627 if (bdev) { 630 628 struct bio *bio; ··· 880 882 (unsigned long long)r_sector, dd_idx, 881 883 (unsigned long long)sector); 882 884 883 - /* Array has not started so rcu dereference is safe */ 884 - rdev = rcu_dereference_protected( 885 - conf->disks[dd_idx].rdev, 1); 885 + rdev = conf->disks[dd_idx].rdev; 886 886 if (!rdev || (!test_bit(In_sync, &rdev->flags) && 887 887 sector >= rdev->recovery_offset)) { 888 888 pr_debug("%s:%*s data member disk %d missing\n", ··· 932 936 0, &disk, &sh); 933 937 BUG_ON(sh.pd_idx != le32_to_cpu(e->parity_disk)); 934 938 935 - /* Array has not started so rcu dereference is safe */ 936 - parity_rdev = rcu_dereference_protected( 937 - conf->disks[sh.pd_idx].rdev, 1); 939 + parity_rdev = conf->disks[sh.pd_idx].rdev; 938 940 939 941 BUG_ON(parity_rdev->bdev->bd_dev != log->rdev->bdev->bd_dev); 940 942 pr_debug("%s:%*s write parity at sector %llu, disk %pg\n", ··· 1398 1404 1399 1405 for (i = 0; i < ppl_conf->count; i++) { 1400 1406 struct ppl_log *log = &ppl_conf->child_logs[i]; 1401 - /* Array has not started so rcu dereference is safe */ 1402 - struct md_rdev *rdev = 1403 - rcu_dereference_protected(conf->disks[i].rdev, 1); 1407 + struct md_rdev *rdev = conf->disks[i].rdev; 1404 1408 1405 1409 mutex_init(&log->io_mutex); 1406 1410 spin_lock_init(&log->io_list_lock);

+61 -142

drivers/md/raid5.c

··· 36 36 */ 37 37 38 38 #include <linux/blkdev.h> 39 - #include <linux/delay.h> 40 39 #include <linux/kthread.h> 41 40 #include <linux/raid/pq.h> 42 41 #include <linux/async_tx.h> ··· 693 694 int degraded, degraded2; 694 695 int i; 695 696 696 - rcu_read_lock(); 697 697 degraded = 0; 698 698 for (i = 0; i < conf->previous_raid_disks; i++) { 699 - struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 699 + struct md_rdev *rdev = READ_ONCE(conf->disks[i].rdev); 700 + 700 701 if (rdev && test_bit(Faulty, &rdev->flags)) 701 - rdev = rcu_dereference(conf->disks[i].replacement); 702 + rdev = READ_ONCE(conf->disks[i].replacement); 702 703 if (!rdev || test_bit(Faulty, &rdev->flags)) 703 704 degraded++; 704 705 else if (test_bit(In_sync, &rdev->flags)) ··· 716 717 if (conf->raid_disks >= conf->previous_raid_disks) 717 718 degraded++; 718 719 } 719 - rcu_read_unlock(); 720 720 if (conf->raid_disks == conf->previous_raid_disks) 721 721 return degraded; 722 - rcu_read_lock(); 723 722 degraded2 = 0; 724 723 for (i = 0; i < conf->raid_disks; i++) { 725 - struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 724 + struct md_rdev *rdev = READ_ONCE(conf->disks[i].rdev); 725 + 726 726 if (rdev && test_bit(Faulty, &rdev->flags)) 727 - rdev = rcu_dereference(conf->disks[i].replacement); 727 + rdev = READ_ONCE(conf->disks[i].replacement); 728 728 if (!rdev || test_bit(Faulty, &rdev->flags)) 729 729 degraded2++; 730 730 else if (test_bit(In_sync, &rdev->flags)) ··· 737 739 if (conf->raid_disks <= conf->previous_raid_disks) 738 740 degraded2++; 739 741 } 740 - rcu_read_unlock(); 741 742 if (degraded2 > degraded) 742 743 return degraded2; 743 744 return degraded; ··· 1181 1184 bi = &dev->req; 1182 1185 rbi = &dev->rreq; /* For writing to replacement */ 1183 1186 1184 - rcu_read_lock(); 1185 - rrdev = rcu_dereference(conf->disks[i].replacement); 1186 - smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */ 1187 - rdev = rcu_dereference(conf->disks[i].rdev); 1188 - if (!rdev) { 1189 - rdev = rrdev; 1190 - rrdev = NULL; 1191 - } 1187 + rdev = conf->disks[i].rdev; 1188 + rrdev = conf->disks[i].replacement; 1192 1189 if (op_is_write(op)) { 1193 1190 if (replace_only) 1194 1191 rdev = NULL; ··· 1203 1212 rrdev = NULL; 1204 1213 if (rrdev) 1205 1214 atomic_inc(&rrdev->nr_pending); 1206 - rcu_read_unlock(); 1207 1215 1208 1216 /* We have already checked bad blocks for reads. Now 1209 1217 * need to check for writes. We never accept write errors ··· 2721 2731 conf->slab_cache = NULL; 2722 2732 } 2723 2733 2724 - /* 2725 - * This helper wraps rcu_dereference_protected() and can be used when 2726 - * it is known that the nr_pending of the rdev is elevated. 2727 - */ 2728 - static struct md_rdev *rdev_pend_deref(struct md_rdev __rcu *rdev) 2729 - { 2730 - return rcu_dereference_protected(rdev, 2731 - atomic_read(&rcu_access_pointer(rdev)->nr_pending)); 2732 - } 2733 - 2734 - /* 2735 - * This helper wraps rcu_dereference_protected() and should be used 2736 - * when it is known that the mddev_lock() is held. This is safe 2737 - * seeing raid5_remove_disk() has the same lock held. 2738 - */ 2739 - static struct md_rdev *rdev_mdlock_deref(struct mddev *mddev, 2740 - struct md_rdev __rcu *rdev) 2741 - { 2742 - return rcu_dereference_protected(rdev, 2743 - lockdep_is_held(&mddev->reconfig_mutex)); 2744 - } 2745 - 2746 2734 static void raid5_end_read_request(struct bio * bi) 2747 2735 { 2748 2736 struct stripe_head *sh = bi->bi_private; ··· 2746 2778 * In that case it moved down to 'rdev'. 2747 2779 * rdev is not removed until all requests are finished. 2748 2780 */ 2749 - rdev = rdev_pend_deref(conf->disks[i].replacement); 2781 + rdev = conf->disks[i].replacement; 2750 2782 if (!rdev) 2751 - rdev = rdev_pend_deref(conf->disks[i].rdev); 2783 + rdev = conf->disks[i].rdev; 2752 2784 2753 2785 if (use_new_offset(conf, sh)) 2754 2786 s = sh->sector + rdev->new_data_offset; ··· 2861 2893 2862 2894 for (i = 0 ; i < disks; i++) { 2863 2895 if (bi == &sh->dev[i].req) { 2864 - rdev = rdev_pend_deref(conf->disks[i].rdev); 2896 + rdev = conf->disks[i].rdev; 2865 2897 break; 2866 2898 } 2867 2899 if (bi == &sh->dev[i].rreq) { 2868 - rdev = rdev_pend_deref(conf->disks[i].replacement); 2900 + rdev = conf->disks[i].replacement; 2869 2901 if (rdev) 2870 2902 replacement = 1; 2871 2903 else ··· 2873 2905 * replaced it. rdev is not removed 2874 2906 * until all requests are finished. 2875 2907 */ 2876 - rdev = rdev_pend_deref(conf->disks[i].rdev); 2908 + rdev = conf->disks[i].rdev; 2877 2909 break; 2878 2910 } 2879 2911 } ··· 3635 3667 int bitmap_end = 0; 3636 3668 3637 3669 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 3638 - struct md_rdev *rdev; 3639 - rcu_read_lock(); 3640 - rdev = rcu_dereference(conf->disks[i].rdev); 3670 + struct md_rdev *rdev = conf->disks[i].rdev; 3671 + 3641 3672 if (rdev && test_bit(In_sync, &rdev->flags) && 3642 3673 !test_bit(Faulty, &rdev->flags)) 3643 3674 atomic_inc(&rdev->nr_pending); 3644 3675 else 3645 3676 rdev = NULL; 3646 - rcu_read_unlock(); 3647 3677 if (rdev) { 3648 3678 if (!rdev_set_badblocks( 3649 3679 rdev, ··· 3759 3793 /* During recovery devices cannot be removed, so 3760 3794 * locking and refcounting of rdevs is not needed 3761 3795 */ 3762 - rcu_read_lock(); 3763 3796 for (i = 0; i < conf->raid_disks; i++) { 3764 - struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 3797 + struct md_rdev *rdev = conf->disks[i].rdev; 3798 + 3765 3799 if (rdev 3766 3800 && !test_bit(Faulty, &rdev->flags) 3767 3801 && !test_bit(In_sync, &rdev->flags) 3768 3802 && !rdev_set_badblocks(rdev, sh->sector, 3769 3803 RAID5_STRIPE_SECTORS(conf), 0)) 3770 3804 abort = 1; 3771 - rdev = rcu_dereference(conf->disks[i].replacement); 3805 + rdev = conf->disks[i].replacement; 3806 + 3772 3807 if (rdev 3773 3808 && !test_bit(Faulty, &rdev->flags) 3774 3809 && !test_bit(In_sync, &rdev->flags) ··· 3777 3810 RAID5_STRIPE_SECTORS(conf), 0)) 3778 3811 abort = 1; 3779 3812 } 3780 - rcu_read_unlock(); 3781 3813 if (abort) 3782 3814 conf->recovery_disabled = 3783 3815 conf->mddev->recovery_disabled; ··· 3789 3823 struct md_rdev *rdev; 3790 3824 int rv = 0; 3791 3825 3792 - rcu_read_lock(); 3793 - rdev = rcu_dereference(sh->raid_conf->disks[disk_idx].replacement); 3826 + rdev = sh->raid_conf->disks[disk_idx].replacement; 3794 3827 if (rdev 3795 3828 && !test_bit(Faulty, &rdev->flags) 3796 3829 && !test_bit(In_sync, &rdev->flags) 3797 3830 && (rdev->recovery_offset <= sh->sector 3798 3831 || rdev->mddev->recovery_cp <= sh->sector)) 3799 3832 rv = 1; 3800 - rcu_read_unlock(); 3801 3833 return rv; 3802 3834 } 3803 3835 ··· 4672 4708 s->log_failed = r5l_log_disk_error(conf); 4673 4709 4674 4710 /* Now to look around and see what can be done */ 4675 - rcu_read_lock(); 4676 4711 for (i=disks; i--; ) { 4677 4712 struct md_rdev *rdev; 4678 4713 sector_t first_bad; ··· 4716 4753 /* Prefer to use the replacement for reads, but only 4717 4754 * if it is recovered enough and has no bad blocks. 4718 4755 */ 4719 - rdev = rcu_dereference(conf->disks[i].replacement); 4756 + rdev = conf->disks[i].replacement; 4720 4757 if (rdev && !test_bit(Faulty, &rdev->flags) && 4721 4758 rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) && 4722 4759 !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), ··· 4727 4764 set_bit(R5_NeedReplace, &dev->flags); 4728 4765 else 4729 4766 clear_bit(R5_NeedReplace, &dev->flags); 4730 - rdev = rcu_dereference(conf->disks[i].rdev); 4767 + rdev = conf->disks[i].rdev; 4731 4768 clear_bit(R5_ReadRepl, &dev->flags); 4732 4769 } 4733 4770 if (rdev && test_bit(Faulty, &rdev->flags)) ··· 4774 4811 if (test_bit(R5_WriteError, &dev->flags)) { 4775 4812 /* This flag does not apply to '.replacement' 4776 4813 * only to .rdev, so make sure to check that*/ 4777 - struct md_rdev *rdev2 = rcu_dereference( 4778 - conf->disks[i].rdev); 4814 + struct md_rdev *rdev2 = conf->disks[i].rdev; 4815 + 4779 4816 if (rdev2 == rdev) 4780 4817 clear_bit(R5_Insync, &dev->flags); 4781 4818 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { ··· 4787 4824 if (test_bit(R5_MadeGood, &dev->flags)) { 4788 4825 /* This flag does not apply to '.replacement' 4789 4826 * only to .rdev, so make sure to check that*/ 4790 - struct md_rdev *rdev2 = rcu_dereference( 4791 - conf->disks[i].rdev); 4827 + struct md_rdev *rdev2 = conf->disks[i].rdev; 4828 + 4792 4829 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 4793 4830 s->handle_bad_blocks = 1; 4794 4831 atomic_inc(&rdev2->nr_pending); ··· 4796 4833 clear_bit(R5_MadeGood, &dev->flags); 4797 4834 } 4798 4835 if (test_bit(R5_MadeGoodRepl, &dev->flags)) { 4799 - struct md_rdev *rdev2 = rcu_dereference( 4800 - conf->disks[i].replacement); 4836 + struct md_rdev *rdev2 = conf->disks[i].replacement; 4837 + 4801 4838 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 4802 4839 s->handle_bad_blocks = 1; 4803 4840 atomic_inc(&rdev2->nr_pending); ··· 4818 4855 if (rdev && !test_bit(Faulty, &rdev->flags)) 4819 4856 do_recovery = 1; 4820 4857 else if (!rdev) { 4821 - rdev = rcu_dereference( 4822 - conf->disks[i].replacement); 4858 + rdev = conf->disks[i].replacement; 4823 4859 if (rdev && !test_bit(Faulty, &rdev->flags)) 4824 4860 do_recovery = 1; 4825 4861 } ··· 4845 4883 else 4846 4884 s->replacing = 1; 4847 4885 } 4848 - rcu_read_unlock(); 4849 4886 } 4850 4887 4851 4888 /* ··· 5301 5340 struct r5dev *dev = &sh->dev[i]; 5302 5341 if (test_and_clear_bit(R5_WriteError, &dev->flags)) { 5303 5342 /* We own a safe reference to the rdev */ 5304 - rdev = rdev_pend_deref(conf->disks[i].rdev); 5343 + rdev = conf->disks[i].rdev; 5305 5344 if (!rdev_set_badblocks(rdev, sh->sector, 5306 5345 RAID5_STRIPE_SECTORS(conf), 0)) 5307 5346 md_error(conf->mddev, rdev); 5308 5347 rdev_dec_pending(rdev, conf->mddev); 5309 5348 } 5310 5349 if (test_and_clear_bit(R5_MadeGood, &dev->flags)) { 5311 - rdev = rdev_pend_deref(conf->disks[i].rdev); 5350 + rdev = conf->disks[i].rdev; 5312 5351 rdev_clear_badblocks(rdev, sh->sector, 5313 5352 RAID5_STRIPE_SECTORS(conf), 0); 5314 5353 rdev_dec_pending(rdev, conf->mddev); 5315 5354 } 5316 5355 if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) { 5317 - rdev = rdev_pend_deref(conf->disks[i].replacement); 5356 + rdev = conf->disks[i].replacement; 5318 5357 if (!rdev) 5319 5358 /* rdev have been moved down */ 5320 - rdev = rdev_pend_deref(conf->disks[i].rdev); 5359 + rdev = conf->disks[i].rdev; 5321 5360 rdev_clear_badblocks(rdev, sh->sector, 5322 5361 RAID5_STRIPE_SECTORS(conf), 0); 5323 5362 rdev_dec_pending(rdev, conf->mddev); ··· 5476 5515 &dd_idx, NULL); 5477 5516 end_sector = sector + bio_sectors(raid_bio); 5478 5517 5479 - rcu_read_lock(); 5480 5518 if (r5c_big_stripe_cached(conf, sector)) 5481 - goto out_rcu_unlock; 5519 + return 0; 5482 5520 5483 - rdev = rcu_dereference(conf->disks[dd_idx].replacement); 5521 + rdev = conf->disks[dd_idx].replacement; 5484 5522 if (!rdev || test_bit(Faulty, &rdev->flags) || 5485 5523 rdev->recovery_offset < end_sector) { 5486 - rdev = rcu_dereference(conf->disks[dd_idx].rdev); 5524 + rdev = conf->disks[dd_idx].rdev; 5487 5525 if (!rdev) 5488 - goto out_rcu_unlock; 5526 + return 0; 5489 5527 if (test_bit(Faulty, &rdev->flags) || 5490 5528 !(test_bit(In_sync, &rdev->flags) || 5491 5529 rdev->recovery_offset >= end_sector)) 5492 - goto out_rcu_unlock; 5530 + return 0; 5493 5531 } 5494 5532 5495 5533 atomic_inc(&rdev->nr_pending); 5496 - rcu_read_unlock(); 5497 5534 5498 5535 if (is_badblock(rdev, sector, bio_sectors(raid_bio), &first_bad, 5499 5536 &bad_sectors)) { ··· 5535 5576 raid_bio->bi_iter.bi_sector); 5536 5577 submit_bio_noacct(align_bio); 5537 5578 return 1; 5538 - 5539 - out_rcu_unlock: 5540 - rcu_read_unlock(); 5541 - return 0; 5542 5579 } 5543 5580 5544 5581 static struct bio *chunk_aligned_read(struct mddev *mddev, struct bio *raid_bio) ··· 6537 6582 * Note in case of > 1 drive failures it's possible we're rebuilding 6538 6583 * one drive while leaving another faulty drive in array. 6539 6584 */ 6540 - rcu_read_lock(); 6541 6585 for (i = 0; i < conf->raid_disks; i++) { 6542 - struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 6586 + struct md_rdev *rdev = conf->disks[i].rdev; 6543 6587 6544 6588 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) 6545 6589 still_degraded = 1; 6546 6590 } 6547 - rcu_read_unlock(); 6548 6591 6549 6592 md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); 6550 6593 ··· 6773 6820 spin_unlock_irq(&conf->device_lock); 6774 6821 md_check_recovery(mddev); 6775 6822 spin_lock_irq(&conf->device_lock); 6776 - 6777 - /* 6778 - * Waiting on MD_SB_CHANGE_PENDING below may deadlock 6779 - * seeing md_check_recovery() is needed to clear 6780 - * the flag when using mdmon. 6781 - */ 6782 - continue; 6783 6823 } 6784 - 6785 - wait_event_lock_irq(mddev->sb_wait, 6786 - !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags), 6787 - conf->device_lock); 6788 6824 } 6789 6825 pr_debug("%d stripes handled\n", handled); 6790 6826 ··· 7853 7911 7854 7912 for (i = 0; i < conf->raid_disks && conf->previous_raid_disks; 7855 7913 i++) { 7856 - rdev = rdev_mdlock_deref(mddev, conf->disks[i].rdev); 7857 - if (!rdev && conf->disks[i].replacement) { 7858 - /* The replacement is all we have yet */ 7859 - rdev = rdev_mdlock_deref(mddev, 7860 - conf->disks[i].replacement); 7861 - conf->disks[i].replacement = NULL; 7862 - clear_bit(Replacement, &rdev->flags); 7863 - rcu_assign_pointer(conf->disks[i].rdev, rdev); 7864 - } 7914 + rdev = conf->disks[i].rdev; 7865 7915 if (!rdev) 7866 7916 continue; 7867 - if (rcu_access_pointer(conf->disks[i].replacement) && 7917 + if (conf->disks[i].replacement && 7868 7918 conf->reshape_progress != MaxSector) { 7869 7919 /* replacements and reshape simply do not mix. */ 7870 7920 pr_warn("md: cannot handle concurrent replacement and reshape.\n"); ··· 8040 8106 struct r5conf *conf = mddev->private; 8041 8107 int i; 8042 8108 8109 + lockdep_assert_held(&mddev->lock); 8110 + 8043 8111 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, 8044 8112 conf->chunk_sectors / 2, mddev->layout); 8045 8113 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); 8046 - rcu_read_lock(); 8047 8114 for (i = 0; i < conf->raid_disks; i++) { 8048 - struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 8115 + struct md_rdev *rdev = READ_ONCE(conf->disks[i].rdev); 8116 + 8049 8117 seq_printf (seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 8050 8118 } 8051 - rcu_read_unlock(); 8052 8119 seq_printf (seq, "]"); 8053 8120 } 8054 8121 ··· 8087 8152 unsigned long flags; 8088 8153 8089 8154 for (i = 0; i < conf->raid_disks; i++) { 8090 - rdev = rdev_mdlock_deref(mddev, conf->disks[i].rdev); 8091 - replacement = rdev_mdlock_deref(mddev, 8092 - conf->disks[i].replacement); 8155 + rdev = conf->disks[i].rdev; 8156 + replacement = conf->disks[i].replacement; 8093 8157 if (replacement 8094 8158 && replacement->recovery_offset == MaxSector 8095 8159 && !test_bit(Faulty, &replacement->flags) ··· 8127 8193 struct r5conf *conf = mddev->private; 8128 8194 int err = 0; 8129 8195 int number = rdev->raid_disk; 8130 - struct md_rdev __rcu **rdevp; 8196 + struct md_rdev **rdevp; 8131 8197 struct disk_info *p; 8132 8198 struct md_rdev *tmp; 8133 8199 ··· 8150 8216 if (unlikely(number >= conf->pool_size)) 8151 8217 return 0; 8152 8218 p = conf->disks + number; 8153 - if (rdev == rcu_access_pointer(p->rdev)) 8219 + if (rdev == p->rdev) 8154 8220 rdevp = &p->rdev; 8155 - else if (rdev == rcu_access_pointer(p->replacement)) 8221 + else if (rdev == p->replacement) 8156 8222 rdevp = &p->replacement; 8157 8223 else 8158 8224 return 0; ··· 8172 8238 if (!test_bit(Faulty, &rdev->flags) && 8173 8239 mddev->recovery_disabled != conf->recovery_disabled && 8174 8240 !has_failed(conf) && 8175 - (!rcu_access_pointer(p->replacement) || 8176 - rcu_access_pointer(p->replacement) == rdev) && 8241 + (!p->replacement || p->replacement == rdev) && 8177 8242 number < conf->raid_disks) { 8178 8243 err = -EBUSY; 8179 8244 goto abort; 8180 8245 } 8181 - *rdevp = NULL; 8182 - if (!test_bit(RemoveSynchronized, &rdev->flags)) { 8183 - lockdep_assert_held(&mddev->reconfig_mutex); 8184 - synchronize_rcu(); 8185 - if (atomic_read(&rdev->nr_pending)) { 8186 - /* lost the race, try later */ 8187 - err = -EBUSY; 8188 - rcu_assign_pointer(*rdevp, rdev); 8189 - } 8190 - } 8246 + WRITE_ONCE(*rdevp, NULL); 8191 8247 if (!err) { 8192 8248 err = log_modify(conf, rdev, false); 8193 8249 if (err) 8194 8250 goto abort; 8195 8251 } 8196 8252 8197 - tmp = rcu_access_pointer(p->replacement); 8253 + tmp = p->replacement; 8198 8254 if (tmp) { 8199 8255 /* We must have just cleared 'rdev' */ 8200 - rcu_assign_pointer(p->rdev, tmp); 8256 + WRITE_ONCE(p->rdev, tmp); 8201 8257 clear_bit(Replacement, &tmp->flags); 8202 - smp_mb(); /* Make sure other CPUs may see both as identical 8203 - * but will never see neither - if they are careful 8204 - */ 8205 - rcu_assign_pointer(p->replacement, NULL); 8258 + WRITE_ONCE(p->replacement, NULL); 8206 8259 8207 8260 if (!err) 8208 8261 err = log_modify(conf, tmp, true); ··· 8257 8336 rdev->raid_disk = disk; 8258 8337 if (rdev->saved_raid_disk != disk) 8259 8338 conf->fullsync = 1; 8260 - rcu_assign_pointer(p->rdev, rdev); 8339 + WRITE_ONCE(p->rdev, rdev); 8261 8340 8262 8341 err = log_modify(conf, rdev, true); 8263 8342 ··· 8266 8345 } 8267 8346 for (disk = first; disk <= last; disk++) { 8268 8347 p = conf->disks + disk; 8269 - tmp = rdev_mdlock_deref(mddev, p->rdev); 8348 + tmp = p->rdev; 8270 8349 if (test_bit(WantReplacement, &tmp->flags) && 8271 8350 mddev->reshape_position == MaxSector && 8272 8351 p->replacement == NULL) { ··· 8275 8354 rdev->raid_disk = disk; 8276 8355 err = 0; 8277 8356 conf->fullsync = 1; 8278 - rcu_assign_pointer(p->replacement, rdev); 8357 + WRITE_ONCE(p->replacement, rdev); 8279 8358 break; 8280 8359 } 8281 8360 } ··· 8408 8487 if (mddev->recovery_cp < MaxSector) 8409 8488 return -EBUSY; 8410 8489 for (i = 0; i < conf->raid_disks; i++) 8411 - if (rdev_mdlock_deref(mddev, conf->disks[i].replacement)) 8490 + if (conf->disks[i].replacement) 8412 8491 return -EBUSY; 8413 8492 8414 8493 rdev_for_each(rdev, mddev) { ··· 8579 8658 for (d = conf->raid_disks ; 8580 8659 d < conf->raid_disks - mddev->delta_disks; 8581 8660 d++) { 8582 - rdev = rdev_mdlock_deref(mddev, 8583 - conf->disks[d].rdev); 8661 + rdev = conf->disks[d].rdev; 8584 8662 if (rdev) 8585 8663 clear_bit(In_sync, &rdev->flags); 8586 - rdev = rdev_mdlock_deref(mddev, 8587 - conf->disks[d].replacement); 8664 + rdev = conf->disks[d].replacement; 8588 8665 if (rdev) 8589 8666 clear_bit(In_sync, &rdev->flags); 8590 8667 }

+2 -2

drivers/md/raid5.h

··· 473 473 */ 474 474 475 475 struct disk_info { 476 - struct md_rdev __rcu *rdev; 477 - struct md_rdev __rcu *replacement; 476 + struct md_rdev *rdev; 477 + struct md_rdev *replacement; 478 478 struct page *extra_page; /* extra page to use in prexor */ 479 479 }; 480 480

+1 -3

drivers/mtd/mtd_blkdevs.c

··· 376 376 blk_queue_flag_set(QUEUE_FLAG_NONROT, new->rq); 377 377 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, new->rq); 378 378 379 - if (tr->discard) { 379 + if (tr->discard) 380 380 blk_queue_max_discard_sectors(new->rq, UINT_MAX); 381 - new->rq->limits.discard_granularity = tr->blksize; 382 - } 383 381 384 382 gd->queue = new->rq; 385 383

+128 -96

drivers/nvme/host/core.c

··· 20 20 #include <linux/ptrace.h> 21 21 #include <linux/nvme_ioctl.h> 22 22 #include <linux/pm_qos.h> 23 + #include <linux/ratelimit.h> 23 24 #include <asm/unaligned.h> 24 25 25 26 #include "nvme.h" ··· 313 312 struct nvme_request *nr = nvme_req(req); 314 313 315 314 if (ns) { 316 - pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %llu blocks, %s (sct 0x%x / sc 0x%x) %s%s\n", 315 + pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %u blocks, %s (sct 0x%x / sc 0x%x) %s%s\n", 317 316 ns->disk ? ns->disk->disk_name : "?", 318 317 nvme_get_opcode_str(nr->cmd->common.opcode), 319 318 nr->cmd->common.opcode, 320 - (unsigned long long)nvme_sect_to_lba(ns, blk_rq_pos(req)), 321 - (unsigned long long)blk_rq_bytes(req) >> ns->lba_shift, 319 + nvme_sect_to_lba(ns->head, blk_rq_pos(req)), 320 + blk_rq_bytes(req) >> ns->head->lba_shift, 322 321 nvme_get_error_status_str(nr->status), 323 322 nr->status >> 8 & 7, /* Status Code Type */ 324 323 nr->status & 0xff, /* Status Code */ ··· 373 372 static inline void nvme_end_req_zoned(struct request *req) 374 373 { 375 374 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && 376 - req_op(req) == REQ_OP_ZONE_APPEND) 377 - req->__sector = nvme_lba_to_sect(req->q->queuedata, 375 + req_op(req) == REQ_OP_ZONE_APPEND) { 376 + struct nvme_ns *ns = req->q->queuedata; 377 + 378 + req->__sector = nvme_lba_to_sect(ns->head, 378 379 le64_to_cpu(nvme_req(req)->result.u64)); 380 + } 379 381 } 380 382 381 383 static inline void nvme_end_req(struct request *req) ··· 797 793 } 798 794 799 795 if (queue_max_discard_segments(req->q) == 1) { 800 - u64 slba = nvme_sect_to_lba(ns, blk_rq_pos(req)); 801 - u32 nlb = blk_rq_sectors(req) >> (ns->lba_shift - 9); 796 + u64 slba = nvme_sect_to_lba(ns->head, blk_rq_pos(req)); 797 + u32 nlb = blk_rq_sectors(req) >> (ns->head->lba_shift - 9); 802 798 803 799 range[0].cattr = cpu_to_le32(0); 804 800 range[0].nlb = cpu_to_le32(nlb); ··· 806 802 n = 1; 807 803 } else { 808 804 __rq_for_each_bio(bio, req) { 809 - u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector); 810 - u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift; 805 + u64 slba = nvme_sect_to_lba(ns->head, 806 + bio->bi_iter.bi_sector); 807 + u32 nlb = bio->bi_iter.bi_size >> ns->head->lba_shift; 811 808 812 809 if (n < segments) { 813 810 range[n].cattr = cpu_to_le32(0); ··· 846 841 u64 ref48; 847 842 848 843 /* both rw and write zeroes share the same reftag format */ 849 - switch (ns->guard_type) { 844 + switch (ns->head->guard_type) { 850 845 case NVME_NVM_NS_16B_GUARD: 851 846 cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req)); 852 847 break; ··· 874 869 cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes; 875 870 cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id); 876 871 cmnd->write_zeroes.slba = 877 - cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req))); 872 + cpu_to_le64(nvme_sect_to_lba(ns->head, blk_rq_pos(req))); 878 873 cmnd->write_zeroes.length = 879 - cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); 874 + cpu_to_le16((blk_rq_bytes(req) >> ns->head->lba_shift) - 1); 880 875 881 - if (!(req->cmd_flags & REQ_NOUNMAP) && (ns->features & NVME_NS_DEAC)) 876 + if (!(req->cmd_flags & REQ_NOUNMAP) && 877 + (ns->head->features & NVME_NS_DEAC)) 882 878 cmnd->write_zeroes.control |= cpu_to_le16(NVME_WZ_DEAC); 883 879 884 - if (nvme_ns_has_pi(ns)) { 880 + if (nvme_ns_has_pi(ns->head)) { 885 881 cmnd->write_zeroes.control |= cpu_to_le16(NVME_RW_PRINFO_PRACT); 886 882 887 - switch (ns->pi_type) { 883 + switch (ns->head->pi_type) { 888 884 case NVME_NS_DPS_PI_TYPE1: 889 885 case NVME_NS_DPS_PI_TYPE2: 890 886 nvme_set_ref_tag(ns, cmnd, req); ··· 917 911 cmnd->rw.cdw2 = 0; 918 912 cmnd->rw.cdw3 = 0; 919 913 cmnd->rw.metadata = 0; 920 - cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req))); 921 - cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); 914 + cmnd->rw.slba = 915 + cpu_to_le64(nvme_sect_to_lba(ns->head, blk_rq_pos(req))); 916 + cmnd->rw.length = 917 + cpu_to_le16((blk_rq_bytes(req) >> ns->head->lba_shift) - 1); 922 918 cmnd->rw.reftag = 0; 923 919 cmnd->rw.apptag = 0; 924 920 cmnd->rw.appmask = 0; 925 921 926 - if (ns->ms) { 922 + if (ns->head->ms) { 927 923 /* 928 924 * If formated with metadata, the block layer always provides a 929 925 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled. Else ··· 933 925 * namespace capacity to zero to prevent any I/O. 934 926 */ 935 927 if (!blk_integrity_rq(req)) { 936 - if (WARN_ON_ONCE(!nvme_ns_has_pi(ns))) 928 + if (WARN_ON_ONCE(!nvme_ns_has_pi(ns->head))) 937 929 return BLK_STS_NOTSUPP; 938 930 control |= NVME_RW_PRINFO_PRACT; 939 931 } 940 932 941 - switch (ns->pi_type) { 933 + switch (ns->head->pi_type) { 942 934 case NVME_NS_DPS_PI_TYPE3: 943 935 control |= NVME_RW_PRINFO_PRCHK_GUARD; 944 936 break; ··· 1460 1452 return status; 1461 1453 } 1462 1454 1463 - static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid, 1455 + int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid, 1464 1456 struct nvme_id_ns **id) 1465 1457 { 1466 1458 struct nvme_command c = { }; ··· 1679 1671 } 1680 1672 1681 1673 #ifdef CONFIG_BLK_DEV_INTEGRITY 1682 - static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns, 1683 - u32 max_integrity_segments) 1674 + static void nvme_init_integrity(struct gendisk *disk, 1675 + struct nvme_ns_head *head, u32 max_integrity_segments) 1684 1676 { 1685 1677 struct blk_integrity integrity = { }; 1686 1678 1687 - switch (ns->pi_type) { 1679 + switch (head->pi_type) { 1688 1680 case NVME_NS_DPS_PI_TYPE3: 1689 - switch (ns->guard_type) { 1681 + switch (head->guard_type) { 1690 1682 case NVME_NVM_NS_16B_GUARD: 1691 1683 integrity.profile = &t10_pi_type3_crc; 1692 1684 integrity.tag_size = sizeof(u16) + sizeof(u32); ··· 1704 1696 break; 1705 1697 case NVME_NS_DPS_PI_TYPE1: 1706 1698 case NVME_NS_DPS_PI_TYPE2: 1707 - switch (ns->guard_type) { 1699 + switch (head->guard_type) { 1708 1700 case NVME_NVM_NS_16B_GUARD: 1709 1701 integrity.profile = &t10_pi_type1_crc; 1710 1702 integrity.tag_size = sizeof(u16); ··· 1725 1717 break; 1726 1718 } 1727 1719 1728 - integrity.tuple_size = ns->ms; 1720 + integrity.tuple_size = head->ms; 1729 1721 blk_integrity_register(disk, &integrity); 1730 1722 blk_queue_max_integrity_segments(disk->queue, max_integrity_segments); 1731 1723 } 1732 1724 #else 1733 - static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns, 1734 - u32 max_integrity_segments) 1725 + static void nvme_init_integrity(struct gendisk *disk, 1726 + struct nvme_ns_head *head, u32 max_integrity_segments) 1735 1727 { 1736 1728 } 1737 1729 #endif /* CONFIG_BLK_DEV_INTEGRITY */ 1738 1730 1739 - static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns) 1731 + static void nvme_config_discard(struct nvme_ctrl *ctrl, struct gendisk *disk, 1732 + struct nvme_ns_head *head) 1740 1733 { 1741 - struct nvme_ctrl *ctrl = ns->ctrl; 1742 1734 struct request_queue *queue = disk->queue; 1743 1735 u32 size = queue_logical_block_size(queue); 1744 1736 1745 - if (ctrl->dmrsl && ctrl->dmrsl <= nvme_sect_to_lba(ns, UINT_MAX)) 1746 - ctrl->max_discard_sectors = nvme_lba_to_sect(ns, ctrl->dmrsl); 1737 + if (ctrl->dmrsl && ctrl->dmrsl <= nvme_sect_to_lba(head, UINT_MAX)) 1738 + ctrl->max_discard_sectors = 1739 + nvme_lba_to_sect(head, ctrl->dmrsl); 1747 1740 1748 1741 if (ctrl->max_discard_sectors == 0) { 1749 1742 blk_queue_max_discard_sectors(queue, 0); ··· 1775 1766 a->csi == b->csi; 1776 1767 } 1777 1768 1778 - static int nvme_init_ms(struct nvme_ns *ns, struct nvme_id_ns *id) 1769 + static int nvme_init_ms(struct nvme_ctrl *ctrl, struct nvme_ns_head *head, 1770 + struct nvme_id_ns *id) 1779 1771 { 1780 1772 bool first = id->dps & NVME_NS_DPS_PI_FIRST; 1781 1773 unsigned lbaf = nvme_lbaf_index(id->flbas); 1782 - struct nvme_ctrl *ctrl = ns->ctrl; 1783 1774 struct nvme_command c = { }; 1784 1775 struct nvme_id_ns_nvm *nvm; 1785 1776 int ret = 0; 1786 1777 u32 elbaf; 1787 1778 1788 - ns->pi_size = 0; 1789 - ns->ms = le16_to_cpu(id->lbaf[lbaf].ms); 1779 + head->pi_size = 0; 1780 + head->ms = le16_to_cpu(id->lbaf[lbaf].ms); 1790 1781 if (!(ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)) { 1791 - ns->pi_size = sizeof(struct t10_pi_tuple); 1792 - ns->guard_type = NVME_NVM_NS_16B_GUARD; 1782 + head->pi_size = sizeof(struct t10_pi_tuple); 1783 + head->guard_type = NVME_NVM_NS_16B_GUARD; 1793 1784 goto set_pi; 1794 1785 } 1795 1786 ··· 1798 1789 return -ENOMEM; 1799 1790 1800 1791 c.identify.opcode = nvme_admin_identify; 1801 - c.identify.nsid = cpu_to_le32(ns->head->ns_id); 1792 + c.identify.nsid = cpu_to_le32(head->ns_id); 1802 1793 c.identify.cns = NVME_ID_CNS_CS_NS; 1803 1794 c.identify.csi = NVME_CSI_NVM; 1804 1795 1805 - ret = nvme_submit_sync_cmd(ns->ctrl->admin_q, &c, nvm, sizeof(*nvm)); 1796 + ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, nvm, sizeof(*nvm)); 1806 1797 if (ret) 1807 1798 goto free_data; 1808 1799 ··· 1812 1803 if (nvme_elbaf_sts(elbaf)) 1813 1804 goto free_data; 1814 1805 1815 - ns->guard_type = nvme_elbaf_guard_type(elbaf); 1816 - switch (ns->guard_type) { 1806 + head->guard_type = nvme_elbaf_guard_type(elbaf); 1807 + switch (head->guard_type) { 1817 1808 case NVME_NVM_NS_64B_GUARD: 1818 - ns->pi_size = sizeof(struct crc64_pi_tuple); 1809 + head->pi_size = sizeof(struct crc64_pi_tuple); 1819 1810 break; 1820 1811 case NVME_NVM_NS_16B_GUARD: 1821 - ns->pi_size = sizeof(struct t10_pi_tuple); 1812 + head->pi_size = sizeof(struct t10_pi_tuple); 1822 1813 break; 1823 1814 default: 1824 1815 break; ··· 1827 1818 free_data: 1828 1819 kfree(nvm); 1829 1820 set_pi: 1830 - if (ns->pi_size && (first || ns->ms == ns->pi_size)) 1831 - ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK; 1821 + if (head->pi_size && (first || head->ms == head->pi_size)) 1822 + head->pi_type = id->dps & NVME_NS_DPS_PI_MASK; 1832 1823 else 1833 - ns->pi_type = 0; 1824 + head->pi_type = 0; 1834 1825 1835 1826 return ret; 1836 1827 } 1837 1828 1838 - static int nvme_configure_metadata(struct nvme_ns *ns, struct nvme_id_ns *id) 1829 + static int nvme_configure_metadata(struct nvme_ctrl *ctrl, 1830 + struct nvme_ns_head *head, struct nvme_id_ns *id) 1839 1831 { 1840 - struct nvme_ctrl *ctrl = ns->ctrl; 1841 1832 int ret; 1842 1833 1843 - ret = nvme_init_ms(ns, id); 1834 + ret = nvme_init_ms(ctrl, head, id); 1844 1835 if (ret) 1845 1836 return ret; 1846 1837 1847 - ns->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS); 1848 - if (!ns->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED)) 1838 + head->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS); 1839 + if (!head->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED)) 1849 1840 return 0; 1850 1841 1851 1842 if (ctrl->ops->flags & NVME_F_FABRICS) { ··· 1857 1848 if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT))) 1858 1849 return 0; 1859 1850 1860 - ns->features |= NVME_NS_EXT_LBAS; 1851 + head->features |= NVME_NS_EXT_LBAS; 1861 1852 1862 1853 /* 1863 1854 * The current fabrics transport drivers support namespace ··· 1868 1859 * Note, this check will need to be modified if any drivers 1869 1860 * gain the ability to use other metadata formats. 1870 1861 */ 1871 - if (ctrl->max_integrity_segments && nvme_ns_has_pi(ns)) 1872 - ns->features |= NVME_NS_METADATA_SUPPORTED; 1862 + if (ctrl->max_integrity_segments && nvme_ns_has_pi(head)) 1863 + head->features |= NVME_NS_METADATA_SUPPORTED; 1873 1864 } else { 1874 1865 /* 1875 1866 * For PCIe controllers, we can't easily remap the separate ··· 1878 1869 * We allow extended LBAs for the passthrough interface, though. 1879 1870 */ 1880 1871 if (id->flbas & NVME_NS_FLBAS_META_EXT) 1881 - ns->features |= NVME_NS_EXT_LBAS; 1872 + head->features |= NVME_NS_EXT_LBAS; 1882 1873 else 1883 - ns->features |= NVME_NS_METADATA_SUPPORTED; 1874 + head->features |= NVME_NS_METADATA_SUPPORTED; 1884 1875 } 1885 1876 return 0; 1886 1877 } ··· 1903 1894 blk_queue_write_cache(q, vwc, vwc); 1904 1895 } 1905 1896 1906 - static void nvme_update_disk_info(struct gendisk *disk, 1907 - struct nvme_ns *ns, struct nvme_id_ns *id) 1897 + static void nvme_update_disk_info(struct nvme_ctrl *ctrl, struct gendisk *disk, 1898 + struct nvme_ns_head *head, struct nvme_id_ns *id) 1908 1899 { 1909 - sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze)); 1910 - u32 bs = 1U << ns->lba_shift; 1900 + sector_t capacity = nvme_lba_to_sect(head, le64_to_cpu(id->nsze)); 1901 + u32 bs = 1U << head->lba_shift; 1911 1902 u32 atomic_bs, phys_bs, io_opt = 0; 1912 1903 1913 1904 /* ··· 1915 1906 * or smaller than a sector size yet, so catch this early and don't 1916 1907 * allow block I/O. 1917 1908 */ 1918 - if (ns->lba_shift > PAGE_SHIFT || ns->lba_shift < SECTOR_SHIFT) { 1909 + if (head->lba_shift > PAGE_SHIFT || head->lba_shift < SECTOR_SHIFT) { 1919 1910 capacity = 0; 1920 1911 bs = (1 << 9); 1921 1912 } ··· 1932 1923 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf) 1933 1924 atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs; 1934 1925 else 1935 - atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs; 1926 + atomic_bs = (1 + ctrl->subsys->awupf) * bs; 1936 1927 } 1937 1928 1938 1929 if (id->nsfeat & NVME_NS_FEAT_IO_OPT) { ··· 1958 1949 * I/O to namespaces with metadata except when the namespace supports 1959 1950 * PI, as it can strip/insert in that case. 1960 1951 */ 1961 - if (ns->ms) { 1952 + if (head->ms) { 1962 1953 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && 1963 - (ns->features & NVME_NS_METADATA_SUPPORTED)) 1964 - nvme_init_integrity(disk, ns, 1965 - ns->ctrl->max_integrity_segments); 1966 - else if (!nvme_ns_has_pi(ns)) 1954 + (head->features & NVME_NS_METADATA_SUPPORTED)) 1955 + nvme_init_integrity(disk, head, 1956 + ctrl->max_integrity_segments); 1957 + else if (!nvme_ns_has_pi(head)) 1967 1958 capacity = 0; 1968 1959 } 1969 1960 1970 1961 set_capacity_and_notify(disk, capacity); 1971 1962 1972 - nvme_config_discard(disk, ns); 1963 + nvme_config_discard(ctrl, disk, head); 1973 1964 blk_queue_max_write_zeroes_sectors(disk->queue, 1974 - ns->ctrl->max_zeroes_sectors); 1965 + ctrl->max_zeroes_sectors); 1975 1966 } 1976 1967 1977 1968 static bool nvme_ns_is_readonly(struct nvme_ns *ns, struct nvme_ns_info *info) ··· 1994 1985 is_power_of_2(ctrl->max_hw_sectors)) 1995 1986 iob = ctrl->max_hw_sectors; 1996 1987 else 1997 - iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob)); 1988 + iob = nvme_lba_to_sect(ns->head, le16_to_cpu(id->noiob)); 1998 1989 1999 1990 if (!iob) 2000 1991 return; ··· 2061 2052 2062 2053 blk_mq_freeze_queue(ns->disk->queue); 2063 2054 lbaf = nvme_lbaf_index(id->flbas); 2064 - ns->lba_shift = id->lbaf[lbaf].ds; 2055 + ns->head->lba_shift = id->lbaf[lbaf].ds; 2056 + ns->head->nuse = le64_to_cpu(id->nuse); 2065 2057 nvme_set_queue_limits(ns->ctrl, ns->queue); 2066 2058 2067 - ret = nvme_configure_metadata(ns, id); 2059 + ret = nvme_configure_metadata(ns->ctrl, ns->head, id); 2068 2060 if (ret < 0) { 2069 2061 blk_mq_unfreeze_queue(ns->disk->queue); 2070 2062 goto out; 2071 2063 } 2072 2064 nvme_set_chunk_sectors(ns, id); 2073 - nvme_update_disk_info(ns->disk, ns, id); 2065 + nvme_update_disk_info(ns->ctrl, ns->disk, ns->head, id); 2074 2066 2075 2067 if (ns->head->ids.csi == NVME_CSI_ZNS) { 2076 2068 ret = nvme_update_zone_info(ns, lbaf); ··· 2088 2078 * do not return zeroes. 2089 2079 */ 2090 2080 if ((id->dlfeat & 0x7) == 0x1 && (id->dlfeat & (1 << 3))) 2091 - ns->features |= NVME_NS_DEAC; 2081 + ns->head->features |= NVME_NS_DEAC; 2092 2082 set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info)); 2093 2083 set_bit(NVME_NS_READY, &ns->flags); 2094 2084 blk_mq_unfreeze_queue(ns->disk->queue); ··· 2101 2091 2102 2092 if (nvme_ns_head_multipath(ns->head)) { 2103 2093 blk_mq_freeze_queue(ns->head->disk->queue); 2104 - nvme_update_disk_info(ns->head->disk, ns, id); 2094 + nvme_update_disk_info(ns->ctrl, ns->head->disk, ns->head, id); 2105 2095 set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info)); 2106 2096 nvme_mpath_revalidate_paths(ns); 2107 2097 blk_stack_limits(&ns->head->disk->queue->limits, ··· 3036 3026 return 0; 3037 3027 } 3038 3028 3029 + static int nvme_check_ctrl_fabric_info(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) 3030 + { 3031 + /* 3032 + * In fabrics we need to verify the cntlid matches the 3033 + * admin connect 3034 + */ 3035 + if (ctrl->cntlid != le16_to_cpu(id->cntlid)) { 3036 + dev_err(ctrl->device, 3037 + "Mismatching cntlid: Connect %u vs Identify %u, rejecting\n", 3038 + ctrl->cntlid, le16_to_cpu(id->cntlid)); 3039 + return -EINVAL; 3040 + } 3041 + 3042 + if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) { 3043 + dev_err(ctrl->device, 3044 + "keep-alive support is mandatory for fabrics\n"); 3045 + return -EINVAL; 3046 + } 3047 + 3048 + if (!nvme_discovery_ctrl(ctrl) && ctrl->ioccsz < 4) { 3049 + dev_err(ctrl->device, 3050 + "I/O queue command capsule supported size %d < 4\n", 3051 + ctrl->ioccsz); 3052 + return -EINVAL; 3053 + } 3054 + 3055 + if (!nvme_discovery_ctrl(ctrl) && ctrl->iorcsz < 1) { 3056 + dev_err(ctrl->device, 3057 + "I/O queue response capsule supported size %d < 1\n", 3058 + ctrl->iorcsz); 3059 + return -EINVAL; 3060 + } 3061 + 3062 + return 0; 3063 + } 3064 + 3039 3065 static int nvme_init_identify(struct nvme_ctrl *ctrl) 3040 3066 { 3041 3067 struct nvme_id_ctrl *id; ··· 3184 3138 ctrl->iorcsz = le32_to_cpu(id->iorcsz); 3185 3139 ctrl->maxcmd = le16_to_cpu(id->maxcmd); 3186 3140 3187 - /* 3188 - * In fabrics we need to verify the cntlid matches the 3189 - * admin connect 3190 - */ 3191 - if (ctrl->cntlid != le16_to_cpu(id->cntlid)) { 3192 - dev_err(ctrl->device, 3193 - "Mismatching cntlid: Connect %u vs Identify " 3194 - "%u, rejecting\n", 3195 - ctrl->cntlid, le16_to_cpu(id->cntlid)); 3196 - ret = -EINVAL; 3141 + ret = nvme_check_ctrl_fabric_info(ctrl, id); 3142 + if (ret) 3197 3143 goto out_free; 3198 - } 3199 - 3200 - if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) { 3201 - dev_err(ctrl->device, 3202 - "keep-alive support is mandatory for fabrics\n"); 3203 - ret = -EINVAL; 3204 - goto out_free; 3205 - } 3206 3144 } else { 3207 3145 ctrl->hmpre = le32_to_cpu(id->hmpre); 3208 3146 ctrl->hmmin = le32_to_cpu(id->hmmin); ··· 3445 3415 head->ns_id = info->nsid; 3446 3416 head->ids = info->ids; 3447 3417 head->shared = info->is_shared; 3418 + ratelimit_state_init(&head->rs_nuse, 5 * HZ, 1); 3419 + ratelimit_set_flags(&head->rs_nuse, RATELIMIT_MSG_ON_RELEASE); 3448 3420 kref_init(&head->ref); 3449 3421 3450 3422 if (head->ids.csi) { ··· 3706 3674 up_write(&ctrl->namespaces_rwsem); 3707 3675 nvme_get_ctrl(ctrl); 3708 3676 3709 - if (device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups)) 3677 + if (device_add_disk(ctrl->device, ns->disk, nvme_ns_attr_groups)) 3710 3678 goto out_cleanup_ns_from_list; 3711 3679 3712 3680 if (!nvme_ns_head_multipath(ns->head))

+2 -4

drivers/nvme/host/fc.c

··· 3498 3498 3499 3499 ctrl->ctrl.opts = opts; 3500 3500 ctrl->ctrl.nr_reconnects = 0; 3501 - if (lport->dev) 3502 - ctrl->ctrl.numa_node = dev_to_node(lport->dev); 3503 - else 3504 - ctrl->ctrl.numa_node = NUMA_NO_NODE; 3505 3501 INIT_LIST_HEAD(&ctrl->ctrl_list); 3506 3502 ctrl->lport = lport; 3507 3503 ctrl->rport = rport; ··· 3542 3546 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0); 3543 3547 if (ret) 3544 3548 goto out_free_queues; 3549 + if (lport->dev) 3550 + ctrl->ctrl.numa_node = dev_to_node(lport->dev); 3545 3551 3546 3552 /* at this point, teardown path changes to ref counting on nvme ctrl */ 3547 3553

+34 -173

drivers/nvme/host/ioctl.c

··· 97 97 return (void __user *)ptrval; 98 98 } 99 99 100 - static void *nvme_add_user_metadata(struct request *req, void __user *ubuf, 101 - unsigned len, u32 seed) 102 - { 103 - struct bio_integrity_payload *bip; 104 - int ret = -ENOMEM; 105 - void *buf; 106 - struct bio *bio = req->bio; 107 - 108 - buf = kmalloc(len, GFP_KERNEL); 109 - if (!buf) 110 - goto out; 111 - 112 - if (req_op(req) == REQ_OP_DRV_OUT) { 113 - ret = -EFAULT; 114 - if (copy_from_user(buf, ubuf, len)) 115 - goto out_free_meta; 116 - } else { 117 - memset(buf, 0, len); 118 - } 119 - 120 - bip = bio_integrity_alloc(bio, GFP_KERNEL, 1); 121 - if (IS_ERR(bip)) { 122 - ret = PTR_ERR(bip); 123 - goto out_free_meta; 124 - } 125 - 126 - bip->bip_iter.bi_sector = seed; 127 - ret = bio_integrity_add_page(bio, virt_to_page(buf), len, 128 - offset_in_page(buf)); 129 - if (ret != len) { 130 - ret = -ENOMEM; 131 - goto out_free_meta; 132 - } 133 - 134 - req->cmd_flags |= REQ_INTEGRITY; 135 - return buf; 136 - out_free_meta: 137 - kfree(buf); 138 - out: 139 - return ERR_PTR(ret); 140 - } 141 - 142 - static int nvme_finish_user_metadata(struct request *req, void __user *ubuf, 143 - void *meta, unsigned len, int ret) 144 - { 145 - if (!ret && req_op(req) == REQ_OP_DRV_IN && 146 - copy_to_user(ubuf, meta, len)) 147 - ret = -EFAULT; 148 - kfree(meta); 149 - return ret; 150 - } 151 - 152 100 static struct request *nvme_alloc_user_request(struct request_queue *q, 153 101 struct nvme_command *cmd, blk_opf_t rq_flags, 154 102 blk_mq_req_flags_t blk_flags) ··· 113 165 114 166 static int nvme_map_user_request(struct request *req, u64 ubuffer, 115 167 unsigned bufflen, void __user *meta_buffer, unsigned meta_len, 116 - u32 meta_seed, void **metap, struct io_uring_cmd *ioucmd, 117 - unsigned int flags) 168 + u32 meta_seed, struct io_uring_cmd *ioucmd, unsigned int flags) 118 169 { 119 170 struct request_queue *q = req->q; 120 171 struct nvme_ns *ns = q->queuedata; 121 172 struct block_device *bdev = ns ? ns->disk->part0 : NULL; 122 173 struct bio *bio = NULL; 123 - void *meta = NULL; 124 174 int ret; 125 175 126 176 if (ioucmd && (ioucmd->flags & IORING_URING_CMD_FIXED)) { ··· 140 194 141 195 if (ret) 142 196 goto out; 143 - bio = req->bio; 144 - if (bdev) 145 - bio_set_dev(bio, bdev); 146 197 147 - if (bdev && meta_buffer && meta_len) { 148 - meta = nvme_add_user_metadata(req, meta_buffer, meta_len, 149 - meta_seed); 150 - if (IS_ERR(meta)) { 151 - ret = PTR_ERR(meta); 152 - goto out_unmap; 198 + bio = req->bio; 199 + if (bdev) { 200 + bio_set_dev(bio, bdev); 201 + if (meta_buffer && meta_len) { 202 + ret = bio_integrity_map_user(bio, meta_buffer, meta_len, 203 + meta_seed); 204 + if (ret) 205 + goto out_unmap; 206 + req->cmd_flags |= REQ_INTEGRITY; 153 207 } 154 - *metap = meta; 155 208 } 156 209 157 210 return ret; ··· 171 226 struct nvme_ns *ns = q->queuedata; 172 227 struct nvme_ctrl *ctrl; 173 228 struct request *req; 174 - void *meta = NULL; 175 229 struct bio *bio; 176 230 u32 effects; 177 231 int ret; ··· 182 238 req->timeout = timeout; 183 239 if (ubuffer && bufflen) { 184 240 ret = nvme_map_user_request(req, ubuffer, bufflen, meta_buffer, 185 - meta_len, meta_seed, &meta, NULL, flags); 241 + meta_len, meta_seed, NULL, flags); 186 242 if (ret) 187 243 return ret; 188 244 } ··· 194 250 ret = nvme_execute_rq(req, false); 195 251 if (result) 196 252 *result = le64_to_cpu(nvme_req(req)->result.u64); 197 - if (meta) 198 - ret = nvme_finish_user_metadata(req, meta_buffer, meta, 199 - meta_len, ret); 200 253 if (bio) 201 254 blk_rq_unmap_user(bio); 202 255 blk_mq_free_request(req); ··· 225 284 return -EINVAL; 226 285 } 227 286 228 - length = (io.nblocks + 1) << ns->lba_shift; 287 + length = (io.nblocks + 1) << ns->head->lba_shift; 229 288 230 289 if ((io.control & NVME_RW_PRINFO_PRACT) && 231 - ns->ms == sizeof(struct t10_pi_tuple)) { 290 + ns->head->ms == sizeof(struct t10_pi_tuple)) { 232 291 /* 233 292 * Protection information is stripped/inserted by the 234 293 * controller. ··· 238 297 meta_len = 0; 239 298 metadata = NULL; 240 299 } else { 241 - meta_len = (io.nblocks + 1) * ns->ms; 300 + meta_len = (io.nblocks + 1) * ns->head->ms; 242 301 metadata = nvme_to_user_ptr(io.metadata); 243 302 } 244 303 245 - if (ns->features & NVME_NS_EXT_LBAS) { 304 + if (ns->head->features & NVME_NS_EXT_LBAS) { 246 305 length += meta_len; 247 306 meta_len = 0; 248 307 } else if (meta_len) { ··· 388 447 * Expect build errors if this grows larger than that. 389 448 */ 390 449 struct nvme_uring_cmd_pdu { 391 - union { 392 - struct bio *bio; 393 - struct request *req; 394 - }; 395 - u32 meta_len; 396 - u32 nvme_status; 397 - union { 398 - struct { 399 - void *meta; /* kernel-resident buffer */ 400 - void __user *meta_buffer; 401 - }; 402 - u64 result; 403 - } u; 450 + struct request *req; 451 + struct bio *bio; 452 + u64 result; 453 + int status; 404 454 }; 405 455 406 456 static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu( 407 457 struct io_uring_cmd *ioucmd) 408 458 { 409 459 return (struct nvme_uring_cmd_pdu *)&ioucmd->pdu; 410 - } 411 - 412 - static void nvme_uring_task_meta_cb(struct io_uring_cmd *ioucmd, 413 - unsigned issue_flags) 414 - { 415 - struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); 416 - struct request *req = pdu->req; 417 - int status; 418 - u64 result; 419 - 420 - if (nvme_req(req)->flags & NVME_REQ_CANCELLED) 421 - status = -EINTR; 422 - else 423 - status = nvme_req(req)->status; 424 - 425 - result = le64_to_cpu(nvme_req(req)->result.u64); 426 - 427 - if (pdu->meta_len) 428 - status = nvme_finish_user_metadata(req, pdu->u.meta_buffer, 429 - pdu->u.meta, pdu->meta_len, status); 430 - if (req->bio) 431 - blk_rq_unmap_user(req->bio); 432 - blk_mq_free_request(req); 433 - 434 - io_uring_cmd_done(ioucmd, status, result, issue_flags); 435 460 } 436 461 437 462 static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd, ··· 407 500 408 501 if (pdu->bio) 409 502 blk_rq_unmap_user(pdu->bio); 410 - 411 - io_uring_cmd_done(ioucmd, pdu->nvme_status, pdu->u.result, issue_flags); 503 + io_uring_cmd_done(ioucmd, pdu->status, pdu->result, issue_flags); 412 504 } 413 505 414 506 static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req, ··· 416 510 struct io_uring_cmd *ioucmd = req->end_io_data; 417 511 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); 418 512 419 - req->bio = pdu->bio; 420 - if (nvme_req(req)->flags & NVME_REQ_CANCELLED) { 421 - pdu->nvme_status = -EINTR; 422 - } else { 423 - pdu->nvme_status = nvme_req(req)->status; 424 - if (!pdu->nvme_status) 425 - pdu->nvme_status = blk_status_to_errno(err); 426 - } 427 - pdu->u.result = le64_to_cpu(nvme_req(req)->result.u64); 513 + if (nvme_req(req)->flags & NVME_REQ_CANCELLED) 514 + pdu->status = -EINTR; 515 + else 516 + pdu->status = nvme_req(req)->status; 517 + pdu->result = le64_to_cpu(nvme_req(req)->result.u64); 428 518 429 519 /* 430 520 * For iopoll, complete it directly. 431 521 * Otherwise, move the completion to task work. 432 522 */ 433 - if (blk_rq_is_poll(req)) { 434 - WRITE_ONCE(ioucmd->cookie, NULL); 523 + if (blk_rq_is_poll(req)) 435 524 nvme_uring_task_cb(ioucmd, IO_URING_F_UNLOCKED); 436 - } else { 525 + else 437 526 io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb); 438 - } 439 527 440 528 return RQ_END_IO_FREE; 441 - } 442 - 443 - static enum rq_end_io_ret nvme_uring_cmd_end_io_meta(struct request *req, 444 - blk_status_t err) 445 - { 446 - struct io_uring_cmd *ioucmd = req->end_io_data; 447 - struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); 448 - 449 - req->bio = pdu->bio; 450 - pdu->req = req; 451 - 452 - /* 453 - * For iopoll, complete it directly. 454 - * Otherwise, move the completion to task work. 455 - */ 456 - if (blk_rq_is_poll(req)) { 457 - WRITE_ONCE(ioucmd->cookie, NULL); 458 - nvme_uring_task_meta_cb(ioucmd, IO_URING_F_UNLOCKED); 459 - } else { 460 - io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_meta_cb); 461 - } 462 - 463 - return RQ_END_IO_NONE; 464 529 } 465 530 466 531 static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, ··· 445 568 struct request *req; 446 569 blk_opf_t rq_flags = REQ_ALLOC_CACHE; 447 570 blk_mq_req_flags_t blk_flags = 0; 448 - void *meta = NULL; 449 571 int ret; 450 572 451 573 c.common.opcode = READ_ONCE(cmd->opcode); ··· 492 616 if (d.addr && d.data_len) { 493 617 ret = nvme_map_user_request(req, d.addr, 494 618 d.data_len, nvme_to_user_ptr(d.metadata), 495 - d.metadata_len, 0, &meta, ioucmd, vec); 619 + d.metadata_len, 0, ioucmd, vec); 496 620 if (ret) 497 621 return ret; 498 622 } 499 623 500 - if (blk_rq_is_poll(req)) { 501 - ioucmd->flags |= IORING_URING_CMD_POLLED; 502 - WRITE_ONCE(ioucmd->cookie, req); 503 - } 504 - 505 624 /* to free bio on completion, as req->bio will be null at that time */ 506 625 pdu->bio = req->bio; 507 - pdu->meta_len = d.metadata_len; 626 + pdu->req = req; 508 627 req->end_io_data = ioucmd; 509 - if (pdu->meta_len) { 510 - pdu->u.meta = meta; 511 - pdu->u.meta_buffer = nvme_to_user_ptr(d.metadata); 512 - req->end_io = nvme_uring_cmd_end_io_meta; 513 - } else { 514 - req->end_io = nvme_uring_cmd_end_io; 515 - } 628 + req->end_io = nvme_uring_cmd_end_io; 516 629 blk_execute_rq_nowait(req, false); 517 630 return -EIOCBQUEUED; 518 631 } ··· 652 787 struct io_comp_batch *iob, 653 788 unsigned int poll_flags) 654 789 { 655 - struct request *req; 656 - int ret = 0; 790 + struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); 791 + struct request *req = pdu->req; 657 792 658 - if (!(ioucmd->flags & IORING_URING_CMD_POLLED)) 659 - return 0; 660 - 661 - req = READ_ONCE(ioucmd->cookie); 662 793 if (req && blk_rq_is_poll(req)) 663 - ret = blk_rq_poll(req, iob, poll_flags); 664 - return ret; 794 + return blk_rq_poll(req, iob, poll_flags); 795 + return 0; 665 796 } 666 797 #ifdef CONFIG_NVME_MULTIPATH 667 798 static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,

+1 -1

drivers/nvme/host/multipath.c

··· 579 579 */ 580 580 if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 581 581 rc = device_add_disk(&head->subsys->dev, head->disk, 582 - nvme_ns_id_attr_groups); 582 + nvme_ns_attr_groups); 583 583 if (rc) { 584 584 clear_bit(NVME_NSHEAD_DISK_LIVE, &ns->flags); 585 585 return;

+25 -19

drivers/nvme/host/nvme.h

··· 16 16 #include <linux/rcupdate.h> 17 17 #include <linux/wait.h> 18 18 #include <linux/t10-pi.h> 19 + #include <linux/ratelimit_types.h> 19 20 20 21 #include <trace/events/block.h> 21 22 ··· 451 450 struct list_head list; 452 451 struct srcu_struct srcu; 453 452 struct nvme_subsystem *subsys; 454 - unsigned ns_id; 455 453 struct nvme_ns_ids ids; 456 454 struct list_head entry; 457 455 struct kref ref; 458 456 bool shared; 459 457 int instance; 460 458 struct nvme_effects_log *effects; 459 + u64 nuse; 460 + unsigned ns_id; 461 + int lba_shift; 462 + u16 ms; 463 + u16 pi_size; 464 + u8 pi_type; 465 + u8 guard_type; 466 + u16 sgs; 467 + u32 sws; 468 + #ifdef CONFIG_BLK_DEV_ZONED 469 + u64 zsze; 470 + #endif 471 + unsigned long features; 472 + 473 + struct ratelimit_state rs_nuse; 461 474 462 475 struct cdev cdev; 463 476 struct device cdev_device; ··· 513 498 struct kref kref; 514 499 struct nvme_ns_head *head; 515 500 516 - int lba_shift; 517 - u16 ms; 518 - u16 pi_size; 519 - u16 sgs; 520 - u32 sws; 521 - u8 pi_type; 522 - u8 guard_type; 523 - #ifdef CONFIG_BLK_DEV_ZONED 524 - u64 zsze; 525 - #endif 526 - unsigned long features; 527 501 unsigned long flags; 528 502 #define NVME_NS_REMOVING 0 529 503 #define NVME_NS_ANA_PENDING 2 ··· 527 523 }; 528 524 529 525 /* NVMe ns supports metadata actions by the controller (generate/strip) */ 530 - static inline bool nvme_ns_has_pi(struct nvme_ns *ns) 526 + static inline bool nvme_ns_has_pi(struct nvme_ns_head *head) 531 527 { 532 - return ns->pi_type && ns->ms == ns->pi_size; 528 + return head->pi_type && head->ms == head->pi_size; 533 529 } 534 530 535 531 struct nvme_ctrl_ops { ··· 661 657 /* 662 658 * Convert a 512B sector number to a device logical block number. 663 659 */ 664 - static inline u64 nvme_sect_to_lba(struct nvme_ns *ns, sector_t sector) 660 + static inline u64 nvme_sect_to_lba(struct nvme_ns_head *head, sector_t sector) 665 661 { 666 - return sector >> (ns->lba_shift - SECTOR_SHIFT); 662 + return sector >> (head->lba_shift - SECTOR_SHIFT); 667 663 } 668 664 669 665 /* 670 666 * Convert a device logical block number to a 512B sector number. 671 667 */ 672 - static inline sector_t nvme_lba_to_sect(struct nvme_ns *ns, u64 lba) 668 + static inline sector_t nvme_lba_to_sect(struct nvme_ns_head *head, u64 lba) 673 669 { 674 - return lba << (ns->lba_shift - SECTOR_SHIFT); 670 + return lba << (head->lba_shift - SECTOR_SHIFT); 675 671 } 676 672 677 673 /* ··· 877 873 unsigned int issue_flags); 878 874 int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd, 879 875 unsigned int issue_flags); 876 + int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid, 877 + struct nvme_id_ns **id); 880 878 int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo); 881 879 int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags); 882 880 883 - extern const struct attribute_group *nvme_ns_id_attr_groups[]; 881 + extern const struct attribute_group *nvme_ns_attr_groups[]; 884 882 extern const struct pr_ops nvme_pr_ops; 885 883 extern const struct block_device_operations nvme_ns_head_ops; 886 884 extern const struct attribute_group nvme_dev_attrs_group;

+2 -2

drivers/nvme/host/rdma.c

··· 1423 1423 goto mr_put; 1424 1424 1425 1425 nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_bdev->bd_disk), c, 1426 - req->mr->sig_attrs, ns->pi_type); 1426 + req->mr->sig_attrs, ns->head->pi_type); 1427 1427 nvme_rdma_set_prot_checks(c, &req->mr->sig_attrs->check_mask); 1428 1428 1429 1429 ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey)); ··· 2017 2017 queue->pi_support && 2018 2018 (c->common.opcode == nvme_cmd_write || 2019 2019 c->common.opcode == nvme_cmd_read) && 2020 - nvme_ns_has_pi(ns)) 2020 + nvme_ns_has_pi(ns->head)) 2021 2021 req->use_sig_mr = true; 2022 2022 else 2023 2023 req->use_sig_mr = false;

+92 -7

drivers/nvme/host/sysfs.c

··· 114 114 } 115 115 static DEVICE_ATTR_RO(nsid); 116 116 117 - static struct attribute *nvme_ns_id_attrs[] = { 117 + static ssize_t csi_show(struct device *dev, struct device_attribute *attr, 118 + char *buf) 119 + { 120 + return sysfs_emit(buf, "%u\n", dev_to_ns_head(dev)->ids.csi); 121 + } 122 + static DEVICE_ATTR_RO(csi); 123 + 124 + static ssize_t metadata_bytes_show(struct device *dev, 125 + struct device_attribute *attr, char *buf) 126 + { 127 + return sysfs_emit(buf, "%u\n", dev_to_ns_head(dev)->ms); 128 + } 129 + static DEVICE_ATTR_RO(metadata_bytes); 130 + 131 + static int ns_head_update_nuse(struct nvme_ns_head *head) 132 + { 133 + struct nvme_id_ns *id; 134 + struct nvme_ns *ns; 135 + int srcu_idx, ret = -EWOULDBLOCK; 136 + 137 + /* Avoid issuing commands too often by rate limiting the update */ 138 + if (!__ratelimit(&head->rs_nuse)) 139 + return 0; 140 + 141 + srcu_idx = srcu_read_lock(&head->srcu); 142 + ns = nvme_find_path(head); 143 + if (!ns) 144 + goto out_unlock; 145 + 146 + ret = nvme_identify_ns(ns->ctrl, head->ns_id, &id); 147 + if (ret) 148 + goto out_unlock; 149 + 150 + head->nuse = le64_to_cpu(id->nuse); 151 + kfree(id); 152 + 153 + out_unlock: 154 + srcu_read_unlock(&head->srcu, srcu_idx); 155 + return ret; 156 + } 157 + 158 + static int ns_update_nuse(struct nvme_ns *ns) 159 + { 160 + struct nvme_id_ns *id; 161 + int ret; 162 + 163 + /* Avoid issuing commands too often by rate limiting the update. */ 164 + if (!__ratelimit(&ns->head->rs_nuse)) 165 + return 0; 166 + 167 + ret = nvme_identify_ns(ns->ctrl, ns->head->ns_id, &id); 168 + if (ret) 169 + goto out_free_id; 170 + 171 + ns->head->nuse = le64_to_cpu(id->nuse); 172 + 173 + out_free_id: 174 + kfree(id); 175 + 176 + return ret; 177 + } 178 + 179 + static ssize_t nuse_show(struct device *dev, struct device_attribute *attr, 180 + char *buf) 181 + { 182 + struct nvme_ns_head *head = dev_to_ns_head(dev); 183 + struct gendisk *disk = dev_to_disk(dev); 184 + struct block_device *bdev = disk->part0; 185 + int ret; 186 + 187 + if (IS_ENABLED(CONFIG_NVME_MULTIPATH) && 188 + bdev->bd_disk->fops == &nvme_ns_head_ops) 189 + ret = ns_head_update_nuse(head); 190 + else 191 + ret = ns_update_nuse(bdev->bd_disk->private_data); 192 + if (ret) 193 + return ret; 194 + 195 + return sysfs_emit(buf, "%llu\n", head->nuse); 196 + } 197 + static DEVICE_ATTR_RO(nuse); 198 + 199 + static struct attribute *nvme_ns_attrs[] = { 118 200 &dev_attr_wwid.attr, 119 201 &dev_attr_uuid.attr, 120 202 &dev_attr_nguid.attr, 121 203 &dev_attr_eui.attr, 204 + &dev_attr_csi.attr, 122 205 &dev_attr_nsid.attr, 206 + &dev_attr_metadata_bytes.attr, 207 + &dev_attr_nuse.attr, 123 208 #ifdef CONFIG_NVME_MULTIPATH 124 209 &dev_attr_ana_grpid.attr, 125 210 &dev_attr_ana_state.attr, ··· 212 127 NULL, 213 128 }; 214 129 215 - static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj, 130 + static umode_t nvme_ns_attrs_are_visible(struct kobject *kobj, 216 131 struct attribute *a, int n) 217 132 { 218 133 struct device *dev = container_of(kobj, struct device, kobj); ··· 242 157 return a->mode; 243 158 } 244 159 245 - static const struct attribute_group nvme_ns_id_attr_group = { 246 - .attrs = nvme_ns_id_attrs, 247 - .is_visible = nvme_ns_id_attrs_are_visible, 160 + static const struct attribute_group nvme_ns_attr_group = { 161 + .attrs = nvme_ns_attrs, 162 + .is_visible = nvme_ns_attrs_are_visible, 248 163 }; 249 164 250 - const struct attribute_group *nvme_ns_id_attr_groups[] = { 251 - &nvme_ns_id_attr_group, 165 + const struct attribute_group *nvme_ns_attr_groups[] = { 166 + &nvme_ns_attr_group, 252 167 NULL, 253 168 }; 254 169

+20 -17

drivers/nvme/host/zns.c

··· 11 11 { 12 12 struct request_queue *q = ns->queue; 13 13 14 - blk_queue_chunk_sectors(q, ns->zsze); 14 + blk_queue_chunk_sectors(q, ns->head->zsze); 15 15 blk_queue_max_zone_append_sectors(q, ns->ctrl->max_zone_append); 16 16 17 17 return blk_revalidate_disk_zones(ns->disk, NULL); ··· 99 99 goto free_data; 100 100 } 101 101 102 - ns->zsze = nvme_lba_to_sect(ns, le64_to_cpu(id->lbafe[lbaf].zsze)); 103 - if (!is_power_of_2(ns->zsze)) { 102 + ns->head->zsze = 103 + nvme_lba_to_sect(ns->head, le64_to_cpu(id->lbafe[lbaf].zsze)); 104 + if (!is_power_of_2(ns->head->zsze)) { 104 105 dev_warn(ns->ctrl->device, 105 106 "invalid zone size:%llu for namespace:%u\n", 106 - ns->zsze, ns->head->ns_id); 107 + ns->head->zsze, ns->head->ns_id); 107 108 status = -ENODEV; 108 109 goto free_data; 109 110 } 110 111 111 - disk_set_zoned(ns->disk, BLK_ZONED_HM); 112 + disk_set_zoned(ns->disk); 112 113 blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); 113 114 disk_set_max_open_zones(ns->disk, le32_to_cpu(id->mor) + 1); 114 115 disk_set_max_active_zones(ns->disk, le32_to_cpu(id->mar) + 1); ··· 129 128 sizeof(struct nvme_zone_descriptor); 130 129 131 130 nr_zones = min_t(unsigned int, nr_zones, 132 - get_capacity(ns->disk) >> ilog2(ns->zsze)); 131 + get_capacity(ns->disk) >> ilog2(ns->head->zsze)); 133 132 134 133 bufsize = sizeof(struct nvme_zone_report) + 135 134 nr_zones * sizeof(struct nvme_zone_descriptor); ··· 148 147 return NULL; 149 148 } 150 149 151 - static int nvme_zone_parse_entry(struct nvme_ns *ns, 150 + static int nvme_zone_parse_entry(struct nvme_ctrl *ctrl, 151 + struct nvme_ns_head *head, 152 152 struct nvme_zone_descriptor *entry, 153 153 unsigned int idx, report_zones_cb cb, 154 154 void *data) ··· 157 155 struct blk_zone zone = { }; 158 156 159 157 if ((entry->zt & 0xf) != NVME_ZONE_TYPE_SEQWRITE_REQ) { 160 - dev_err(ns->ctrl->device, "invalid zone type %#x\n", 158 + dev_err(ctrl->device, "invalid zone type %#x\n", 161 159 entry->zt); 162 160 return -EINVAL; 163 161 } 164 162 165 163 zone.type = BLK_ZONE_TYPE_SEQWRITE_REQ; 166 164 zone.cond = entry->zs >> 4; 167 - zone.len = ns->zsze; 168 - zone.capacity = nvme_lba_to_sect(ns, le64_to_cpu(entry->zcap)); 169 - zone.start = nvme_lba_to_sect(ns, le64_to_cpu(entry->zslba)); 165 + zone.len = head->zsze; 166 + zone.capacity = nvme_lba_to_sect(head, le64_to_cpu(entry->zcap)); 167 + zone.start = nvme_lba_to_sect(head, le64_to_cpu(entry->zslba)); 170 168 if (zone.cond == BLK_ZONE_COND_FULL) 171 169 zone.wp = zone.start + zone.len; 172 170 else 173 - zone.wp = nvme_lba_to_sect(ns, le64_to_cpu(entry->wp)); 171 + zone.wp = nvme_lba_to_sect(head, le64_to_cpu(entry->wp)); 174 172 175 173 return cb(&zone, idx, data); 176 174 } ··· 198 196 c.zmr.zrasf = NVME_ZRASF_ZONE_REPORT_ALL; 199 197 c.zmr.pr = NVME_REPORT_ZONE_PARTIAL; 200 198 201 - sector &= ~(ns->zsze - 1); 199 + sector &= ~(ns->head->zsze - 1); 202 200 while (zone_idx < nr_zones && sector < get_capacity(ns->disk)) { 203 201 memset(report, 0, buflen); 204 202 205 - c.zmr.slba = cpu_to_le64(nvme_sect_to_lba(ns, sector)); 203 + c.zmr.slba = cpu_to_le64(nvme_sect_to_lba(ns->head, sector)); 206 204 ret = nvme_submit_sync_cmd(ns->queue, &c, report, buflen); 207 205 if (ret) { 208 206 if (ret > 0) ··· 215 213 break; 216 214 217 215 for (i = 0; i < nz && zone_idx < nr_zones; i++) { 218 - ret = nvme_zone_parse_entry(ns, &report->entries[i], 216 + ret = nvme_zone_parse_entry(ns->ctrl, ns->head, 217 + &report->entries[i], 219 218 zone_idx, cb, data); 220 219 if (ret) 221 220 goto out_free; 222 221 zone_idx++; 223 222 } 224 223 225 - sector += ns->zsze * nz; 224 + sector += ns->head->zsze * nz; 226 225 } 227 226 228 227 if (zone_idx > 0) ··· 242 239 243 240 c->zms.opcode = nvme_cmd_zone_mgmt_send; 244 241 c->zms.nsid = cpu_to_le32(ns->head->ns_id); 245 - c->zms.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req))); 242 + c->zms.slba = cpu_to_le64(nvme_sect_to_lba(ns->head, blk_rq_pos(req))); 246 243 c->zms.zsa = action; 247 244 248 245 if (req_op(req) == REQ_OP_ZONE_RESET_ALL)

+2 -2

drivers/nvme/target/configfs.c

··· 1276 1276 return -EINVAL; 1277 1277 1278 1278 down_write(&nvmet_config_sem); 1279 - if (cntlid_min >= to_subsys(item)->cntlid_max) 1279 + if (cntlid_min > to_subsys(item)->cntlid_max) 1280 1280 goto out_unlock; 1281 1281 to_subsys(item)->cntlid_min = cntlid_min; 1282 1282 up_write(&nvmet_config_sem); ··· 1306 1306 return -EINVAL; 1307 1307 1308 1308 down_write(&nvmet_config_sem); 1309 - if (cntlid_max <= to_subsys(item)->cntlid_min) 1309 + if (cntlid_max < to_subsys(item)->cntlid_min) 1310 1310 goto out_unlock; 1311 1311 to_subsys(item)->cntlid_max = cntlid_max; 1312 1312 up_write(&nvmet_config_sem);

-3

drivers/nvme/target/core.c

··· 1425 1425 if (!ctrl->sqs) 1426 1426 goto out_free_changed_ns_list; 1427 1427 1428 - if (subsys->cntlid_min > subsys->cntlid_max) 1429 - goto out_free_sqs; 1430 - 1431 1428 ret = ida_alloc_range(&cntlid_ida, 1432 1429 subsys->cntlid_min, subsys->cntlid_max, 1433 1430 GFP_KERNEL);

+2 -2

drivers/nvme/target/passthru.c

··· 602 602 goto out_put_file; 603 603 } 604 604 605 - old = xa_cmpxchg(&passthru_subsystems, ctrl->cntlid, NULL, 605 + old = xa_cmpxchg(&passthru_subsystems, ctrl->instance, NULL, 606 606 subsys, GFP_KERNEL); 607 607 if (xa_is_err(old)) { 608 608 ret = xa_err(old); ··· 635 635 static void __nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys) 636 636 { 637 637 if (subsys->passthru_ctrl) { 638 - xa_erase(&passthru_subsystems, subsys->passthru_ctrl->cntlid); 638 + xa_erase(&passthru_subsystems, subsys->passthru_ctrl->instance); 639 639 module_put(subsys->passthru_ctrl->ops->module); 640 640 nvme_put_ctrl(subsys->passthru_ctrl); 641 641 }

+13 -14

drivers/scsi/scsi_debug.c

··· 339 339 bool used; 340 340 341 341 /* For ZBC devices */ 342 - enum blk_zoned_model zmodel; 342 + bool zoned; 343 343 unsigned int zcap; 344 344 unsigned int zsize; 345 345 unsigned int zsize_shift; ··· 844 844 static bool sdebug_statistics = DEF_STATISTICS; 845 845 static bool sdebug_wp; 846 846 static bool sdebug_allow_restart; 847 - /* Following enum: 0: no zbc, def; 1: host aware; 2: host managed */ 848 - static enum blk_zoned_model sdeb_zbc_model = BLK_ZONED_NONE; 847 + static enum { 848 + BLK_ZONED_NONE = 0, 849 + BLK_ZONED_HA = 1, 850 + BLK_ZONED_HM = 2, 851 + } sdeb_zbc_model = BLK_ZONED_NONE; 849 852 static char *sdeb_zbc_model_s; 850 853 851 854 enum sam_lun_addr_method {SAM_LUN_AM_PERIPHERAL = 0x0, ··· 1818 1815 arr[1] = 1; /* non rotating medium (e.g. solid state) */ 1819 1816 arr[2] = 0; 1820 1817 arr[3] = 5; /* less than 1.8" */ 1821 - if (devip->zmodel == BLK_ZONED_HA) 1822 - arr[4] = 1 << 4; /* zoned field = 01b */ 1823 1818 1824 1819 return 0x3c; 1825 1820 } ··· 1884 1883 if (! arr) 1885 1884 return DID_REQUEUE << 16; 1886 1885 is_disk = (sdebug_ptype == TYPE_DISK); 1887 - is_zbc = (devip->zmodel != BLK_ZONED_NONE); 1886 + is_zbc = devip->zoned; 1888 1887 is_disk_zbc = (is_disk || is_zbc); 1889 1888 have_wlun = scsi_is_wlun(scp->device->lun); 1890 1889 if (have_wlun) ··· 2196 2195 * Since the scsi_debug READ CAPACITY implementation always reports the 2197 2196 * total disk capacity, set RC BASIS = 1 for host-managed ZBC devices. 2198 2197 */ 2199 - if (devip->zmodel == BLK_ZONED_HM) 2198 + if (devip->zoned) 2200 2199 arr[12] |= 1 << 4; 2201 2200 2202 2201 arr[15] = sdebug_lowest_aligned & 0xff; ··· 2649 2648 msense_6 = (MODE_SENSE == cmd[0]); 2650 2649 llbaa = msense_6 ? false : !!(cmd[1] & 0x10); 2651 2650 is_disk = (sdebug_ptype == TYPE_DISK); 2652 - is_zbc = (devip->zmodel != BLK_ZONED_NONE); 2651 + is_zbc = devip->zoned; 2653 2652 if ((is_disk || is_zbc) && !dbd) 2654 2653 bd_len = llbaa ? 16 : 8; 2655 2654 else ··· 3195 3194 struct sdeb_zone_state *zsp_end = zbc_zone(devip, lba + num - 1); 3196 3195 3197 3196 if (!write) { 3198 - if (devip->zmodel == BLK_ZONED_HA) 3199 - return 0; 3200 3197 /* For host-managed, reads cannot cross zone types boundaries */ 3201 3198 if (zsp->z_type != zsp_end->z_type) { 3202 3199 mk_sense_buffer(scp, ILLEGAL_REQUEST, ··· 5321 5322 if (devip->zcap < devip->zsize) 5322 5323 devip->nr_zones += devip->nr_seq_zones; 5323 5324 5324 - if (devip->zmodel == BLK_ZONED_HM) { 5325 + if (devip->zoned) { 5325 5326 /* zbc_max_open_zones can be 0, meaning "not reported" */ 5326 5327 if (sdeb_zbc_max_open >= devip->nr_zones - 1) 5327 5328 devip->max_open = (devip->nr_zones - 1) / 2; ··· 5346 5347 zsp->z_size = 5347 5348 min_t(u64, devip->zsize, capacity - zstart); 5348 5349 } else if ((zstart & (devip->zsize - 1)) == 0) { 5349 - if (devip->zmodel == BLK_ZONED_HM) 5350 + if (devip->zoned) 5350 5351 zsp->z_type = ZBC_ZTYPE_SWR; 5351 5352 else 5352 5353 zsp->z_type = ZBC_ZTYPE_SWP; ··· 5389 5390 } 5390 5391 devip->sdbg_host = sdbg_host; 5391 5392 if (sdeb_zbc_in_use) { 5392 - devip->zmodel = sdeb_zbc_model; 5393 + devip->zoned = sdeb_zbc_model == BLK_ZONED_HM; 5393 5394 if (sdebug_device_create_zones(devip)) { 5394 5395 kfree(devip); 5395 5396 return NULL; 5396 5397 } 5397 5398 } else { 5398 - devip->zmodel = BLK_ZONED_NONE; 5399 + devip->zoned = false; 5399 5400 } 5400 5401 devip->create_ts = ktime_get_boottime(); 5401 5402 atomic_set(&devip->stopped, (sdeb_tur_ms_to_ready > 0 ? 2 : 0));

+23 -26

drivers/scsi/sd.c

··· 3117 3117 struct request_queue *q = sdkp->disk->queue; 3118 3118 struct scsi_vpd *vpd; 3119 3119 u16 rot; 3120 - u8 zoned; 3121 3120 3122 3121 rcu_read_lock(); 3123 3122 vpd = rcu_dereference(sdkp->device->vpd_pgb1); ··· 3127 3128 } 3128 3129 3129 3130 rot = get_unaligned_be16(&vpd->data[4]); 3130 - zoned = (vpd->data[8] >> 4) & 3; 3131 + sdkp->zoned = (vpd->data[8] >> 4) & 3; 3131 3132 rcu_read_unlock(); 3132 3133 3133 3134 if (rot == 1) { ··· 3135 3136 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, q); 3136 3137 } 3137 3138 3139 + 3140 + #ifdef CONFIG_BLK_DEV_ZONED /* sd_probe rejects ZBD devices early otherwise */ 3138 3141 if (sdkp->device->type == TYPE_ZBC) { 3139 3142 /* 3140 - * Host-managed: Per ZBC and ZAC specifications, writes in 3141 - * sequential write required zones of host-managed devices must 3142 - * be aligned to the device physical block size. 3143 + * Host-managed. 3143 3144 */ 3144 - disk_set_zoned(sdkp->disk, BLK_ZONED_HM); 3145 + disk_set_zoned(sdkp->disk); 3146 + 3147 + /* 3148 + * Per ZBC and ZAC specifications, writes in sequential write 3149 + * required zones of host-managed devices must be aligned to 3150 + * the device physical block size. 3151 + */ 3145 3152 blk_queue_zone_write_granularity(q, sdkp->physical_block_size); 3146 3153 } else { 3147 - sdkp->zoned = zoned; 3148 - if (sdkp->zoned == 1) { 3149 - /* Host-aware */ 3150 - disk_set_zoned(sdkp->disk, BLK_ZONED_HA); 3151 - } else { 3152 - /* Regular disk or drive managed disk */ 3153 - disk_set_zoned(sdkp->disk, BLK_ZONED_NONE); 3154 - } 3154 + /* 3155 + * Host-aware devices are treated as conventional. 3156 + */ 3157 + WARN_ON_ONCE(blk_queue_is_zoned(q)); 3155 3158 } 3159 + #endif /* CONFIG_BLK_DEV_ZONED */ 3156 3160 3157 3161 if (!sdkp->first_scan) 3158 3162 return; 3159 3163 3160 - if (blk_queue_is_zoned(q)) { 3161 - sd_printk(KERN_NOTICE, sdkp, "Host-%s zoned block device\n", 3162 - q->limits.zoned == BLK_ZONED_HM ? "managed" : "aware"); 3163 - } else { 3164 - if (sdkp->zoned == 1) 3165 - sd_printk(KERN_NOTICE, sdkp, 3166 - "Host-aware SMR disk used as regular disk\n"); 3167 - else if (sdkp->zoned == 2) 3168 - sd_printk(KERN_NOTICE, sdkp, 3169 - "Drive-managed SMR disk\n"); 3170 - } 3164 + if (blk_queue_is_zoned(q)) 3165 + sd_printk(KERN_NOTICE, sdkp, "Host-managed zoned block device\n"); 3166 + else if (sdkp->zoned == 1) 3167 + sd_printk(KERN_NOTICE, sdkp, "Host-aware SMR disk used as regular disk\n"); 3168 + else if (sdkp->zoned == 2) 3169 + sd_printk(KERN_NOTICE, sdkp, "Drive-managed SMR disk\n"); 3171 3170 } 3172 3171 3173 3172 /** ··· 3499 3502 } else { 3500 3503 q->limits.io_opt = 0; 3501 3504 rw_max = min_not_zero(logical_to_sectors(sdp, dev_max), 3502 - (sector_t)BLK_DEF_MAX_SECTORS); 3505 + (sector_t)BLK_DEF_MAX_SECTORS_CAP); 3503 3506 } 3504 3507 3505 3508 /*

+1 -15

drivers/scsi/sd_zbc.c

··· 836 836 837 837 /* 838 838 * For all zoned disks, initialize zone append emulation data if not 839 - * already done. This is necessary also for host-aware disks used as 840 - * regular disks due to the presence of partitions as these partitions 841 - * may be deleted and the disk zoned model changed back from 842 - * BLK_ZONED_NONE to BLK_ZONED_HA. 839 + * already done. 843 840 */ 844 841 if (sd_is_zoned(sdkp) && !sdkp->zone_wp_update_buf) { 845 842 ret = sd_zbc_init_disk(sdkp); ··· 928 931 sdkp->device->use_16_for_rw = 1; 929 932 sdkp->device->use_10_for_rw = 0; 930 933 sdkp->device->use_16_for_sync = 1; 931 - 932 - if (!blk_queue_is_zoned(q)) { 933 - /* 934 - * This can happen for a host aware disk with partitions. 935 - * The block device zone model was already cleared by 936 - * disk_set_zoned(). Only free the scsi disk zone 937 - * information and exit early. 938 - */ 939 - sd_zbc_free_zone_info(sdkp); 940 - return 0; 941 - } 942 934 943 935 /* Check zoned block device characteristics (unconstrained reads) */ 944 936 ret = sd_zbc_check_zoned_characteristics(sdkp, buf);

+3 -20

fs/btrfs/zoned.c

··· 578 578 579 579 kvfree(zones); 580 580 581 - switch (bdev_zoned_model(bdev)) { 582 - case BLK_ZONED_HM: 581 + if (bdev_is_zoned(bdev)) { 583 582 model = "host-managed zoned"; 584 583 emulated = ""; 585 - break; 586 - case BLK_ZONED_HA: 587 - model = "host-aware zoned"; 588 - emulated = ""; 589 - break; 590 - case BLK_ZONED_NONE: 584 + } else { 591 585 model = "regular"; 592 586 emulated = "emulated "; 593 - break; 594 - default: 595 - /* Just in case */ 596 - btrfs_err_in_rcu(fs_info, "zoned: unsupported model %d on %s", 597 - bdev_zoned_model(bdev), 598 - rcu_str_deref(device->name)); 599 - ret = -EOPNOTSUPP; 600 - goto out_free_zone_info; 601 587 } 602 588 603 589 btrfs_info_in_rcu(fs_info, ··· 595 609 596 610 out: 597 611 kvfree(zones); 598 - out_free_zone_info: 599 612 btrfs_destroy_dev_zone_info(device); 600 - 601 613 return ret; 602 614 } 603 615 ··· 672 688 struct btrfs_device *device; 673 689 674 690 list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { 675 - if (device->bdev && 676 - bdev_zoned_model(device->bdev) == BLK_ZONED_HM) { 691 + if (device->bdev && bdev_is_zoned(device->bdev)) { 677 692 btrfs_err(fs_info, 678 693 "zoned: mode not enabled but zoned device found: %pg", 679 694 device->bdev);

+1 -1

fs/btrfs/zoned.h

··· 320 320 } 321 321 322 322 /* Do not allow Host Managed zoned device. */ 323 - return bdev_zoned_model(bdev) != BLK_ZONED_HM; 323 + return !bdev_is_zoned(bdev); 324 324 } 325 325 326 326 static inline bool btrfs_check_super_location(struct btrfs_device *device, u64 pos)

+1 -1

fs/f2fs/data.c

··· 995 995 } 996 996 blkaddr -= FDEV(devi).start_blk; 997 997 } 998 - return bdev_zoned_model(FDEV(devi).bdev) == BLK_ZONED_HM && 998 + return bdev_is_zoned(FDEV(devi).bdev) && 999 999 f2fs_blkz_is_seq(sbi, devi, blkaddr) && 1000 1000 (blkaddr % sbi->blocks_per_blkz == sbi->blocks_per_blkz - 1); 1001 1001 }

+7 -10

fs/f2fs/super.c

··· 4279 4279 sbi->aligned_blksize = false; 4280 4280 4281 4281 #ifdef CONFIG_BLK_DEV_ZONED 4282 - if (bdev_zoned_model(FDEV(i).bdev) == BLK_ZONED_HM && 4283 - !f2fs_sb_has_blkzoned(sbi)) { 4284 - f2fs_err(sbi, "Zoned block device feature not enabled"); 4285 - return -EINVAL; 4286 - } 4287 - if (bdev_zoned_model(FDEV(i).bdev) != BLK_ZONED_NONE) { 4282 + if (bdev_is_zoned(FDEV(i).bdev)) { 4283 + if (!f2fs_sb_has_blkzoned(sbi)) { 4284 + f2fs_err(sbi, "Zoned block device feature not enabled"); 4285 + return -EINVAL; 4286 + } 4288 4287 if (init_blkz_info(sbi, i)) { 4289 4288 f2fs_err(sbi, "Failed to initialize F2FS blkzone information"); 4290 4289 return -EINVAL; 4291 4290 } 4292 4291 if (max_devices == 1) 4293 4292 break; 4294 - f2fs_info(sbi, "Mount Device [%2d]: %20s, %8u, %8x - %8x (zone: %s)", 4293 + f2fs_info(sbi, "Mount Device [%2d]: %20s, %8u, %8x - %8x (zone: Host-managed)", 4295 4294 i, FDEV(i).path, 4296 4295 FDEV(i).total_segments, 4297 - FDEV(i).start_blk, FDEV(i).end_blk, 4298 - bdev_zoned_model(FDEV(i).bdev) == BLK_ZONED_HA ? 4299 - "Host-aware" : "Host-managed"); 4296 + FDEV(i).start_blk, FDEV(i).end_blk); 4300 4297 continue; 4301 4298 } 4302 4299 #endif

+9

include/linux/bio.h

··· 324 324 BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */ 325 325 BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */ 326 326 BIP_IP_CHECKSUM = 1 << 4, /* IP checksum */ 327 + BIP_INTEGRITY_USER = 1 << 5, /* Integrity payload is user address */ 328 + BIP_COPY_USER = 1 << 6, /* Kernel bounce buffer in use */ 327 329 }; 328 330 329 331 /* ··· 720 718 for_each_bio(_bio) \ 721 719 bip_for_each_vec(_bvl, _bio->bi_integrity, _iter) 722 720 721 + int bio_integrity_map_user(struct bio *bio, void __user *ubuf, ssize_t len, u32 seed); 723 722 extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int); 724 723 extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int); 725 724 extern bool bio_integrity_prep(struct bio *); ··· 790 787 unsigned int len, unsigned int offset) 791 788 { 792 789 return 0; 790 + } 791 + 792 + static inline int bio_integrity_map_user(struct bio *bio, void __user *ubuf, 793 + ssize_t len, u32 seed) 794 + { 795 + return -EINVAL; 793 796 } 794 797 795 798 #endif /* CONFIG_BLK_DEV_INTEGRITY */

+6

include/linux/blk-mq.h

··· 830 830 */ 831 831 static inline bool blk_mq_need_time_stamp(struct request *rq) 832 832 { 833 + /* 834 + * passthrough io doesn't use iostat accounting, cgroup stats 835 + * and io scheduler functionalities. 836 + */ 837 + if (blk_rq_is_passthrough(rq)) 838 + return false; 833 839 return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_USE_SCHED)); 834 840 } 835 841

+4 -4

include/linux/blk_types.h

··· 378 378 REQ_OP_DISCARD = (__force blk_opf_t)3, 379 379 /* securely erase sectors */ 380 380 REQ_OP_SECURE_ERASE = (__force blk_opf_t)5, 381 + /* write data at the current zone write pointer */ 382 + REQ_OP_ZONE_APPEND = (__force blk_opf_t)7, 381 383 /* write the zero filled sector many times */ 382 384 REQ_OP_WRITE_ZEROES = (__force blk_opf_t)9, 383 385 /* Open a zone */ ··· 388 386 REQ_OP_ZONE_CLOSE = (__force blk_opf_t)11, 389 387 /* Transition a zone to full */ 390 388 REQ_OP_ZONE_FINISH = (__force blk_opf_t)12, 391 - /* write data at the current zone write pointer */ 392 - REQ_OP_ZONE_APPEND = (__force blk_opf_t)13, 393 389 /* reset a zone write pointer */ 394 - REQ_OP_ZONE_RESET = (__force blk_opf_t)15, 390 + REQ_OP_ZONE_RESET = (__force blk_opf_t)13, 395 391 /* reset all the zone present on the device */ 396 - REQ_OP_ZONE_RESET_ALL = (__force blk_opf_t)17, 392 + REQ_OP_ZONE_RESET_ALL = (__force blk_opf_t)15, 397 393 398 394 /* Driver private requests */ 399 395 REQ_OP_DRV_IN = (__force blk_opf_t)34,

+71 -94

include/linux/blkdev.h

··· 266 266 } 267 267 268 268 /* 269 - * Zoned block device models (zoned limit). 270 - * 271 - * Note: This needs to be ordered from the least to the most severe 272 - * restrictions for the inheritance in blk_stack_limits() to work. 273 - */ 274 - enum blk_zoned_model { 275 - BLK_ZONED_NONE = 0, /* Regular block device */ 276 - BLK_ZONED_HA, /* Host-aware zoned block device */ 277 - BLK_ZONED_HM, /* Host-managed zoned block device */ 278 - }; 279 - 280 - /* 281 269 * BLK_BOUNCE_NONE: never bounce (default) 282 270 * BLK_BOUNCE_HIGH: bounce all highmem pages 283 271 */ ··· 306 318 unsigned char misaligned; 307 319 unsigned char discard_misaligned; 308 320 unsigned char raid_partial_stripes_expensive; 309 - enum blk_zoned_model zoned; 321 + bool zoned; 310 322 311 323 /* 312 324 * Drivers that set dma_alignment to less than 511 must be prepared to ··· 319 331 typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx, 320 332 void *data); 321 333 322 - void disk_set_zoned(struct gendisk *disk, enum blk_zoned_model model); 334 + void disk_set_zoned(struct gendisk *disk); 323 335 324 - #ifdef CONFIG_BLK_DEV_ZONED 325 336 #define BLK_ALL_ZONES ((unsigned int)-1) 326 337 int blkdev_report_zones(struct block_device *bdev, sector_t sector, 327 - unsigned int nr_zones, report_zones_cb cb, void *data); 328 - unsigned int bdev_nr_zones(struct block_device *bdev); 329 - extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op, 330 - sector_t sectors, sector_t nr_sectors, 331 - gfp_t gfp_mask); 338 + unsigned int nr_zones, report_zones_cb cb, void *data); 339 + int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op, 340 + sector_t sectors, sector_t nr_sectors, gfp_t gfp_mask); 332 341 int blk_revalidate_disk_zones(struct gendisk *disk, 333 - void (*update_driver_data)(struct gendisk *disk)); 334 - #else /* CONFIG_BLK_DEV_ZONED */ 335 - static inline unsigned int bdev_nr_zones(struct block_device *bdev) 336 - { 337 - return 0; 338 - } 339 - #endif /* CONFIG_BLK_DEV_ZONED */ 342 + void (*update_driver_data)(struct gendisk *disk)); 340 343 341 344 /* 342 345 * Independent access ranges: struct blk_independent_access_range describes ··· 357 378 }; 358 379 359 380 struct request_queue { 360 - struct request *last_merge; 361 - struct elevator_queue *elevator; 362 - 363 - struct percpu_ref q_usage_counter; 364 - 365 - struct blk_queue_stats *stats; 366 - struct rq_qos *rq_qos; 367 - struct mutex rq_qos_mutex; 368 - 369 - const struct blk_mq_ops *mq_ops; 370 - 371 - /* sw queues */ 372 - struct blk_mq_ctx __percpu *queue_ctx; 373 - 374 - unsigned int queue_depth; 375 - 376 - /* hw dispatch queues */ 377 - struct xarray hctx_table; 378 - unsigned int nr_hw_queues; 379 - 380 381 /* 381 382 * The queue owner gets to use this for whatever they like. 382 383 * ll_rw_blk doesn't touch it. 383 384 */ 384 385 void *queuedata; 385 386 387 + struct elevator_queue *elevator; 388 + 389 + const struct blk_mq_ops *mq_ops; 390 + 391 + /* sw queues */ 392 + struct blk_mq_ctx __percpu *queue_ctx; 393 + 386 394 /* 387 395 * various queue flags, see QUEUE_* below 388 396 */ 389 397 unsigned long queue_flags; 390 - /* 391 - * Number of contexts that have called blk_set_pm_only(). If this 392 - * counter is above zero then only RQF_PM requests are processed. 393 - */ 394 - atomic_t pm_only; 395 398 396 - /* 397 - * ida allocated id for this queue. Used to index queues from 398 - * ioctx. 399 - */ 400 - int id; 399 + unsigned int rq_timeout; 400 + 401 + unsigned int queue_depth; 402 + 403 + refcount_t refs; 404 + 405 + /* hw dispatch queues */ 406 + unsigned int nr_hw_queues; 407 + struct xarray hctx_table; 408 + 409 + struct percpu_ref q_usage_counter; 410 + 411 + struct request *last_merge; 401 412 402 413 spinlock_t queue_lock; 403 414 404 - struct gendisk *disk; 415 + int quiesce_depth; 405 416 406 - refcount_t refs; 417 + struct gendisk *disk; 407 418 408 419 /* 409 420 * mq queue kobject 410 421 */ 411 422 struct kobject *mq_kobj; 423 + 424 + struct queue_limits limits; 412 425 413 426 #ifdef CONFIG_BLK_DEV_INTEGRITY 414 427 struct blk_integrity integrity; ··· 412 441 #endif 413 442 414 443 /* 444 + * Number of contexts that have called blk_set_pm_only(). If this 445 + * counter is above zero then only RQF_PM requests are processed. 446 + */ 447 + atomic_t pm_only; 448 + 449 + struct blk_queue_stats *stats; 450 + struct rq_qos *rq_qos; 451 + struct mutex rq_qos_mutex; 452 + 453 + /* 454 + * ida allocated id for this queue. Used to index queues from 455 + * ioctx. 456 + */ 457 + int id; 458 + 459 + unsigned int dma_pad_mask; 460 + 461 + /* 415 462 * queue settings 416 463 */ 417 464 unsigned long nr_requests; /* Max # of requests */ 418 - 419 - unsigned int dma_pad_mask; 420 465 421 466 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 422 467 struct blk_crypto_profile *crypto_profile; 423 468 struct kobject *crypto_kobject; 424 469 #endif 425 470 426 - unsigned int rq_timeout; 427 - 428 471 struct timer_list timeout; 429 472 struct work_struct timeout_work; 430 473 431 474 atomic_t nr_active_requests_shared_tags; 475 + 476 + unsigned int required_elevator_features; 432 477 433 478 struct blk_mq_tags *sched_shared_tags; 434 479 ··· 456 469 struct mutex blkcg_mutex; 457 470 #endif 458 471 459 - struct queue_limits limits; 460 - 461 - unsigned int required_elevator_features; 462 - 463 472 int node; 473 + 474 + spinlock_t requeue_lock; 475 + struct list_head requeue_list; 476 + struct delayed_work requeue_work; 477 + 464 478 #ifdef CONFIG_BLK_DEV_IO_TRACE 465 479 struct blk_trace __rcu *blk_trace; 466 480 #endif ··· 470 482 */ 471 483 struct blk_flush_queue *fq; 472 484 struct list_head flush_list; 473 - 474 - struct list_head requeue_list; 475 - spinlock_t requeue_lock; 476 - struct delayed_work requeue_work; 477 485 478 486 struct mutex sysfs_lock; 479 487 struct mutex sysfs_dir_lock; ··· 494 510 * percpu_ref_kill() and percpu_ref_reinit(). 495 511 */ 496 512 struct mutex mq_freeze_lock; 497 - 498 - int quiesce_depth; 499 513 500 514 struct blk_mq_tag_set *tag_set; 501 515 struct list_head tag_set_list; ··· 607 625 } 608 626 #endif 609 627 610 - static inline enum blk_zoned_model 611 - blk_queue_zoned_model(struct request_queue *q) 612 - { 613 - if (IS_ENABLED(CONFIG_BLK_DEV_ZONED)) 614 - return q->limits.zoned; 615 - return BLK_ZONED_NONE; 616 - } 617 - 618 628 static inline bool blk_queue_is_zoned(struct request_queue *q) 619 629 { 620 - switch (blk_queue_zoned_model(q)) { 621 - case BLK_ZONED_HA: 622 - case BLK_ZONED_HM: 623 - return true; 624 - default: 625 - return false; 626 - } 630 + return IS_ENABLED(CONFIG_BLK_DEV_ZONED) && q->limits.zoned; 627 631 } 628 632 629 633 #ifdef CONFIG_BLK_DEV_ZONED 634 + unsigned int bdev_nr_zones(struct block_device *bdev); 635 + 630 636 static inline unsigned int disk_nr_zones(struct gendisk *disk) 631 637 { 632 638 return blk_queue_is_zoned(disk->queue) ? disk->nr_zones : 0; ··· 659 689 } 660 690 661 691 #else /* CONFIG_BLK_DEV_ZONED */ 692 + static inline unsigned int bdev_nr_zones(struct block_device *bdev) 693 + { 694 + return 0; 695 + } 696 + 662 697 static inline unsigned int disk_nr_zones(struct gendisk *disk) 663 698 { 664 699 return 0; ··· 1057 1082 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, 1058 1083 }; 1059 1084 1060 - #define BLK_DEF_MAX_SECTORS 2560u 1085 + /* 1086 + * Default upper limit for the software max_sectors limit used for 1087 + * regular file system I/O. This can be increased through sysfs. 1088 + * 1089 + * Not to be confused with the max_hw_sector limit that is entirely 1090 + * controlled by the driver, usually based on hardware limits. 1091 + */ 1092 + #define BLK_DEF_MAX_SECTORS_CAP 2560u 1061 1093 1062 1094 static inline unsigned long queue_segment_boundary(const struct request_queue *q) 1063 1095 { ··· 1241 1259 static inline bool bdev_nowait(struct block_device *bdev) 1242 1260 { 1243 1261 return test_bit(QUEUE_FLAG_NOWAIT, &bdev_get_queue(bdev)->queue_flags); 1244 - } 1245 - 1246 - static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev) 1247 - { 1248 - return blk_queue_zoned_model(bdev_get_queue(bdev)); 1249 1262 } 1250 1263 1251 1264 static inline bool bdev_is_zoned(struct block_device *bdev)

+2 -7

include/linux/io_uring.h

··· 28 28 29 29 /* only top 8 bits of sqe->uring_cmd_flags for kernel internal use */ 30 30 #define IORING_URING_CMD_CANCELABLE (1U << 30) 31 - #define IORING_URING_CMD_POLLED (1U << 31) 32 31 33 32 struct io_uring_cmd { 34 33 struct file *file; 35 34 const struct io_uring_sqe *sqe; 36 - union { 37 - /* callback to defer completions to task context */ 38 - void (*task_work_cb)(struct io_uring_cmd *cmd, unsigned); 39 - /* used for polled completion */ 40 - void *cookie; 41 - }; 35 + /* callback to defer completions to task context */ 36 + void (*task_work_cb)(struct io_uring_cmd *cmd, unsigned); 42 37 u32 cmd_op; 43 38 u32 flags; 44 39 u8 pdu[32]; /* available inline for free use */

+2 -6

include/uapi/linux/raid/md_p.h

··· 2 2 /* 3 3 md_p.h : physical layout of Linux RAID devices 4 4 Copyright (C) 1996-98 Ingo Molnar, Gadi Oxman 5 - 5 + 6 6 This program is free software; you can redistribute it and/or modify 7 7 it under the terms of the GNU General Public License as published by 8 8 the Free Software Foundation; either version 2, or (at your option) 9 9 any later version. 10 - 11 - You should have received a copy of the GNU General Public License 12 - (for example /usr/src/linux/COPYING); if not, write to the Free 13 - Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 14 10 */ 15 11 16 12 #ifndef _MD_P_H ··· 233 237 char set_name[32]; /* set and interpreted by user-space */ 234 238 235 239 __le64 ctime; /* lo 40 bits are seconds, top 24 are microseconds or 0*/ 236 - __le32 level; /* -4 (multipath), -1 (linear), 0,1,4,5 */ 240 + __le32 level; /* 0,1,4,5 */ 237 241 __le32 layout; /* only for raid5 and raid10 currently */ 238 242 __le64 size; /* used size of component devices, in 512byte sectors */ 239 243

+1 -10

include/uapi/linux/raid/md_u.h

··· 2 2 /* 3 3 md_u.h : user <=> kernel API between Linux raidtools and RAID drivers 4 4 Copyright (C) 1998 Ingo Molnar 5 - 5 + 6 6 This program is free software; you can redistribute it and/or modify 7 7 it under the terms of the GNU General Public License as published by 8 8 the Free Software Foundation; either version 2, or (at your option) 9 9 any later version. 10 - 11 - You should have received a copy of the GNU General Public License 12 - (for example /usr/src/linux/COPYING); if not, write to the Free 13 - Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 14 10 */ 15 11 16 12 #ifndef _UAPI_MD_U_H ··· 102 106 int chunk_size; /* 1 chunk size in bytes */ 103 107 104 108 } mdu_array_info_t; 105 - 106 - /* non-obvious values for 'level' */ 107 - #define LEVEL_MULTIPATH (-4) 108 - #define LEVEL_LINEAR (-1) 109 - #define LEVEL_FAULTY (-5) 110 109 111 110 /* we need a value for 'no level specified' and 0 112 111 * means 'raid0', so we need something else. This is

-1

io_uring/uring_cmd.c

··· 182 182 return -EOPNOTSUPP; 183 183 issue_flags |= IO_URING_F_IOPOLL; 184 184 req->iopoll_completed = 0; 185 - WRITE_ONCE(ioucmd->cookie, NULL); 186 185 } 187 186 188 187 ret = file->f_op->uring_cmd(ioucmd, issue_flags);

Configure Feed

Configure Feed