Merge tag 'for-6.7/dm-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

+62 -25

drivers/md/dm-bufio.c

··· 254 254 255 255 typedef enum evict_result (*le_predicate)(struct lru_entry *le, void *context); 256 256 257 - static struct lru_entry *lru_evict(struct lru *lru, le_predicate pred, void *context) 257 + static struct lru_entry *lru_evict(struct lru *lru, le_predicate pred, void *context, bool no_sleep) 258 258 { 259 259 unsigned long tested = 0; 260 260 struct list_head *h = lru->cursor; ··· 295 295 296 296 h = h->next; 297 297 298 - cond_resched(); 298 + if (!no_sleep) 299 + cond_resched(); 299 300 } 300 301 301 302 return NULL; ··· 383 382 */ 384 383 385 384 struct buffer_tree { 386 - struct rw_semaphore lock; 385 + union { 386 + struct rw_semaphore lock; 387 + rwlock_t spinlock; 388 + } u; 387 389 struct rb_root root; 388 390 } ____cacheline_aligned_in_smp; 389 391 ··· 397 393 * on the locks. 398 394 */ 399 395 unsigned int num_locks; 396 + bool no_sleep; 400 397 struct buffer_tree trees[]; 401 398 }; 399 + 400 + static DEFINE_STATIC_KEY_FALSE(no_sleep_enabled); 402 401 403 402 static inline unsigned int cache_index(sector_t block, unsigned int num_locks) 404 403 { ··· 410 403 411 404 static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t block) 412 405 { 413 - down_read(&bc->trees[cache_index(block, bc->num_locks)].lock); 406 + if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 407 + read_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 408 + else 409 + down_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 414 410 } 415 411 416 412 static inline void cache_read_unlock(struct dm_buffer_cache *bc, sector_t block) 417 413 { 418 - up_read(&bc->trees[cache_index(block, bc->num_locks)].lock); 414 + if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 415 + read_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 416 + else 417 + up_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 419 418 } 420 419 421 420 static inline void cache_write_lock(struct dm_buffer_cache *bc, sector_t block) 422 421 { 423 - down_write(&bc->trees[cache_index(block, bc->num_locks)].lock); 422 + if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 423 + write_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 424 + else 425 + down_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 424 426 } 425 427 426 428 static inline void cache_write_unlock(struct dm_buffer_cache *bc, sector_t block) 427 429 { 428 - up_write(&bc->trees[cache_index(block, bc->num_locks)].lock); 430 + if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 431 + write_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 432 + else 433 + up_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 429 434 } 430 435 431 436 /* ··· 461 442 462 443 static void __lh_lock(struct lock_history *lh, unsigned int index) 463 444 { 464 - if (lh->write) 465 - down_write(&lh->cache->trees[index].lock); 466 - else 467 - down_read(&lh->cache->trees[index].lock); 445 + if (lh->write) { 446 + if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) 447 + write_lock_bh(&lh->cache->trees[index].u.spinlock); 448 + else 449 + down_write(&lh->cache->trees[index].u.lock); 450 + } else { 451 + if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) 452 + read_lock_bh(&lh->cache->trees[index].u.spinlock); 453 + else 454 + down_read(&lh->cache->trees[index].u.lock); 455 + } 468 456 } 469 457 470 458 static void __lh_unlock(struct lock_history *lh, unsigned int index) 471 459 { 472 - if (lh->write) 473 - up_write(&lh->cache->trees[index].lock); 474 - else 475 - up_read(&lh->cache->trees[index].lock); 460 + if (lh->write) { 461 + if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) 462 + write_unlock_bh(&lh->cache->trees[index].u.spinlock); 463 + else 464 + up_write(&lh->cache->trees[index].u.lock); 465 + } else { 466 + if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) 467 + read_unlock_bh(&lh->cache->trees[index].u.spinlock); 468 + else 469 + up_read(&lh->cache->trees[index].u.lock); 470 + } 476 471 } 477 472 478 473 /* ··· 535 502 return le_to_buffer(le); 536 503 } 537 504 538 - static void cache_init(struct dm_buffer_cache *bc, unsigned int num_locks) 505 + static void cache_init(struct dm_buffer_cache *bc, unsigned int num_locks, bool no_sleep) 539 506 { 540 507 unsigned int i; 541 508 542 509 bc->num_locks = num_locks; 510 + bc->no_sleep = no_sleep; 543 511 544 512 for (i = 0; i < bc->num_locks; i++) { 545 - init_rwsem(&bc->trees[i].lock); 513 + if (no_sleep) 514 + rwlock_init(&bc->trees[i].u.spinlock); 515 + else 516 + init_rwsem(&bc->trees[i].u.lock); 546 517 bc->trees[i].root = RB_ROOT; 547 518 } 548 519 ··· 685 648 struct lru_entry *le; 686 649 struct dm_buffer *b; 687 650 688 - le = lru_evict(&bc->lru[list_mode], __evict_pred, &w); 651 + le = lru_evict(&bc->lru[list_mode], __evict_pred, &w, bc->no_sleep); 689 652 if (!le) 690 653 return NULL; 691 654 ··· 739 702 struct evict_wrapper w = {.lh = lh, .pred = pred, .context = context}; 740 703 741 704 while (true) { 742 - le = lru_evict(&bc->lru[old_mode], __evict_pred, &w); 705 + le = lru_evict(&bc->lru[old_mode], __evict_pred, &w, bc->no_sleep); 743 706 if (!le) 744 707 break; 745 708 ··· 952 915 { 953 916 unsigned int i; 954 917 918 + BUG_ON(bc->no_sleep); 955 919 for (i = 0; i < bc->num_locks; i++) { 956 - down_write(&bc->trees[i].lock); 920 + down_write(&bc->trees[i].u.lock); 957 921 __remove_range(bc, &bc->trees[i].root, begin, end, pred, release); 958 - up_write(&bc->trees[i].lock); 922 + up_write(&bc->trees[i].u.lock); 959 923 } 960 924 } 961 925 ··· 1016 978 1017 979 struct dm_buffer_cache cache; /* must be last member */ 1018 980 }; 1019 - 1020 - static DEFINE_STATIC_KEY_FALSE(no_sleep_enabled); 1021 981 1022 982 /*----------------------------------------------------------------*/ 1023 983 ··· 1907 1871 if (need_submit) 1908 1872 submit_io(b, REQ_OP_READ, read_endio); 1909 1873 1910 - wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); 1874 + if (nf != NF_GET) /* we already tested this condition above */ 1875 + wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); 1911 1876 1912 1877 if (b->read_error) { 1913 1878 int error = blk_status_to_errno(b->read_error); ··· 2458 2421 r = -ENOMEM; 2459 2422 goto bad_client; 2460 2423 } 2461 - cache_init(&c->cache, num_locks); 2424 + cache_init(&c->cache, num_locks, (flags & DM_BUFIO_CLIENT_NO_SLEEP) != 0); 2462 2425 2463 2426 c->bdev = bdev; 2464 2427 c->block_size = block_size;

+1 -1

drivers/md/dm-crypt.c

··· 1673 1673 unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1674 1674 gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM; 1675 1675 unsigned int remaining_size; 1676 - unsigned int order = MAX_ORDER - 1; 1676 + unsigned int order = MAX_ORDER; 1677 1677 1678 1678 retry: 1679 1679 if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))

+52 -60

drivers/md/dm-delay.c

··· 33 33 struct work_struct flush_expired_bios; 34 34 struct list_head delayed_bios; 35 35 struct task_struct *worker; 36 - atomic_t may_delay; 36 + bool may_delay; 37 37 38 38 struct delay_class read; 39 39 struct delay_class write; ··· 73 73 return !!dc->worker; 74 74 } 75 75 76 - static void flush_delayed_bios_fast(struct delay_c *dc, bool flush_all) 77 - { 78 - struct dm_delay_info *delayed, *next; 79 - 80 - mutex_lock(&delayed_bios_lock); 81 - list_for_each_entry_safe(delayed, next, &dc->delayed_bios, list) { 82 - if (flush_all || time_after_eq(jiffies, delayed->expires)) { 83 - struct bio *bio = dm_bio_from_per_bio_data(delayed, 84 - sizeof(struct dm_delay_info)); 85 - list_del(&delayed->list); 86 - dm_submit_bio_remap(bio, NULL); 87 - delayed->class->ops--; 88 - } 89 - } 90 - mutex_unlock(&delayed_bios_lock); 91 - } 92 - 93 - static int flush_worker_fn(void *data) 94 - { 95 - struct delay_c *dc = data; 96 - 97 - while (1) { 98 - flush_delayed_bios_fast(dc, false); 99 - if (unlikely(list_empty(&dc->delayed_bios))) { 100 - set_current_state(TASK_INTERRUPTIBLE); 101 - schedule(); 102 - } else 103 - cond_resched(); 104 - } 105 - 106 - return 0; 107 - } 108 - 109 76 static void flush_bios(struct bio *bio) 110 77 { 111 78 struct bio *n; ··· 85 118 } 86 119 } 87 120 88 - static struct bio *flush_delayed_bios(struct delay_c *dc, bool flush_all) 121 + static void flush_delayed_bios(struct delay_c *dc, bool flush_all) 89 122 { 90 123 struct dm_delay_info *delayed, *next; 124 + struct bio_list flush_bio_list; 91 125 unsigned long next_expires = 0; 92 - unsigned long start_timer = 0; 93 - struct bio_list flush_bios = { }; 126 + bool start_timer = false; 127 + bio_list_init(&flush_bio_list); 94 128 95 129 mutex_lock(&delayed_bios_lock); 96 130 list_for_each_entry_safe(delayed, next, &dc->delayed_bios, list) { 131 + cond_resched(); 97 132 if (flush_all || time_after_eq(jiffies, delayed->expires)) { 98 133 struct bio *bio = dm_bio_from_per_bio_data(delayed, 99 134 sizeof(struct dm_delay_info)); 100 135 list_del(&delayed->list); 101 - bio_list_add(&flush_bios, bio); 136 + bio_list_add(&flush_bio_list, bio); 102 137 delayed->class->ops--; 103 138 continue; 104 139 } 105 140 106 - if (!start_timer) { 107 - start_timer = 1; 108 - next_expires = delayed->expires; 109 - } else 110 - next_expires = min(next_expires, delayed->expires); 141 + if (!delay_is_fast(dc)) { 142 + if (!start_timer) { 143 + start_timer = true; 144 + next_expires = delayed->expires; 145 + } else { 146 + next_expires = min(next_expires, delayed->expires); 147 + } 148 + } 111 149 } 112 150 mutex_unlock(&delayed_bios_lock); 113 151 114 152 if (start_timer) 115 153 queue_timeout(dc, next_expires); 116 154 117 - return bio_list_get(&flush_bios); 155 + flush_bios(bio_list_get(&flush_bio_list)); 156 + } 157 + 158 + static int flush_worker_fn(void *data) 159 + { 160 + struct delay_c *dc = data; 161 + 162 + while (!kthread_should_stop()) { 163 + flush_delayed_bios(dc, false); 164 + mutex_lock(&delayed_bios_lock); 165 + if (unlikely(list_empty(&dc->delayed_bios))) { 166 + set_current_state(TASK_INTERRUPTIBLE); 167 + mutex_unlock(&delayed_bios_lock); 168 + schedule(); 169 + } else { 170 + mutex_unlock(&delayed_bios_lock); 171 + cond_resched(); 172 + } 173 + } 174 + 175 + return 0; 118 176 } 119 177 120 178 static void flush_expired_bios(struct work_struct *work) ··· 147 155 struct delay_c *dc; 148 156 149 157 dc = container_of(work, struct delay_c, flush_expired_bios); 150 - if (delay_is_fast(dc)) 151 - flush_delayed_bios_fast(dc, false); 152 - else 153 - flush_bios(flush_delayed_bios(dc, false)); 158 + flush_delayed_bios(dc, false); 154 159 } 155 160 156 161 static void delay_dtr(struct dm_target *ti) ··· 166 177 if (dc->worker) 167 178 kthread_stop(dc->worker); 168 179 169 - if (!delay_is_fast(dc)) 170 - mutex_destroy(&dc->timer_lock); 180 + mutex_destroy(&dc->timer_lock); 171 181 172 182 kfree(dc); 173 183 } ··· 224 236 225 237 ti->private = dc; 226 238 INIT_LIST_HEAD(&dc->delayed_bios); 227 - atomic_set(&dc->may_delay, 1); 239 + mutex_init(&dc->timer_lock); 240 + dc->may_delay = true; 228 241 dc->argc = argc; 229 242 230 243 ret = delay_class_ctr(ti, &dc->read, argv); ··· 271 282 "dm-delay-flush-worker"); 272 283 if (IS_ERR(dc->worker)) { 273 284 ret = PTR_ERR(dc->worker); 285 + dc->worker = NULL; 274 286 goto bad; 275 287 } 276 288 } else { 277 289 timer_setup(&dc->delay_timer, handle_delayed_timer, 0); 278 290 INIT_WORK(&dc->flush_expired_bios, flush_expired_bios); 279 - mutex_init(&dc->timer_lock); 280 291 dc->kdelayd_wq = alloc_workqueue("kdelayd", WQ_MEM_RECLAIM, 0); 281 292 if (!dc->kdelayd_wq) { 282 293 ret = -EINVAL; ··· 301 312 struct dm_delay_info *delayed; 302 313 unsigned long expires = 0; 303 314 304 - if (!c->delay || !atomic_read(&dc->may_delay)) 315 + if (!c->delay) 305 316 return DM_MAPIO_REMAPPED; 306 317 307 318 delayed = dm_per_bio_data(bio, sizeof(struct dm_delay_info)); ··· 310 321 delayed->expires = expires = jiffies + msecs_to_jiffies(c->delay); 311 322 312 323 mutex_lock(&delayed_bios_lock); 324 + if (unlikely(!dc->may_delay)) { 325 + mutex_unlock(&delayed_bios_lock); 326 + return DM_MAPIO_REMAPPED; 327 + } 313 328 c->ops++; 314 329 list_add_tail(&delayed->list, &dc->delayed_bios); 315 330 mutex_unlock(&delayed_bios_lock); ··· 330 337 { 331 338 struct delay_c *dc = ti->private; 332 339 333 - atomic_set(&dc->may_delay, 0); 340 + mutex_lock(&delayed_bios_lock); 341 + dc->may_delay = false; 342 + mutex_unlock(&delayed_bios_lock); 334 343 335 - if (delay_is_fast(dc)) 336 - flush_delayed_bios_fast(dc, true); 337 - else { 344 + if (!delay_is_fast(dc)) 338 345 del_timer_sync(&dc->delay_timer); 339 - flush_bios(flush_delayed_bios(dc, true)); 340 - } 346 + flush_delayed_bios(dc, true); 341 347 } 342 348 343 349 static void delay_resume(struct dm_target *ti) 344 350 { 345 351 struct delay_c *dc = ti->private; 346 352 347 - atomic_set(&dc->may_delay, 1); 353 + dc->may_delay = true; 348 354 } 349 355 350 356 static int delay_map(struct dm_target *ti, struct bio *bio)

+2 -2

drivers/md/dm-verity-fec.c

··· 185 185 { 186 186 if (unlikely(verity_hash(v, verity_io_hash_req(v, io), 187 187 data, 1 << v->data_dev_block_bits, 188 - verity_io_real_digest(v, io)))) 188 + verity_io_real_digest(v, io), true))) 189 189 return 0; 190 190 191 191 return memcmp(verity_io_real_digest(v, io), want_digest, ··· 386 386 /* Always re-validate the corrected block against the expected hash */ 387 387 r = verity_hash(v, verity_io_hash_req(v, io), fio->output, 388 388 1 << v->data_dev_block_bits, 389 - verity_io_real_digest(v, io)); 389 + verity_io_real_digest(v, io), true); 390 390 if (unlikely(r < 0)) 391 391 return r; 392 392

+12 -11

drivers/md/dm-verity-target.c

··· 135 135 * Wrapper for crypto_ahash_init, which handles verity salting. 136 136 */ 137 137 static int verity_hash_init(struct dm_verity *v, struct ahash_request *req, 138 - struct crypto_wait *wait) 138 + struct crypto_wait *wait, bool may_sleep) 139 139 { 140 140 int r; 141 141 142 142 ahash_request_set_tfm(req, v->tfm); 143 - ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | 144 - CRYPTO_TFM_REQ_MAY_BACKLOG, 145 - crypto_req_done, (void *)wait); 143 + ahash_request_set_callback(req, 144 + may_sleep ? CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG : 0, 145 + crypto_req_done, (void *)wait); 146 146 crypto_init_wait(wait); 147 147 148 148 r = crypto_wait_req(crypto_ahash_init(req), wait); 149 149 150 150 if (unlikely(r < 0)) { 151 - DMERR("crypto_ahash_init failed: %d", r); 151 + if (r != -ENOMEM) 152 + DMERR("crypto_ahash_init failed: %d", r); 152 153 return r; 153 154 } 154 155 ··· 180 179 } 181 180 182 181 int verity_hash(struct dm_verity *v, struct ahash_request *req, 183 - const u8 *data, size_t len, u8 *digest) 182 + const u8 *data, size_t len, u8 *digest, bool may_sleep) 184 183 { 185 184 int r; 186 185 struct crypto_wait wait; 187 186 188 - r = verity_hash_init(v, req, &wait); 187 + r = verity_hash_init(v, req, &wait, may_sleep); 189 188 if (unlikely(r < 0)) 190 189 goto out; 191 190 ··· 323 322 324 323 r = verity_hash(v, verity_io_hash_req(v, io), 325 324 data, 1 << v->hash_dev_block_bits, 326 - verity_io_real_digest(v, io)); 325 + verity_io_real_digest(v, io), !io->in_tasklet); 327 326 if (unlikely(r < 0)) 328 327 goto release_ret_r; 329 328 ··· 557 556 continue; 558 557 } 559 558 560 - r = verity_hash_init(v, req, &wait); 559 + r = verity_hash_init(v, req, &wait, !io->in_tasklet); 561 560 if (unlikely(r < 0)) 562 561 return r; 563 562 ··· 653 652 654 653 io->in_tasklet = true; 655 654 err = verity_verify_io(io); 656 - if (err == -EAGAIN) { 655 + if (err == -EAGAIN || err == -ENOMEM) { 657 656 /* fallback to retrying with work-queue */ 658 657 INIT_WORK(&io->work, verity_work); 659 658 queue_work(io->v->verify_wq, &io->work); ··· 1034 1033 goto out; 1035 1034 1036 1035 r = verity_hash(v, req, zero_data, 1 << v->data_dev_block_bits, 1037 - v->zero_digest); 1036 + v->zero_digest, true); 1038 1037 1039 1038 out: 1040 1039 kfree(req);

+1 -1

drivers/md/dm-verity.h

··· 128 128 u8 *data, size_t len)); 129 129 130 130 extern int verity_hash(struct dm_verity *v, struct ahash_request *req, 131 - const u8 *data, size_t len, u8 *digest); 131 + const u8 *data, size_t len, u8 *digest, bool may_sleep); 132 132 133 133 extern int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, 134 134 sector_t block, u8 *digest, bool *is_zero);

Configure Feed

Configure Feed