Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
dm io: Support IO priority
Some IO will dispatch from kworker with different io_context settings
than the submitting task, we may need to specify a priority to avoid
losing priority.
Add IO priority parameter to dm_io() and update all callers.
Co-developed-by: Yibin Ding <yibin.ding@unisoc.com>
Signed-off-by: Yibin Ding <yibin.ding@unisoc.com>
Signed-off-by: Hongyu Jin <hongyu.jin@unisoc.com>
Reviewed-by: Eric Biggers <ebiggers@google.com>
Reviewed-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Mike Snitzer <snitzer@kernel.org>
authored by
Hongyu Jin
and committed by
Mike Snitzer
6e5f0f63
b9d3ac48
+38
-34
+3
-3
drivers/md/dm-bufio.c
+3
-3
drivers/md/dm-bufio.c
···
1315
1315
io_req.mem.ptr.vma = (char *)b->data + offset;
1316
1316
}
1317
1317
1318
-
r = dm_io(&io_req, 1, ®ion, NULL);
1318
+
r = dm_io(&io_req, 1, ®ion, NULL, IOPRIO_DEFAULT);
1319
1319
if (unlikely(r))
1320
1320
b->end_io(b, errno_to_blk_status(r));
1321
1321
}
···
2167
2167
if (WARN_ON_ONCE(dm_bufio_in_request()))
2168
2168
return -EINVAL;
2169
2169
2170
-
return dm_io(&io_req, 1, &io_reg, NULL);
2170
+
return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT);
2171
2171
}
2172
2172
EXPORT_SYMBOL_GPL(dm_bufio_issue_flush);
2173
2173
···
2191
2191
if (WARN_ON_ONCE(dm_bufio_in_request()))
2192
2192
return -EINVAL; /* discards are optional */
2193
2193
2194
-
return dm_io(&io_req, 1, &io_reg, NULL);
2194
+
return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT);
2195
2195
}
2196
2196
EXPORT_SYMBOL_GPL(dm_bufio_issue_discard);
2197
2197
+6
-6
drivers/md/dm-integrity.c
+6
-6
drivers/md/dm-integrity.c
···
555
555
}
556
556
}
557
557
558
-
r = dm_io(&io_req, 1, &io_loc, NULL);
558
+
r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
559
559
if (unlikely(r))
560
560
return r;
561
561
···
1073
1073
io_loc.sector = ic->start + SB_SECTORS + sector;
1074
1074
io_loc.count = n_sectors;
1075
1075
1076
-
r = dm_io(&io_req, 1, &io_loc, NULL);
1076
+
r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
1077
1077
if (unlikely(r)) {
1078
1078
dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ?
1079
1079
"reading journal" : "writing journal", r);
···
1190
1190
io_loc.sector = target;
1191
1191
io_loc.count = n_sectors;
1192
1192
1193
-
r = dm_io(&io_req, 1, &io_loc, NULL);
1193
+
r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
1194
1194
if (unlikely(r)) {
1195
1195
WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
1196
1196
fn(-1UL, data);
···
1519
1519
fr.io_reg.count = 0,
1520
1520
fr.ic = ic;
1521
1521
init_completion(&fr.comp);
1522
-
r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL);
1522
+
r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT);
1523
1523
BUG_ON(r);
1524
1524
}
1525
1525
···
1728
1728
io_loc.sector = sector;
1729
1729
io_loc.count = ic->sectors_per_block;
1730
1730
1731
-
r = dm_io(&io_req, 1, &io_loc, NULL);
1731
+
r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
1732
1732
if (unlikely(r)) {
1733
1733
dio->bi_status = errno_to_blk_status(r);
1734
1734
goto free_ret;
···
2808
2808
io_loc.sector = get_data_sector(ic, area, offset);
2809
2809
io_loc.count = n_sectors;
2810
2810
2811
-
r = dm_io(&io_req, 1, &io_loc, NULL);
2811
+
r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
2812
2812
if (unlikely(r)) {
2813
2813
dm_integrity_io_error(ic, "reading data", r);
2814
2814
goto err;
+13
-10
drivers/md/dm-io.c
+13
-10
drivers/md/dm-io.c
···
305
305
*/
306
306
static void do_region(const blk_opf_t opf, unsigned int region,
307
307
struct dm_io_region *where, struct dpages *dp,
308
-
struct io *io)
308
+
struct io *io, unsigned short ioprio)
309
309
{
310
310
struct bio *bio;
311
311
struct page *page;
···
354
354
&io->client->bios);
355
355
bio->bi_iter.bi_sector = where->sector + (where->count - remaining);
356
356
bio->bi_end_io = endio;
357
+
bio->bi_ioprio = ioprio;
357
358
store_io_and_region_in_bio(bio, io, region);
358
359
359
360
if (op == REQ_OP_DISCARD || op == REQ_OP_WRITE_ZEROES) {
···
384
383
385
384
static void dispatch_io(blk_opf_t opf, unsigned int num_regions,
386
385
struct dm_io_region *where, struct dpages *dp,
387
-
struct io *io, int sync)
386
+
struct io *io, int sync, unsigned short ioprio)
388
387
{
389
388
int i;
390
389
struct dpages old_pages = *dp;
···
401
400
for (i = 0; i < num_regions; i++) {
402
401
*dp = old_pages;
403
402
if (where[i].count || (opf & REQ_PREFLUSH))
404
-
do_region(opf, i, where + i, dp, io);
403
+
do_region(opf, i, where + i, dp, io, ioprio);
405
404
}
406
405
407
406
/*
···
426
425
427
426
static int sync_io(struct dm_io_client *client, unsigned int num_regions,
428
427
struct dm_io_region *where, blk_opf_t opf, struct dpages *dp,
429
-
unsigned long *error_bits)
428
+
unsigned long *error_bits, unsigned short ioprio)
430
429
{
431
430
struct io *io;
432
431
struct sync_io sio;
···
448
447
io->vma_invalidate_address = dp->vma_invalidate_address;
449
448
io->vma_invalidate_size = dp->vma_invalidate_size;
450
449
451
-
dispatch_io(opf, num_regions, where, dp, io, 1);
450
+
dispatch_io(opf, num_regions, where, dp, io, 1, ioprio);
452
451
453
452
wait_for_completion_io(&sio.wait);
454
453
···
460
459
461
460
static int async_io(struct dm_io_client *client, unsigned int num_regions,
462
461
struct dm_io_region *where, blk_opf_t opf,
463
-
struct dpages *dp, io_notify_fn fn, void *context)
462
+
struct dpages *dp, io_notify_fn fn, void *context,
463
+
unsigned short ioprio)
464
464
{
465
465
struct io *io;
466
466
···
481
479
io->vma_invalidate_address = dp->vma_invalidate_address;
482
480
io->vma_invalidate_size = dp->vma_invalidate_size;
483
481
484
-
dispatch_io(opf, num_regions, where, dp, io, 0);
482
+
dispatch_io(opf, num_regions, where, dp, io, 0, ioprio);
485
483
return 0;
486
484
}
487
485
···
523
521
}
524
522
525
523
int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
526
-
struct dm_io_region *where, unsigned long *sync_error_bits)
524
+
struct dm_io_region *where, unsigned long *sync_error_bits,
525
+
unsigned short ioprio)
527
526
{
528
527
int r;
529
528
struct dpages dp;
···
535
532
536
533
if (!io_req->notify.fn)
537
534
return sync_io(io_req->client, num_regions, where,
538
-
io_req->bi_opf, &dp, sync_error_bits);
535
+
io_req->bi_opf, &dp, sync_error_bits, ioprio);
539
536
540
537
return async_io(io_req->client, num_regions, where,
541
538
io_req->bi_opf, &dp, io_req->notify.fn,
542
-
io_req->notify.context);
539
+
io_req->notify.context, ioprio);
543
540
}
544
541
EXPORT_SYMBOL(dm_io);
545
542
+2
-2
drivers/md/dm-kcopyd.c
+2
-2
drivers/md/dm-kcopyd.c
···
578
578
io_job_start(job->kc->throttle);
579
579
580
580
if (job->op == REQ_OP_READ)
581
-
r = dm_io(&io_req, 1, &job->source, NULL);
581
+
r = dm_io(&io_req, 1, &job->source, NULL, IOPRIO_DEFAULT);
582
582
else
583
-
r = dm_io(&io_req, job->num_dests, job->dests, NULL);
583
+
r = dm_io(&io_req, job->num_dests, job->dests, NULL, IOPRIO_DEFAULT);
584
584
585
585
return r;
586
586
}
+2
-2
drivers/md/dm-log.c
+2
-2
drivers/md/dm-log.c
···
300
300
{
301
301
lc->io_req.bi_opf = op;
302
302
303
-
return dm_io(&lc->io_req, 1, &lc->header_location, NULL);
303
+
return dm_io(&lc->io_req, 1, &lc->header_location, NULL, IOPRIO_DEFAULT);
304
304
}
305
305
306
306
static int flush_header(struct log_c *lc)
···
313
313
314
314
lc->io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
315
315
316
-
return dm_io(&lc->io_req, 1, &null_location, NULL);
316
+
return dm_io(&lc->io_req, 1, &null_location, NULL, IOPRIO_DEFAULT);
317
317
}
318
318
319
319
static int read_header(struct log_c *log)
+3
-3
drivers/md/dm-raid1.c
+3
-3
drivers/md/dm-raid1.c
···
278
278
}
279
279
280
280
error_bits = -1;
281
-
dm_io(&io_req, ms->nr_mirrors, io, &error_bits);
281
+
dm_io(&io_req, ms->nr_mirrors, io, &error_bits, IOPRIO_DEFAULT);
282
282
if (unlikely(error_bits != 0)) {
283
283
for (i = 0; i < ms->nr_mirrors; i++)
284
284
if (test_bit(i, &error_bits))
···
554
554
555
555
map_region(&io, m, bio);
556
556
bio_set_m(bio, m);
557
-
BUG_ON(dm_io(&io_req, 1, &io, NULL));
557
+
BUG_ON(dm_io(&io_req, 1, &io, NULL, IOPRIO_DEFAULT));
558
558
}
559
559
560
560
static inline int region_in_sync(struct mirror_set *ms, region_t region,
···
681
681
*/
682
682
bio_set_m(bio, get_default_mirror(ms));
683
683
684
-
BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL));
684
+
BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, IOPRIO_DEFAULT));
685
685
}
686
686
687
687
static void do_writes(struct mirror_set *ms, struct bio_list *writes)
+2
-2
drivers/md/dm-snap-persistent.c
+2
-2
drivers/md/dm-snap-persistent.c
···
223
223
{
224
224
struct mdata_req *req = container_of(work, struct mdata_req, work);
225
225
226
-
req->result = dm_io(req->io_req, 1, req->where, NULL);
226
+
req->result = dm_io(req->io_req, 1, req->where, NULL, IOPRIO_DEFAULT);
227
227
}
228
228
229
229
/*
···
247
247
struct mdata_req req;
248
248
249
249
if (!metadata)
250
-
return dm_io(&io_req, 1, &where, NULL);
250
+
return dm_io(&io_req, 1, &where, NULL, IOPRIO_DEFAULT);
251
251
252
252
req.where = &where;
253
253
req.io_req = &io_req;
+1
-1
drivers/md/dm-verity-target.c
+1
-1
drivers/md/dm-verity-target.c
···
511
511
io_loc.bdev = v->data_dev->bdev;
512
512
io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT);
513
513
io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT);
514
-
r = dm_io(&io_req, 1, &io_loc, NULL);
514
+
r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
515
515
if (unlikely(r))
516
516
goto free_ret;
517
517
+4
-4
drivers/md/dm-writecache.c
+4
-4
drivers/md/dm-writecache.c
···
531
531
req.notify.context = &endio;
532
532
533
533
/* writing via async dm-io (implied by notify.fn above) won't return an error */
534
-
(void) dm_io(&req, 1, ®ion, NULL);
534
+
(void) dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT);
535
535
i = j;
536
536
}
537
537
···
568
568
req.notify.fn = NULL;
569
569
req.notify.context = NULL;
570
570
571
-
r = dm_io(&req, 1, ®ion, NULL);
571
+
r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT);
572
572
if (unlikely(r))
573
573
writecache_error(wc, r, "error writing superblock");
574
574
}
···
596
596
req.client = wc->dm_io;
597
597
req.notify.fn = NULL;
598
598
599
-
r = dm_io(&req, 1, ®ion, NULL);
599
+
r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT);
600
600
if (unlikely(r))
601
601
writecache_error(wc, r, "error flushing metadata: %d", r);
602
602
}
···
990
990
req.client = wc->dm_io;
991
991
req.notify.fn = NULL;
992
992
993
-
return dm_io(&req, 1, ®ion, NULL);
993
+
return dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT);
994
994
}
995
995
996
996
static void writecache_resume(struct dm_target *ti)
+2
-1
include/linux/dm-io.h
+2
-1
include/linux/dm-io.h
···
80
80
* error occurred doing io to the corresponding region.
81
81
*/
82
82
int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
83
-
struct dm_io_region *region, unsigned int long *sync_error_bits);
83
+
struct dm_io_region *region, unsigned int long *sync_error_bits,
84
+
unsigned short ioprio);
84
85
85
86
#endif /* __KERNEL__ */
86
87
#endif /* _LINUX_DM_IO_H */