Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
dm: Use the block layer zone append emulation
For targets requiring zone append operation emulation with regular
writes (e.g. dm-crypt), we can use the block layer emulation provided by
zone write plugging. Remove DM implemented zone append emulation and
enable the block layer one.
This is done by setting the max_zone_append_sectors limit of the
mapped device queue to 0 for mapped devices that have a target table
that cannot support native zone append operations (e.g. dm-crypt).
Such mapped devices are flagged with the DMF_EMULATE_ZONE_APPEND flag.
dm_split_and_process_bio() is modified to execute
blk_zone_write_plug_bio() for such device to let the block layer
transform zone append operations into regular writes. This is done
after ensuring that the submitted BIO is split if it straddles zone
boundaries. Both changes are implemented unsing the inline helpers
dm_zone_write_plug_bio() and dm_zone_bio_needs_split() respectively.
dm_revalidate_zones() is also modified to use the block layer provided
function blk_revalidate_disk_zones() so that all zone resources needed
for zone append emulation are initialized by the block layer without DM
core needing to do anything. Since the device table is not yet live when
dm_revalidate_zones() is executed, enabling the use of
blk_revalidate_disk_zones() requires adding a pointer to the device
table in struct mapped_device. This avoids errors in
dm_blk_report_zones() trying to get the table with dm_get_live_table().
The mapped device table pointer is set to the table passed as argument
to dm_revalidate_zones() before calling blk_revalidate_disk_zones() and
reset to NULL after this function returns to restore the live table
handling for user call of report zones.
All the code related to zone append emulation is removed from
dm-zone.c. This leads to simplifications of the functions __map_bio()
and dm_zone_endio(). This later function now only needs to deal with
completions of real zone append operations for targets that support it.
Signed-off-by: Damien Le Moal <dlemoal@kernel.org>
Reviewed-by: Mike Snitzer <snitzer@kernel.org>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Tested-by: Hans Holmberg <hans.holmberg@wdc.com>
Tested-by: Dennis Maisenbacher <dennis.maisenbacher@wdc.com>
Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
Link: https://lore.kernel.org/r/20240408014128.205141-13-dlemoal@kernel.org
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Damien Le Moal
and committed by
Jens Axboe
f211268e
946dd71e
+94
-460
+1
-1
drivers/md/dm-core.h
+1
-1
drivers/md/dm-core.h
+44
-434
drivers/md/dm-zone.c
+44
-434
drivers/md/dm-zone.c
···
60
60
struct dm_table *map;
61
61
int srcu_idx, ret;
62
62
63
-
if (dm_suspended_md(md))
64
-
return -EAGAIN;
63
+
if (!md->zone_revalidate_map) {
64
+
/* Regular user context */
65
+
if (dm_suspended_md(md))
66
+
return -EAGAIN;
65
67
66
-
map = dm_get_live_table(md, &srcu_idx);
67
-
if (!map)
68
-
return -EIO;
68
+
map = dm_get_live_table(md, &srcu_idx);
69
+
if (!map)
70
+
return -EIO;
71
+
} else {
72
+
/* Zone revalidation during __bind() */
73
+
map = md->zone_revalidate_map;
74
+
}
69
75
70
76
ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data);
71
77
72
-
dm_put_live_table(md, srcu_idx);
78
+
if (!md->zone_revalidate_map)
79
+
dm_put_live_table(md, srcu_idx);
73
80
74
81
return ret;
75
82
}
···
145
138
}
146
139
}
147
140
148
-
void dm_cleanup_zoned_dev(struct mapped_device *md)
149
-
{
150
-
if (md->disk) {
151
-
bitmap_free(md->disk->conv_zones_bitmap);
152
-
md->disk->conv_zones_bitmap = NULL;
153
-
bitmap_free(md->disk->seq_zones_wlock);
154
-
md->disk->seq_zones_wlock = NULL;
155
-
}
156
-
157
-
kvfree(md->zwp_offset);
158
-
md->zwp_offset = NULL;
159
-
md->nr_zones = 0;
160
-
}
161
-
162
-
static unsigned int dm_get_zone_wp_offset(struct blk_zone *zone)
163
-
{
164
-
switch (zone->cond) {
165
-
case BLK_ZONE_COND_IMP_OPEN:
166
-
case BLK_ZONE_COND_EXP_OPEN:
167
-
case BLK_ZONE_COND_CLOSED:
168
-
return zone->wp - zone->start;
169
-
case BLK_ZONE_COND_FULL:
170
-
return zone->len;
171
-
case BLK_ZONE_COND_EMPTY:
172
-
case BLK_ZONE_COND_NOT_WP:
173
-
case BLK_ZONE_COND_OFFLINE:
174
-
case BLK_ZONE_COND_READONLY:
175
-
default:
176
-
/*
177
-
* Conventional, offline and read-only zones do not have a valid
178
-
* write pointer. Use 0 as for an empty zone.
179
-
*/
180
-
return 0;
181
-
}
182
-
}
183
-
184
-
static int dm_zone_revalidate_cb(struct blk_zone *zone, unsigned int idx,
185
-
void *data)
186
-
{
187
-
struct mapped_device *md = data;
188
-
struct gendisk *disk = md->disk;
189
-
190
-
switch (zone->type) {
191
-
case BLK_ZONE_TYPE_CONVENTIONAL:
192
-
if (!disk->conv_zones_bitmap) {
193
-
disk->conv_zones_bitmap = bitmap_zalloc(disk->nr_zones,
194
-
GFP_NOIO);
195
-
if (!disk->conv_zones_bitmap)
196
-
return -ENOMEM;
197
-
}
198
-
set_bit(idx, disk->conv_zones_bitmap);
199
-
break;
200
-
case BLK_ZONE_TYPE_SEQWRITE_REQ:
201
-
case BLK_ZONE_TYPE_SEQWRITE_PREF:
202
-
if (!disk->seq_zones_wlock) {
203
-
disk->seq_zones_wlock = bitmap_zalloc(disk->nr_zones,
204
-
GFP_NOIO);
205
-
if (!disk->seq_zones_wlock)
206
-
return -ENOMEM;
207
-
}
208
-
if (!md->zwp_offset) {
209
-
md->zwp_offset =
210
-
kvcalloc(disk->nr_zones, sizeof(unsigned int),
211
-
GFP_KERNEL);
212
-
if (!md->zwp_offset)
213
-
return -ENOMEM;
214
-
}
215
-
md->zwp_offset[idx] = dm_get_zone_wp_offset(zone);
216
-
217
-
break;
218
-
default:
219
-
DMERR("Invalid zone type 0x%x at sectors %llu",
220
-
(int)zone->type, zone->start);
221
-
return -ENODEV;
222
-
}
223
-
224
-
return 0;
225
-
}
226
-
227
141
/*
228
142
* Revalidate the zones of a mapped device to initialize resource necessary
229
143
* for zone append emulation. Note that we cannot simply use the block layer
···
154
226
static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t)
155
227
{
156
228
struct gendisk *disk = md->disk;
157
-
unsigned int noio_flag;
158
229
int ret;
159
230
160
-
/*
161
-
* Check if something changed. If yes, cleanup the current resources
162
-
* and reallocate everything.
163
-
*/
231
+
/* Revalidate only if something changed. */
164
232
if (!disk->nr_zones || disk->nr_zones != md->nr_zones)
165
-
dm_cleanup_zoned_dev(md);
233
+
md->nr_zones = 0;
234
+
166
235
if (md->nr_zones)
167
236
return 0;
168
237
169
238
/*
170
-
* Scan all zones to initialize everything. Ensure that all vmalloc
171
-
* operations in this context are done as if GFP_NOIO was specified.
239
+
* Our table is not live yet. So the call to dm_get_live_table()
240
+
* in dm_blk_report_zones() will fail. Set a temporary pointer to
241
+
* our table for dm_blk_report_zones() to use directly.
172
242
*/
173
-
noio_flag = memalloc_noio_save();
174
-
ret = dm_blk_do_report_zones(md, t, 0, disk->nr_zones,
175
-
dm_zone_revalidate_cb, md);
176
-
memalloc_noio_restore(noio_flag);
177
-
if (ret < 0)
178
-
goto err;
179
-
if (ret != disk->nr_zones) {
180
-
ret = -EIO;
181
-
goto err;
243
+
md->zone_revalidate_map = t;
244
+
ret = blk_revalidate_disk_zones(disk, NULL);
245
+
md->zone_revalidate_map = NULL;
246
+
247
+
if (ret) {
248
+
DMERR("Revalidate zones failed %d", ret);
249
+
return ret;
182
250
}
183
251
184
252
md->nr_zones = disk->nr_zones;
185
253
186
254
return 0;
187
-
188
-
err:
189
-
DMERR("Revalidate zones failed %d", ret);
190
-
dm_cleanup_zoned_dev(md);
191
-
return ret;
192
255
}
193
256
194
257
static int device_not_zone_append_capable(struct dm_target *ti,
···
210
291
struct mapped_device *md = t->md;
211
292
212
293
/*
213
-
* For a zoned target, the number of zones should be updated for the
214
-
* correct value to be exposed in sysfs queue/nr_zones.
294
+
* Check if zone append is natively supported, and if not, set the
295
+
* mapped device queue as needing zone append emulation.
215
296
*/
216
297
WARN_ON_ONCE(queue_is_mq(q));
217
-
md->disk->nr_zones = bdev_nr_zones(md->disk->part0);
218
-
219
-
/* Check if zone append is natively supported */
220
298
if (dm_table_supports_zone_append(t)) {
221
299
clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
222
-
dm_cleanup_zoned_dev(md);
223
-
return 0;
300
+
} else {
301
+
set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
302
+
blk_queue_max_zone_append_sectors(q, 0);
224
303
}
225
304
226
-
/*
227
-
* Mark the mapped device as needing zone append emulation and
228
-
* initialize the emulation resources once the capacity is set.
229
-
*/
230
-
set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
231
305
if (!get_capacity(md->disk))
232
306
return 0;
233
307
308
+
if (!md->disk->nr_zones) {
309
+
DMINFO("%s using %s zone append",
310
+
md->disk->disk_name,
311
+
queue_emulates_zone_append(q) ? "emulated" : "native");
312
+
}
313
+
234
314
return dm_revalidate_zones(md, t);
235
-
}
236
-
237
-
static int dm_update_zone_wp_offset_cb(struct blk_zone *zone, unsigned int idx,
238
-
void *data)
239
-
{
240
-
unsigned int *wp_offset = data;
241
-
242
-
*wp_offset = dm_get_zone_wp_offset(zone);
243
-
244
-
return 0;
245
-
}
246
-
247
-
static int dm_update_zone_wp_offset(struct mapped_device *md, unsigned int zno,
248
-
unsigned int *wp_ofst)
249
-
{
250
-
sector_t sector = zno * bdev_zone_sectors(md->disk->part0);
251
-
unsigned int noio_flag;
252
-
struct dm_table *t;
253
-
int srcu_idx, ret;
254
-
255
-
t = dm_get_live_table(md, &srcu_idx);
256
-
if (!t)
257
-
return -EIO;
258
-
259
-
/*
260
-
* Ensure that all memory allocations in this context are done as if
261
-
* GFP_NOIO was specified.
262
-
*/
263
-
noio_flag = memalloc_noio_save();
264
-
ret = dm_blk_do_report_zones(md, t, sector, 1,
265
-
dm_update_zone_wp_offset_cb, wp_ofst);
266
-
memalloc_noio_restore(noio_flag);
267
-
268
-
dm_put_live_table(md, srcu_idx);
269
-
270
-
if (ret != 1)
271
-
return -EIO;
272
-
273
-
return 0;
274
-
}
275
-
276
-
struct orig_bio_details {
277
-
enum req_op op;
278
-
unsigned int nr_sectors;
279
-
};
280
-
281
-
/*
282
-
* First phase of BIO mapping for targets with zone append emulation:
283
-
* check all BIO that change a zone writer pointer and change zone
284
-
* append operations into regular write operations.
285
-
*/
286
-
static bool dm_zone_map_bio_begin(struct mapped_device *md,
287
-
unsigned int zno, struct bio *clone)
288
-
{
289
-
sector_t zsectors = bdev_zone_sectors(md->disk->part0);
290
-
unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]);
291
-
292
-
/*
293
-
* If the target zone is in an error state, recover by inspecting the
294
-
* zone to get its current write pointer position. Note that since the
295
-
* target zone is already locked, a BIO issuing context should never
296
-
* see the zone write in the DM_ZONE_UPDATING_WP_OFST state.
297
-
*/
298
-
if (zwp_offset == DM_ZONE_INVALID_WP_OFST) {
299
-
if (dm_update_zone_wp_offset(md, zno, &zwp_offset))
300
-
return false;
301
-
WRITE_ONCE(md->zwp_offset[zno], zwp_offset);
302
-
}
303
-
304
-
switch (bio_op(clone)) {
305
-
case REQ_OP_ZONE_RESET:
306
-
case REQ_OP_ZONE_FINISH:
307
-
return true;
308
-
case REQ_OP_WRITE_ZEROES:
309
-
case REQ_OP_WRITE:
310
-
/* Writes must be aligned to the zone write pointer */
311
-
if ((clone->bi_iter.bi_sector & (zsectors - 1)) != zwp_offset)
312
-
return false;
313
-
break;
314
-
case REQ_OP_ZONE_APPEND:
315
-
/*
316
-
* Change zone append operations into a non-mergeable regular
317
-
* writes directed at the current write pointer position of the
318
-
* target zone.
319
-
*/
320
-
clone->bi_opf = REQ_OP_WRITE | REQ_NOMERGE |
321
-
(clone->bi_opf & (~REQ_OP_MASK));
322
-
clone->bi_iter.bi_sector += zwp_offset;
323
-
break;
324
-
default:
325
-
DMWARN_LIMIT("Invalid BIO operation");
326
-
return false;
327
-
}
328
-
329
-
/* Cannot write to a full zone */
330
-
if (zwp_offset >= zsectors)
331
-
return false;
332
-
333
-
return true;
334
-
}
335
-
336
-
/*
337
-
* Second phase of BIO mapping for targets with zone append emulation:
338
-
* update the zone write pointer offset array to account for the additional
339
-
* data written to a zone. Note that at this point, the remapped clone BIO
340
-
* may already have completed, so we do not touch it.
341
-
*/
342
-
static blk_status_t dm_zone_map_bio_end(struct mapped_device *md, unsigned int zno,
343
-
struct orig_bio_details *orig_bio_details,
344
-
unsigned int nr_sectors)
345
-
{
346
-
unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]);
347
-
348
-
/* The clone BIO may already have been completed and failed */
349
-
if (zwp_offset == DM_ZONE_INVALID_WP_OFST)
350
-
return BLK_STS_IOERR;
351
-
352
-
/* Update the zone wp offset */
353
-
switch (orig_bio_details->op) {
354
-
case REQ_OP_ZONE_RESET:
355
-
WRITE_ONCE(md->zwp_offset[zno], 0);
356
-
return BLK_STS_OK;
357
-
case REQ_OP_ZONE_FINISH:
358
-
WRITE_ONCE(md->zwp_offset[zno],
359
-
bdev_zone_sectors(md->disk->part0));
360
-
return BLK_STS_OK;
361
-
case REQ_OP_WRITE_ZEROES:
362
-
case REQ_OP_WRITE:
363
-
WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors);
364
-
return BLK_STS_OK;
365
-
case REQ_OP_ZONE_APPEND:
366
-
/*
367
-
* Check that the target did not truncate the write operation
368
-
* emulating a zone append.
369
-
*/
370
-
if (nr_sectors != orig_bio_details->nr_sectors) {
371
-
DMWARN_LIMIT("Truncated write for zone append");
372
-
return BLK_STS_IOERR;
373
-
}
374
-
WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors);
375
-
return BLK_STS_OK;
376
-
default:
377
-
DMWARN_LIMIT("Invalid BIO operation");
378
-
return BLK_STS_IOERR;
379
-
}
380
-
}
381
-
382
-
static inline void dm_zone_lock(struct gendisk *disk, unsigned int zno,
383
-
struct bio *clone)
384
-
{
385
-
if (WARN_ON_ONCE(bio_flagged(clone, BIO_ZONE_WRITE_LOCKED)))
386
-
return;
387
-
388
-
wait_on_bit_lock_io(disk->seq_zones_wlock, zno, TASK_UNINTERRUPTIBLE);
389
-
bio_set_flag(clone, BIO_ZONE_WRITE_LOCKED);
390
-
}
391
-
392
-
static inline void dm_zone_unlock(struct gendisk *disk, unsigned int zno,
393
-
struct bio *clone)
394
-
{
395
-
if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))
396
-
return;
397
-
398
-
WARN_ON_ONCE(!test_bit(zno, disk->seq_zones_wlock));
399
-
clear_bit_unlock(zno, disk->seq_zones_wlock);
400
-
smp_mb__after_atomic();
401
-
wake_up_bit(disk->seq_zones_wlock, zno);
402
-
403
-
bio_clear_flag(clone, BIO_ZONE_WRITE_LOCKED);
404
-
}
405
-
406
-
static bool dm_need_zone_wp_tracking(struct bio *bio)
407
-
{
408
-
/*
409
-
* Special processing is not needed for operations that do not need the
410
-
* zone write lock, that is, all operations that target conventional
411
-
* zones and all operations that do not modify directly a sequential
412
-
* zone write pointer.
413
-
*/
414
-
if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
415
-
return false;
416
-
switch (bio_op(bio)) {
417
-
case REQ_OP_WRITE_ZEROES:
418
-
case REQ_OP_WRITE:
419
-
case REQ_OP_ZONE_RESET:
420
-
case REQ_OP_ZONE_FINISH:
421
-
case REQ_OP_ZONE_APPEND:
422
-
return bio_zone_is_seq(bio);
423
-
default:
424
-
return false;
425
-
}
426
-
}
427
-
428
-
/*
429
-
* Special IO mapping for targets needing zone append emulation.
430
-
*/
431
-
int dm_zone_map_bio(struct dm_target_io *tio)
432
-
{
433
-
struct dm_io *io = tio->io;
434
-
struct dm_target *ti = tio->ti;
435
-
struct mapped_device *md = io->md;
436
-
struct bio *clone = &tio->clone;
437
-
struct orig_bio_details orig_bio_details;
438
-
unsigned int zno;
439
-
blk_status_t sts;
440
-
int r;
441
-
442
-
/*
443
-
* IOs that do not change a zone write pointer do not need
444
-
* any additional special processing.
445
-
*/
446
-
if (!dm_need_zone_wp_tracking(clone))
447
-
return ti->type->map(ti, clone);
448
-
449
-
/* Lock the target zone */
450
-
zno = bio_zone_no(clone);
451
-
dm_zone_lock(md->disk, zno, clone);
452
-
453
-
orig_bio_details.nr_sectors = bio_sectors(clone);
454
-
orig_bio_details.op = bio_op(clone);
455
-
456
-
/*
457
-
* Check that the bio and the target zone write pointer offset are
458
-
* both valid, and if the bio is a zone append, remap it to a write.
459
-
*/
460
-
if (!dm_zone_map_bio_begin(md, zno, clone)) {
461
-
dm_zone_unlock(md->disk, zno, clone);
462
-
return DM_MAPIO_KILL;
463
-
}
464
-
465
-
/* Let the target do its work */
466
-
r = ti->type->map(ti, clone);
467
-
switch (r) {
468
-
case DM_MAPIO_SUBMITTED:
469
-
/*
470
-
* The target submitted the clone BIO. The target zone will
471
-
* be unlocked on completion of the clone.
472
-
*/
473
-
sts = dm_zone_map_bio_end(md, zno, &orig_bio_details,
474
-
*tio->len_ptr);
475
-
break;
476
-
case DM_MAPIO_REMAPPED:
477
-
/*
478
-
* The target only remapped the clone BIO. In case of error,
479
-
* unlock the target zone here as the clone will not be
480
-
* submitted.
481
-
*/
482
-
sts = dm_zone_map_bio_end(md, zno, &orig_bio_details,
483
-
*tio->len_ptr);
484
-
if (sts != BLK_STS_OK)
485
-
dm_zone_unlock(md->disk, zno, clone);
486
-
break;
487
-
case DM_MAPIO_REQUEUE:
488
-
case DM_MAPIO_KILL:
489
-
default:
490
-
dm_zone_unlock(md->disk, zno, clone);
491
-
sts = BLK_STS_IOERR;
492
-
break;
493
-
}
494
-
495
-
if (sts != BLK_STS_OK)
496
-
return DM_MAPIO_KILL;
497
-
498
-
return r;
499
315
}
500
316
501
317
/*
···
241
587
struct mapped_device *md = io->md;
242
588
struct gendisk *disk = md->disk;
243
589
struct bio *orig_bio = io->orig_bio;
244
-
unsigned int zwp_offset;
245
-
unsigned int zno;
246
590
247
591
/*
248
-
* For targets that do not emulate zone append, we only need to
249
-
* handle native zone-append bios.
592
+
* Get the offset within the zone of the written sector
593
+
* and add that to the original bio sector position.
250
594
*/
251
-
if (!dm_emulate_zone_append(md)) {
252
-
/*
253
-
* Get the offset within the zone of the written sector
254
-
* and add that to the original bio sector position.
255
-
*/
256
-
if (clone->bi_status == BLK_STS_OK &&
257
-
bio_op(clone) == REQ_OP_ZONE_APPEND) {
258
-
sector_t mask =
259
-
(sector_t)bdev_zone_sectors(disk->part0) - 1;
595
+
if (clone->bi_status == BLK_STS_OK &&
596
+
bio_op(clone) == REQ_OP_ZONE_APPEND) {
597
+
sector_t mask = bdev_zone_sectors(disk->part0) - 1;
260
598
261
-
orig_bio->bi_iter.bi_sector +=
262
-
clone->bi_iter.bi_sector & mask;
263
-
}
264
-
265
-
return;
599
+
orig_bio->bi_iter.bi_sector += clone->bi_iter.bi_sector & mask;
266
600
}
267
601
268
-
/*
269
-
* For targets that do emulate zone append, if the clone BIO does not
270
-
* own the target zone write lock, we have nothing to do.
271
-
*/
272
-
if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))
273
-
return;
274
-
275
-
zno = bio_zone_no(orig_bio);
276
-
277
-
if (clone->bi_status != BLK_STS_OK) {
278
-
/*
279
-
* BIOs that modify a zone write pointer may leave the zone
280
-
* in an unknown state in case of failure (e.g. the write
281
-
* pointer was only partially advanced). In this case, set
282
-
* the target zone write pointer as invalid unless it is
283
-
* already being updated.
284
-
*/
285
-
WRITE_ONCE(md->zwp_offset[zno], DM_ZONE_INVALID_WP_OFST);
286
-
} else if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) {
287
-
/*
288
-
* Get the written sector for zone append operation that were
289
-
* emulated using regular write operations.
290
-
*/
291
-
zwp_offset = READ_ONCE(md->zwp_offset[zno]);
292
-
if (WARN_ON_ONCE(zwp_offset < bio_sectors(orig_bio)))
293
-
WRITE_ONCE(md->zwp_offset[zno],
294
-
DM_ZONE_INVALID_WP_OFST);
295
-
else
296
-
orig_bio->bi_iter.bi_sector +=
297
-
zwp_offset - bio_sectors(orig_bio);
298
-
}
299
-
300
-
dm_zone_unlock(disk, zno, clone);
602
+
return;
301
603
}
+49
-23
drivers/md/dm.c
+49
-23
drivers/md/dm.c
···
1422
1422
down(&md->swap_bios_semaphore);
1423
1423
}
1424
1424
1425
-
if (static_branch_unlikely(&zoned_enabled)) {
1426
-
/*
1427
-
* Check if the IO needs a special mapping due to zone append
1428
-
* emulation on zoned target. In this case, dm_zone_map_bio()
1429
-
* calls the target map operation.
1430
-
*/
1431
-
if (unlikely(dm_emulate_zone_append(md)))
1432
-
r = dm_zone_map_bio(tio);
1433
-
else
1434
-
goto do_map;
1435
-
} else {
1436
-
do_map:
1437
-
if (likely(ti->type->map == linear_map))
1438
-
r = linear_map(ti, clone);
1439
-
else if (ti->type->map == stripe_map)
1440
-
r = stripe_map(ti, clone);
1441
-
else
1442
-
r = ti->type->map(ti, clone);
1443
-
}
1425
+
if (likely(ti->type->map == linear_map))
1426
+
r = linear_map(ti, clone);
1427
+
else if (ti->type->map == stripe_map)
1428
+
r = stripe_map(ti, clone);
1429
+
else
1430
+
r = ti->type->map(ti, clone);
1444
1431
1445
1432
switch (r) {
1446
1433
case DM_MAPIO_SUBMITTED:
···
1755
1768
ci->sector_count = 0;
1756
1769
}
1757
1770
1771
+
#ifdef CONFIG_BLK_DEV_ZONED
1772
+
static inline bool dm_zone_bio_needs_split(struct mapped_device *md,
1773
+
struct bio *bio)
1774
+
{
1775
+
/*
1776
+
* For mapped device that need zone append emulation, we must
1777
+
* split any large BIO that straddles zone boundaries.
1778
+
*/
1779
+
return dm_emulate_zone_append(md) && bio_straddles_zones(bio) &&
1780
+
!bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING);
1781
+
}
1782
+
static inline bool dm_zone_plug_bio(struct mapped_device *md, struct bio *bio)
1783
+
{
1784
+
return dm_emulate_zone_append(md) && blk_zone_plug_bio(bio, 0);
1785
+
}
1786
+
#else
1787
+
static inline bool dm_zone_bio_needs_split(struct mapped_device *md,
1788
+
struct bio *bio)
1789
+
{
1790
+
return false;
1791
+
}
1792
+
static inline bool dm_zone_plug_bio(struct mapped_device *md, struct bio *bio)
1793
+
{
1794
+
return false;
1795
+
}
1796
+
#endif
1797
+
1758
1798
/*
1759
1799
* Entry point to split a bio into clones and submit them to the targets.
1760
1800
*/
···
1791
1777
struct clone_info ci;
1792
1778
struct dm_io *io;
1793
1779
blk_status_t error = BLK_STS_OK;
1794
-
bool is_abnormal;
1780
+
bool is_abnormal, need_split;
1795
1781
1796
-
is_abnormal = is_abnormal_io(bio);
1797
-
if (unlikely(is_abnormal)) {
1782
+
need_split = is_abnormal = is_abnormal_io(bio);
1783
+
if (static_branch_unlikely(&zoned_enabled))
1784
+
need_split = is_abnormal || dm_zone_bio_needs_split(md, bio);
1785
+
1786
+
if (unlikely(need_split)) {
1798
1787
/*
1799
1788
* Use bio_split_to_limits() for abnormal IO (e.g. discard, etc)
1800
1789
* otherwise associated queue_limits won't be imposed.
1790
+
* Also split the BIO for mapped devices needing zone append
1791
+
* emulation to ensure that the BIO does not cross zone
1792
+
* boundaries.
1801
1793
*/
1802
1794
bio = bio_split_to_limits(bio);
1803
1795
if (!bio)
1804
1796
return;
1805
1797
}
1798
+
1799
+
/*
1800
+
* Use the block layer zone write plugging for mapped devices that
1801
+
* need zone append emulation (e.g. dm-crypt).
1802
+
*/
1803
+
if (static_branch_unlikely(&zoned_enabled) && dm_zone_plug_bio(md, bio))
1804
+
return;
1806
1805
1807
1806
/* Only support nowait for normal IO */
1808
1807
if (unlikely(bio->bi_opf & REQ_NOWAIT) && !is_abnormal) {
···
2037
2010
md->dax_dev = NULL;
2038
2011
}
2039
2012
2040
-
dm_cleanup_zoned_dev(md);
2041
2013
if (md->disk) {
2042
2014
spin_lock(&_minor_lock);
2043
2015
md->disk->private_data = NULL;
-2
drivers/md/dm.h
-2
drivers/md/dm.h
···
104
104
int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q);
105
105
void dm_zone_endio(struct dm_io *io, struct bio *clone);
106
106
#ifdef CONFIG_BLK_DEV_ZONED
107
-
void dm_cleanup_zoned_dev(struct mapped_device *md);
108
107
int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
109
108
unsigned int nr_zones, report_zones_cb cb, void *data);
110
109
bool dm_is_zone_write(struct mapped_device *md, struct bio *bio);
111
110
int dm_zone_map_bio(struct dm_target_io *io);
112
111
#else
113
-
static inline void dm_cleanup_zoned_dev(struct mapped_device *md) {}
114
112
#define dm_blk_report_zones NULL
115
113
static inline bool dm_is_zone_write(struct mapped_device *md, struct bio *bio)
116
114
{