Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
block: use bio_alloc_bioset for passthru IO by default
Use bio_alloc_bioset for passthru IO by default, so that we can enable
bio cache for irq and polled passthru IO in later.
Signed-off-by: Fengnan Chang <changfengnan@bytedance.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Fengnan Chang
and committed by
Jens Axboe
05ce4c58
559e608c
+37
-55
+36
-54
block/blk-map.c
+36
-54
block/blk-map.c
···
37
37
return bmd;
38
38
}
39
39
40
+
static inline void blk_mq_map_bio_put(struct bio *bio)
41
+
{
42
+
bio_put(bio);
43
+
}
44
+
45
+
static struct bio *blk_rq_map_bio_alloc(struct request *rq,
46
+
unsigned int nr_vecs, gfp_t gfp_mask)
47
+
{
48
+
struct block_device *bdev = rq->q->disk ? rq->q->disk->part0 : NULL;
49
+
struct bio *bio;
50
+
51
+
bio = bio_alloc_bioset(bdev, nr_vecs, rq->cmd_flags, gfp_mask,
52
+
&fs_bio_set);
53
+
if (!bio)
54
+
return NULL;
55
+
56
+
return bio;
57
+
}
58
+
40
59
/**
41
60
* bio_copy_from_iter - copy all pages from iov_iter to bio
42
61
* @bio: The &struct bio which describes the I/O as destination
···
173
154
nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE));
174
155
175
156
ret = -ENOMEM;
176
-
bio = bio_kmalloc(nr_pages, gfp_mask);
157
+
bio = blk_rq_map_bio_alloc(rq, nr_pages, gfp_mask);
177
158
if (!bio)
178
159
goto out_bmd;
179
-
bio_init_inline(bio, NULL, nr_pages, req_op(rq));
180
160
181
161
if (map_data) {
182
162
nr_pages = 1U << map_data->page_order;
···
251
233
cleanup:
252
234
if (!map_data)
253
235
bio_free_pages(bio);
254
-
bio_uninit(bio);
255
-
kfree(bio);
236
+
blk_mq_map_bio_put(bio);
256
237
out_bmd:
257
238
kfree(bmd);
258
239
return ret;
259
-
}
260
-
261
-
static void blk_mq_map_bio_put(struct bio *bio)
262
-
{
263
-
if (bio->bi_opf & REQ_ALLOC_CACHE) {
264
-
bio_put(bio);
265
-
} else {
266
-
bio_uninit(bio);
267
-
kfree(bio);
268
-
}
269
-
}
270
-
271
-
static struct bio *blk_rq_map_bio_alloc(struct request *rq,
272
-
unsigned int nr_vecs, gfp_t gfp_mask)
273
-
{
274
-
struct block_device *bdev = rq->q->disk ? rq->q->disk->part0 : NULL;
275
-
struct bio *bio;
276
-
277
-
if (rq->cmd_flags & REQ_ALLOC_CACHE && (nr_vecs <= BIO_INLINE_VECS)) {
278
-
bio = bio_alloc_bioset(bdev, nr_vecs, rq->cmd_flags, gfp_mask,
279
-
&fs_bio_set);
280
-
if (!bio)
281
-
return NULL;
282
-
} else {
283
-
bio = bio_kmalloc(nr_vecs, gfp_mask);
284
-
if (!bio)
285
-
return NULL;
286
-
bio_init_inline(bio, bdev, nr_vecs, req_op(rq));
287
-
}
288
-
return bio;
289
240
}
290
241
291
242
static int bio_map_user_iov(struct request *rq, struct iov_iter *iter,
···
305
318
static void bio_map_kern_endio(struct bio *bio)
306
319
{
307
320
bio_invalidate_vmalloc_pages(bio);
308
-
bio_uninit(bio);
309
-
kfree(bio);
321
+
blk_mq_map_bio_put(bio);
310
322
}
311
323
312
-
static struct bio *bio_map_kern(void *data, unsigned int len, enum req_op op,
324
+
static struct bio *bio_map_kern(struct request *rq, void *data, unsigned int len,
313
325
gfp_t gfp_mask)
314
326
{
315
327
unsigned int nr_vecs = bio_add_max_vecs(data, len);
316
328
struct bio *bio;
317
329
318
-
bio = bio_kmalloc(nr_vecs, gfp_mask);
330
+
bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask);
319
331
if (!bio)
320
332
return ERR_PTR(-ENOMEM);
321
-
bio_init_inline(bio, NULL, nr_vecs, op);
333
+
322
334
if (is_vmalloc_addr(data)) {
323
335
bio->bi_private = data;
324
336
if (!bio_add_vmalloc(bio, data, len)) {
325
-
bio_uninit(bio);
326
-
kfree(bio);
337
+
blk_mq_map_bio_put(bio);
327
338
return ERR_PTR(-EINVAL);
328
339
}
329
340
} else {
···
334
349
static void bio_copy_kern_endio(struct bio *bio)
335
350
{
336
351
bio_free_pages(bio);
337
-
bio_uninit(bio);
338
-
kfree(bio);
352
+
blk_mq_map_bio_put(bio);
339
353
}
340
354
341
355
static void bio_copy_kern_endio_read(struct bio *bio)
···
353
369
354
370
/**
355
371
* bio_copy_kern - copy kernel address into bio
372
+
* @rq: request to fill
356
373
* @data: pointer to buffer to copy
357
374
* @len: length in bytes
358
375
* @op: bio/request operation
···
362
377
* copy the kernel address into a bio suitable for io to a block
363
378
* device. Returns an error pointer in case of error.
364
379
*/
365
-
static struct bio *bio_copy_kern(void *data, unsigned int len, enum req_op op,
380
+
static struct bio *bio_copy_kern(struct request *rq, void *data, unsigned int len,
366
381
gfp_t gfp_mask)
367
382
{
383
+
enum req_op op = req_op(rq);
368
384
unsigned long kaddr = (unsigned long)data;
369
385
unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
370
386
unsigned long start = kaddr >> PAGE_SHIFT;
···
380
394
return ERR_PTR(-EINVAL);
381
395
382
396
nr_pages = end - start;
383
-
bio = bio_kmalloc(nr_pages, gfp_mask);
397
+
bio = blk_rq_map_bio_alloc(rq, nr_pages, gfp_mask);
384
398
if (!bio)
385
399
return ERR_PTR(-ENOMEM);
386
-
bio_init_inline(bio, NULL, nr_pages, op);
387
400
388
401
while (len) {
389
402
struct page *page;
···
416
431
417
432
cleanup:
418
433
bio_free_pages(bio);
419
-
bio_uninit(bio);
420
-
kfree(bio);
434
+
blk_mq_map_bio_put(bio);
421
435
return ERR_PTR(-ENOMEM);
422
436
}
423
437
···
663
679
return -EINVAL;
664
680
665
681
if (!blk_rq_aligned(rq->q, addr, len) || object_is_on_stack(kbuf))
666
-
bio = bio_copy_kern(kbuf, len, req_op(rq), gfp_mask);
682
+
bio = bio_copy_kern(rq, kbuf, len, gfp_mask);
667
683
else
668
-
bio = bio_map_kern(kbuf, len, req_op(rq), gfp_mask);
684
+
bio = bio_map_kern(rq, kbuf, len, gfp_mask);
669
685
670
686
if (IS_ERR(bio))
671
687
return PTR_ERR(bio);
672
688
673
689
ret = blk_rq_append_bio(rq, bio);
674
-
if (unlikely(ret)) {
675
-
bio_uninit(bio);
676
-
kfree(bio);
677
-
}
690
+
if (unlikely(ret))
691
+
blk_mq_map_bio_put(bio);
678
692
return ret;
679
693
}
680
694
EXPORT_SYMBOL(blk_rq_map_kern);