Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'vfs-7.1-rc1.integrity' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs
Pull vfs integrity updates from Christian Brauner:
"This adds support to generate and verify integrity information (aka
T10 PI) in the file system, instead of the automatic below the covers
support that is currently used.
The implementation is based on refactoring the existing block layer PI
code to be reusable for this use case, and then adding relatively
small wrappers for the file system use case. These are then used in
iomap to implement the semantics, and wired up in XFS with a small
amount of glue code.
Compared to the baseline this does not change performance for writes,
but increases read performance up to 15% for 4k I/O, with the benefit
decreasing with larger I/O sizes as even the baseline maxes out the
device quickly on my older enterprise SSD"
* tag 'vfs-7.1-rc1.integrity' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs:
xfs: support T10 protection information
iomap: support T10 protection information
iomap: support ioends for buffered reads
iomap: add a bioset pointer to iomap_read_folio_ops
ntfs3: remove copy and pasted iomap code
iomap: allow file systems to hook into buffered read bio submission
iomap: only call into ->submit_read when there is a read_ctx
iomap: pass the iomap_iter to ->submit_read
iomap: refactor iomap_bio_read_folio_range
block: pass a maxlen argument to bio_iov_iter_bounce
block: add fs_bio_integrity helpers
block: make max_integrity_io_size public
block: prepare generation / verification helpers for fs usage
block: add a bdev_has_integrity_csum helper
block: factor out a bio_integrity_setup_default helper
block: factor out a bio_integrity_action helper
+468
-236
+1
-1
block/Makefile
+1
-1
block/Makefile
···
26
26
obj-$(CONFIG_IOSCHED_BFQ) += bfq.o
27
27
28
28
obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o \
29
-
bio-integrity-auto.o
29
+
bio-integrity-auto.o bio-integrity-fs.o
30
30
obj-$(CONFIG_BLK_DEV_ZONED) += blk-zoned.o
31
31
obj-$(CONFIG_BLK_WBT) += blk-wbt.o
32
32
obj-$(CONFIG_BLK_DEBUG_FS) += blk-mq-debugfs.o
+11
-69
block/bio-integrity-auto.c
+11
-69
block/bio-integrity-auto.c
···
39
39
container_of(work, struct bio_integrity_data, work);
40
40
struct bio *bio = bid->bio;
41
41
42
-
blk_integrity_verify_iter(bio, &bid->saved_bio_iter);
42
+
bio->bi_status = bio_integrity_verify(bio, &bid->saved_bio_iter);
43
43
bio_integrity_finish(bid);
44
44
bio_endio(bio);
45
45
}
···
48
48
static bool bip_should_check(struct bio_integrity_payload *bip)
49
49
{
50
50
return bip->bip_flags & BIP_CHECK_FLAGS;
51
-
}
52
-
53
-
static bool bi_offload_capable(struct blk_integrity *bi)
54
-
{
55
-
return bi->metadata_size == bi->pi_tuple_size;
56
51
}
57
52
58
53
/**
···
79
84
/**
80
85
* bio_integrity_prep - Prepare bio for integrity I/O
81
86
* @bio: bio to prepare
87
+
* @action: preparation action needed (BI_ACT_*)
82
88
*
83
-
* Checks if the bio already has an integrity payload attached. If it does, the
84
-
* payload has been generated by another kernel subsystem, and we just pass it
85
-
* through.
86
-
* Otherwise allocates integrity payload and for writes the integrity metadata
87
-
* will be generated. For reads, the completion handler will verify the
88
-
* metadata.
89
+
* Allocate the integrity payload. For writes, generate the integrity metadata
90
+
* and for reads, setup the completion handler to verify the metadata.
91
+
*
92
+
* This is used for bios that do not have user integrity payloads attached.
89
93
*/
90
-
bool bio_integrity_prep(struct bio *bio)
94
+
void bio_integrity_prep(struct bio *bio, unsigned int action)
91
95
{
92
-
struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
93
96
struct bio_integrity_data *bid;
94
-
bool set_flags = true;
95
-
gfp_t gfp = GFP_NOIO;
96
-
97
-
if (!bi)
98
-
return true;
99
-
100
-
if (!bio_sectors(bio))
101
-
return true;
102
-
103
-
/* Already protected? */
104
-
if (bio_integrity(bio))
105
-
return true;
106
-
107
-
switch (bio_op(bio)) {
108
-
case REQ_OP_READ:
109
-
if (bi->flags & BLK_INTEGRITY_NOVERIFY) {
110
-
if (bi_offload_capable(bi))
111
-
return true;
112
-
set_flags = false;
113
-
}
114
-
break;
115
-
case REQ_OP_WRITE:
116
-
/*
117
-
* Zero the memory allocated to not leak uninitialized kernel
118
-
* memory to disk for non-integrity metadata where nothing else
119
-
* initializes the memory.
120
-
*/
121
-
if (bi->flags & BLK_INTEGRITY_NOGENERATE) {
122
-
if (bi_offload_capable(bi))
123
-
return true;
124
-
set_flags = false;
125
-
gfp |= __GFP_ZERO;
126
-
} else if (bi->metadata_size > bi->pi_tuple_size)
127
-
gfp |= __GFP_ZERO;
128
-
break;
129
-
default:
130
-
return true;
131
-
}
132
-
133
-
if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
134
-
return true;
135
97
136
98
bid = mempool_alloc(&bid_pool, GFP_NOIO);
137
99
bio_integrity_init(bio, &bid->bip, &bid->bvec, 1);
138
100
bid->bio = bio;
139
101
bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY;
140
-
bio_integrity_alloc_buf(bio, gfp & __GFP_ZERO);
141
-
142
-
bip_set_seed(&bid->bip, bio->bi_iter.bi_sector);
143
-
144
-
if (set_flags) {
145
-
if (bi->csum_type == BLK_INTEGRITY_CSUM_IP)
146
-
bid->bip.bip_flags |= BIP_IP_CHECKSUM;
147
-
if (bi->csum_type)
148
-
bid->bip.bip_flags |= BIP_CHECK_GUARD;
149
-
if (bi->flags & BLK_INTEGRITY_REF_TAG)
150
-
bid->bip.bip_flags |= BIP_CHECK_REFTAG;
151
-
}
102
+
bio_integrity_alloc_buf(bio, action & BI_ACT_ZERO);
103
+
if (action & BI_ACT_CHECK)
104
+
bio_integrity_setup_default(bio);
152
105
153
106
/* Auto-generate integrity metadata if this is a write */
154
107
if (bio_data_dir(bio) == WRITE && bip_should_check(&bid->bip))
155
-
blk_integrity_generate(bio);
108
+
bio_integrity_generate(bio);
156
109
else
157
110
bid->saved_bio_iter = bio->bi_iter;
158
-
return true;
159
111
}
160
112
EXPORT_SYMBOL(bio_integrity_prep);
161
113
+81
block/bio-integrity-fs.c
+81
block/bio-integrity-fs.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* Copyright (c) 2025 Christoph Hellwig.
4
+
*/
5
+
#include <linux/blk-integrity.h>
6
+
#include <linux/bio-integrity.h>
7
+
#include "blk.h"
8
+
9
+
struct fs_bio_integrity_buf {
10
+
struct bio_integrity_payload bip;
11
+
struct bio_vec bvec;
12
+
};
13
+
14
+
static struct kmem_cache *fs_bio_integrity_cache;
15
+
static mempool_t fs_bio_integrity_pool;
16
+
17
+
unsigned int fs_bio_integrity_alloc(struct bio *bio)
18
+
{
19
+
struct fs_bio_integrity_buf *iib;
20
+
unsigned int action;
21
+
22
+
action = bio_integrity_action(bio);
23
+
if (!action)
24
+
return 0;
25
+
26
+
iib = mempool_alloc(&fs_bio_integrity_pool, GFP_NOIO);
27
+
bio_integrity_init(bio, &iib->bip, &iib->bvec, 1);
28
+
29
+
bio_integrity_alloc_buf(bio, action & BI_ACT_ZERO);
30
+
if (action & BI_ACT_CHECK)
31
+
bio_integrity_setup_default(bio);
32
+
return action;
33
+
}
34
+
35
+
void fs_bio_integrity_free(struct bio *bio)
36
+
{
37
+
struct bio_integrity_payload *bip = bio_integrity(bio);
38
+
39
+
bio_integrity_free_buf(bip);
40
+
mempool_free(container_of(bip, struct fs_bio_integrity_buf, bip),
41
+
&fs_bio_integrity_pool);
42
+
43
+
bio->bi_integrity = NULL;
44
+
bio->bi_opf &= ~REQ_INTEGRITY;
45
+
}
46
+
47
+
void fs_bio_integrity_generate(struct bio *bio)
48
+
{
49
+
if (fs_bio_integrity_alloc(bio))
50
+
bio_integrity_generate(bio);
51
+
}
52
+
EXPORT_SYMBOL_GPL(fs_bio_integrity_generate);
53
+
54
+
int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size)
55
+
{
56
+
struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
57
+
struct bio_integrity_payload *bip = bio_integrity(bio);
58
+
59
+
/*
60
+
* Reinitialize bip->bip_iter.
61
+
*
62
+
* This is for use in the submitter after the driver is done with the
63
+
* bio. Requires the submitter to remember the sector and the size.
64
+
*/
65
+
memset(&bip->bip_iter, 0, sizeof(bip->bip_iter));
66
+
bip->bip_iter.bi_sector = sector;
67
+
bip->bip_iter.bi_size = bio_integrity_bytes(bi, size >> SECTOR_SHIFT);
68
+
return blk_status_to_errno(bio_integrity_verify(bio, &bip->bip_iter));
69
+
}
70
+
71
+
static int __init fs_bio_integrity_init(void)
72
+
{
73
+
fs_bio_integrity_cache = kmem_cache_create("fs_bio_integrity",
74
+
sizeof(struct fs_bio_integrity_buf), 0,
75
+
SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
76
+
if (mempool_init_slab_pool(&fs_bio_integrity_pool, BIO_POOL_SIZE,
77
+
fs_bio_integrity_cache))
78
+
panic("fs_bio_integrity: can't create pool\n");
79
+
return 0;
80
+
}
81
+
fs_initcall(fs_bio_integrity_init);
+64
block/bio-integrity.c
+64
block/bio-integrity.c
···
7
7
*/
8
8
9
9
#include <linux/blk-integrity.h>
10
+
#include <linux/t10-pi.h>
10
11
#include "blk.h"
11
12
12
13
struct bio_integrity_alloc {
···
16
15
};
17
16
18
17
static mempool_t integrity_buf_pool;
18
+
19
+
static bool bi_offload_capable(struct blk_integrity *bi)
20
+
{
21
+
return bi->metadata_size == bi->pi_tuple_size;
22
+
}
23
+
24
+
unsigned int __bio_integrity_action(struct bio *bio)
25
+
{
26
+
struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
27
+
28
+
if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
29
+
return 0;
30
+
31
+
switch (bio_op(bio)) {
32
+
case REQ_OP_READ:
33
+
if (bi->flags & BLK_INTEGRITY_NOVERIFY) {
34
+
if (bi_offload_capable(bi))
35
+
return 0;
36
+
return BI_ACT_BUFFER;
37
+
}
38
+
return BI_ACT_BUFFER | BI_ACT_CHECK;
39
+
case REQ_OP_WRITE:
40
+
/*
41
+
* Flush masquerading as write?
42
+
*/
43
+
if (!bio_sectors(bio))
44
+
return 0;
45
+
46
+
/*
47
+
* Zero the memory allocated to not leak uninitialized kernel
48
+
* memory to disk for non-integrity metadata where nothing else
49
+
* initializes the memory.
50
+
*/
51
+
if (bi->flags & BLK_INTEGRITY_NOGENERATE) {
52
+
if (bi_offload_capable(bi))
53
+
return 0;
54
+
return BI_ACT_BUFFER | BI_ACT_ZERO;
55
+
}
56
+
57
+
if (bi->metadata_size > bi->pi_tuple_size)
58
+
return BI_ACT_BUFFER | BI_ACT_CHECK | BI_ACT_ZERO;
59
+
return BI_ACT_BUFFER | BI_ACT_CHECK;
60
+
default:
61
+
return 0;
62
+
}
63
+
}
64
+
EXPORT_SYMBOL_GPL(__bio_integrity_action);
19
65
20
66
void bio_integrity_alloc_buf(struct bio *bio, bool zero_buffer)
21
67
{
···
99
51
mempool_free(bv->bv_page, &integrity_buf_pool);
100
52
else
101
53
kfree(bvec_virt(bv));
54
+
}
55
+
56
+
void bio_integrity_setup_default(struct bio *bio)
57
+
{
58
+
struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
59
+
struct bio_integrity_payload *bip = bio_integrity(bio);
60
+
61
+
bip_set_seed(bip, bio->bi_iter.bi_sector);
62
+
63
+
if (bi->csum_type) {
64
+
bip->bip_flags |= BIP_CHECK_GUARD;
65
+
if (bi->csum_type == BLK_INTEGRITY_CSUM_IP)
66
+
bip->bip_flags |= BIP_IP_CHECKSUM;
67
+
}
68
+
if (bi->flags & BLK_INTEGRITY_REF_TAG)
69
+
bip->bip_flags |= BIP_CHECK_REFTAG;
102
70
}
103
71
104
72
/**
+10
-7
block/bio.c
+10
-7
block/bio.c
···
1327
1327
}
1328
1328
}
1329
1329
1330
-
static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter)
1330
+
static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
1331
+
size_t maxlen)
1331
1332
{
1332
-
size_t total_len = iov_iter_count(iter);
1333
+
size_t total_len = min(maxlen, iov_iter_count(iter));
1333
1334
1334
1335
if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
1335
1336
return -EINVAL;
···
1368
1367
return 0;
1369
1368
}
1370
1369
1371
-
static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter)
1370
+
static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
1371
+
size_t maxlen)
1372
1372
{
1373
-
size_t len = min(iov_iter_count(iter), SZ_1M);
1373
+
size_t len = min3(iov_iter_count(iter), maxlen, SZ_1M);
1374
1374
struct folio *folio;
1375
1375
1376
1376
folio = folio_alloc_greedy(GFP_KERNEL, &len);
···
1410
1408
* bio_iov_iter_bounce - bounce buffer data from an iter into a bio
1411
1409
* @bio: bio to send
1412
1410
* @iter: iter to read from / write into
1411
+
* @maxlen: maximum size to bounce
1413
1412
*
1414
1413
* Helper for direct I/O implementations that need to bounce buffer because
1415
1414
* we need to checksum the data or perform other operations that require
···
1418
1415
* copies the data into it. Needs to be paired with bio_iov_iter_unbounce()
1419
1416
* called on completion.
1420
1417
*/
1421
-
int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter)
1418
+
int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen)
1422
1419
{
1423
1420
if (op_is_write(bio_op(bio)))
1424
-
return bio_iov_iter_bounce_write(bio, iter);
1425
-
return bio_iov_iter_bounce_read(bio, iter);
1421
+
return bio_iov_iter_bounce_write(bio, iter, maxlen);
1422
+
return bio_iov_iter_bounce_read(bio, iter, maxlen);
1426
1423
}
1427
1424
1428
1425
static void bvec_unpin(struct bio_vec *bv, bool mark_dirty)
+4
-2
block/blk-mq.c
+4
-2
block/blk-mq.c
···
3143
3143
struct request_queue *q = bdev_get_queue(bio->bi_bdev);
3144
3144
struct blk_plug *plug = current->plug;
3145
3145
const int is_sync = op_is_sync(bio->bi_opf);
3146
+
unsigned int integrity_action;
3146
3147
struct blk_mq_hw_ctx *hctx;
3147
3148
unsigned int nr_segs;
3148
3149
struct request *rq;
···
3196
3195
if (!bio)
3197
3196
goto queue_exit;
3198
3197
3199
-
if (!bio_integrity_prep(bio))
3200
-
goto queue_exit;
3198
+
integrity_action = bio_integrity_action(bio);
3199
+
if (integrity_action)
3200
+
bio_integrity_prep(bio, integrity_action);
3201
3201
3202
3202
blk_mq_bio_issue_init(q, bio);
3203
3203
if (blk_mq_attempt_bio_merge(q, bio, nr_segs))
-13
block/blk-settings.c
-13
block/blk-settings.c
···
123
123
return 0;
124
124
}
125
125
126
-
/*
127
-
* Maximum size of I/O that needs a block layer integrity buffer. Limited
128
-
* by the number of intervals for which we can fit the integrity buffer into
129
-
* the buffer size. Because the buffer is a single segment it is also limited
130
-
* by the maximum segment size.
131
-
*/
132
-
static inline unsigned int max_integrity_io_size(struct queue_limits *lim)
133
-
{
134
-
return min_t(unsigned int, lim->max_segment_size,
135
-
(BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size) <<
136
-
lim->integrity.interval_exp);
137
-
}
138
-
139
126
static int blk_validate_integrity_limits(struct queue_limits *lim)
140
127
{
141
128
struct blk_integrity *bi = &lim->integrity;
+4
-2
block/blk.h
+4
-2
block/blk.h
···
699
699
const struct blk_holder_ops *hops, struct file *bdev_file);
700
700
int bdev_permission(dev_t dev, blk_mode_t mode, void *holder);
701
701
702
-
void blk_integrity_generate(struct bio *bio);
703
-
void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter);
702
+
void bio_integrity_generate(struct bio *bio);
703
+
blk_status_t bio_integrity_verify(struct bio *bio,
704
+
struct bvec_iter *saved_iter);
705
+
704
706
void blk_integrity_prepare(struct request *rq);
705
707
void blk_integrity_complete(struct request *rq, unsigned int nr_bytes);
706
708
+6
-6
block/t10-pi.c
+6
-6
block/t10-pi.c
···
372
372
}
373
373
}
374
374
375
-
void blk_integrity_generate(struct bio *bio)
375
+
void bio_integrity_generate(struct bio *bio)
376
376
{
377
377
struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
378
378
struct bio_integrity_payload *bip = bio_integrity(bio);
···
404
404
}
405
405
}
406
406
407
-
void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter)
407
+
blk_status_t bio_integrity_verify(struct bio *bio, struct bvec_iter *saved_iter)
408
408
{
409
409
struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
410
410
struct bio_integrity_payload *bip = bio_integrity(bio);
···
439
439
}
440
440
kunmap_local(kaddr);
441
441
442
-
if (ret) {
443
-
bio->bi_status = ret;
444
-
return;
445
-
}
442
+
if (ret)
443
+
return ret;
446
444
}
445
+
446
+
return BLK_STS_OK;
447
447
}
448
448
449
449
void blk_integrity_prepare(struct request *rq)
+4
-2
drivers/nvdimm/btt.c
+4
-2
drivers/nvdimm/btt.c
···
1435
1435
{
1436
1436
struct bio_integrity_payload *bip = bio_integrity(bio);
1437
1437
struct btt *btt = bio->bi_bdev->bd_disk->private_data;
1438
+
unsigned int integrity_action;
1438
1439
struct bvec_iter iter;
1439
1440
unsigned long start;
1440
1441
struct bio_vec bvec;
1441
1442
int err = 0;
1442
1443
bool do_acct;
1443
1444
1444
-
if (!bio_integrity_prep(bio))
1445
-
return;
1445
+
integrity_action = bio_integrity_action(bio);
1446
+
if (integrity_action)
1447
+
bio_integrity_prep(bio, integrity_action);
1446
1448
1447
1449
do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue);
1448
1450
if (do_acct)
+3
-2
fs/fuse/file.c
+3
-2
fs/fuse/file.c
···
947
947
return ret;
948
948
}
949
949
950
-
static void fuse_iomap_read_submit(struct iomap_read_folio_ctx *ctx)
950
+
static void fuse_iomap_submit_read(const struct iomap_iter *iter,
951
+
struct iomap_read_folio_ctx *ctx)
951
952
{
952
953
struct fuse_fill_read_data *data = ctx->read_ctx;
953
954
···
959
958
960
959
static const struct iomap_read_ops fuse_iomap_read_ops = {
961
960
.read_folio_range = fuse_iomap_read_folio_range_async,
962
-
.submit_read = fuse_iomap_read_submit,
961
+
.submit_read = fuse_iomap_submit_read,
963
962
};
964
963
965
964
static int fuse_read_folio(struct file *file, struct folio *folio)
+88
-46
fs/iomap/bio.c
+88
-46
fs/iomap/bio.c
···
3
3
* Copyright (C) 2010 Red Hat, Inc.
4
4
* Copyright (C) 2016-2023 Christoph Hellwig.
5
5
*/
6
+
#include <linux/bio-integrity.h>
6
7
#include <linux/iomap.h>
7
8
#include <linux/pagemap.h>
8
9
#include "internal.h"
···
12
11
static DEFINE_SPINLOCK(failed_read_lock);
13
12
static struct bio_list failed_read_list = BIO_EMPTY_LIST;
14
13
15
-
static void __iomap_read_end_io(struct bio *bio)
14
+
static u32 __iomap_read_end_io(struct bio *bio, int error)
16
15
{
17
-
int error = blk_status_to_errno(bio->bi_status);
18
16
struct folio_iter fi;
17
+
u32 folio_count = 0;
19
18
20
-
bio_for_each_folio_all(fi, bio)
19
+
bio_for_each_folio_all(fi, bio) {
21
20
iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error);
21
+
folio_count++;
22
+
}
23
+
if (bio_integrity(bio))
24
+
fs_bio_integrity_free(bio);
22
25
bio_put(bio);
26
+
return folio_count;
23
27
}
24
28
25
29
static void
···
40
34
spin_unlock_irqrestore(&failed_read_lock, flags);
41
35
42
36
while ((bio = bio_list_pop(&tmp)) != NULL) {
43
-
__iomap_read_end_io(bio);
37
+
__iomap_read_end_io(bio, blk_status_to_errno(bio->bi_status));
44
38
cond_resched();
45
39
}
46
40
}
···
70
64
return;
71
65
}
72
66
73
-
__iomap_read_end_io(bio);
67
+
__iomap_read_end_io(bio, 0);
74
68
}
75
69
76
-
static void iomap_bio_submit_read(struct iomap_read_folio_ctx *ctx)
70
+
u32 iomap_finish_ioend_buffered_read(struct iomap_ioend *ioend)
71
+
{
72
+
return __iomap_read_end_io(&ioend->io_bio, ioend->io_error);
73
+
}
74
+
75
+
static void iomap_bio_submit_read(const struct iomap_iter *iter,
76
+
struct iomap_read_folio_ctx *ctx)
77
77
{
78
78
struct bio *bio = ctx->read_ctx;
79
79
80
-
if (bio)
81
-
submit_bio(bio);
80
+
if (iter->iomap.flags & IOMAP_F_INTEGRITY)
81
+
fs_bio_integrity_alloc(bio);
82
+
submit_bio(bio);
82
83
}
83
84
84
-
static int iomap_bio_read_folio_range(const struct iomap_iter *iter,
85
+
static struct bio_set *iomap_read_bio_set(struct iomap_read_folio_ctx *ctx)
86
+
{
87
+
if (ctx->ops && ctx->ops->bio_set)
88
+
return ctx->ops->bio_set;
89
+
return &fs_bio_set;
90
+
}
91
+
92
+
static void iomap_read_alloc_bio(const struct iomap_iter *iter,
93
+
struct iomap_read_folio_ctx *ctx, size_t plen)
94
+
{
95
+
const struct iomap *iomap = &iter->iomap;
96
+
unsigned int nr_vecs = DIV_ROUND_UP(iomap_length(iter), PAGE_SIZE);
97
+
struct bio_set *bio_set = iomap_read_bio_set(ctx);
98
+
struct folio *folio = ctx->cur_folio;
99
+
gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
100
+
gfp_t orig_gfp = gfp;
101
+
struct bio *bio;
102
+
103
+
/* Submit the existing range if there was one. */
104
+
if (ctx->read_ctx)
105
+
ctx->ops->submit_read(iter, ctx);
106
+
107
+
/* Same as readahead_gfp_mask: */
108
+
if (ctx->rac)
109
+
gfp |= __GFP_NORETRY | __GFP_NOWARN;
110
+
111
+
/*
112
+
* If the bio_alloc fails, try it again for a single page to avoid
113
+
* having to deal with partial page reads. This emulates what
114
+
* do_mpage_read_folio does.
115
+
*/
116
+
bio = bio_alloc_bioset(iomap->bdev, bio_max_segs(nr_vecs), REQ_OP_READ,
117
+
gfp, bio_set);
118
+
if (!bio)
119
+
bio = bio_alloc_bioset(iomap->bdev, 1, REQ_OP_READ, orig_gfp,
120
+
bio_set);
121
+
if (ctx->rac)
122
+
bio->bi_opf |= REQ_RAHEAD;
123
+
bio->bi_iter.bi_sector = iomap_sector(iomap, iter->pos);
124
+
bio->bi_end_io = iomap_read_end_io;
125
+
bio_add_folio_nofail(bio, folio, plen,
126
+
offset_in_folio(folio, iter->pos));
127
+
ctx->read_ctx = bio;
128
+
ctx->read_ctx_file_offset = iter->pos;
129
+
}
130
+
131
+
int iomap_bio_read_folio_range(const struct iomap_iter *iter,
85
132
struct iomap_read_folio_ctx *ctx, size_t plen)
86
133
{
87
134
struct folio *folio = ctx->cur_folio;
88
-
const struct iomap *iomap = &iter->iomap;
89
-
loff_t pos = iter->pos;
90
-
size_t poff = offset_in_folio(folio, pos);
91
-
loff_t length = iomap_length(iter);
92
-
sector_t sector;
93
135
struct bio *bio = ctx->read_ctx;
94
136
95
-
sector = iomap_sector(iomap, pos);
96
-
if (!bio || bio_end_sector(bio) != sector ||
97
-
!bio_add_folio(bio, folio, plen, poff)) {
98
-
gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
99
-
gfp_t orig_gfp = gfp;
100
-
unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE);
101
-
102
-
if (bio)
103
-
submit_bio(bio);
104
-
105
-
if (ctx->rac) /* same as readahead_gfp_mask */
106
-
gfp |= __GFP_NORETRY | __GFP_NOWARN;
107
-
bio = bio_alloc(iomap->bdev, bio_max_segs(nr_vecs), REQ_OP_READ,
108
-
gfp);
109
-
/*
110
-
* If the bio_alloc fails, try it again for a single page to
111
-
* avoid having to deal with partial page reads. This emulates
112
-
* what do_mpage_read_folio does.
113
-
*/
114
-
if (!bio)
115
-
bio = bio_alloc(iomap->bdev, 1, REQ_OP_READ, orig_gfp);
116
-
if (ctx->rac)
117
-
bio->bi_opf |= REQ_RAHEAD;
118
-
bio->bi_iter.bi_sector = sector;
119
-
bio->bi_end_io = iomap_read_end_io;
120
-
bio_add_folio_nofail(bio, folio, plen, poff);
121
-
ctx->read_ctx = bio;
122
-
}
137
+
if (!bio ||
138
+
bio_end_sector(bio) != iomap_sector(&iter->iomap, iter->pos) ||
139
+
bio->bi_iter.bi_size > iomap_max_bio_size(&iter->iomap) - plen ||
140
+
!bio_add_folio(bio, folio, plen, offset_in_folio(folio, iter->pos)))
141
+
iomap_read_alloc_bio(iter, ctx, plen);
123
142
return 0;
124
143
}
144
+
EXPORT_SYMBOL_GPL(iomap_bio_read_folio_range);
125
145
126
146
const struct iomap_read_ops iomap_bio_read_ops = {
127
-
.read_folio_range = iomap_bio_read_folio_range,
128
-
.submit_read = iomap_bio_submit_read,
147
+
.read_folio_range = iomap_bio_read_folio_range,
148
+
.submit_read = iomap_bio_submit_read,
129
149
};
130
150
EXPORT_SYMBOL_GPL(iomap_bio_read_ops);
131
151
···
159
127
struct folio *folio, loff_t pos, size_t len)
160
128
{
161
129
const struct iomap *srcmap = iomap_iter_srcmap(iter);
130
+
sector_t sector = iomap_sector(srcmap, pos);
162
131
struct bio_vec bvec;
163
132
struct bio bio;
133
+
int error;
164
134
165
135
bio_init(&bio, srcmap->bdev, &bvec, 1, REQ_OP_READ);
166
-
bio.bi_iter.bi_sector = iomap_sector(srcmap, pos);
136
+
bio.bi_iter.bi_sector = sector;
167
137
bio_add_folio_nofail(&bio, folio, len, offset_in_folio(folio, pos));
168
-
return submit_bio_wait(&bio);
138
+
if (srcmap->flags & IOMAP_F_INTEGRITY)
139
+
fs_bio_integrity_alloc(&bio);
140
+
error = submit_bio_wait(&bio);
141
+
if (srcmap->flags & IOMAP_F_INTEGRITY) {
142
+
if (!error)
143
+
error = fs_bio_integrity_verify(&bio, sector, len);
144
+
fs_bio_integrity_free(&bio);
145
+
}
146
+
return error;
169
147
}
+4
-4
fs/iomap/buffered-io.c
+4
-4
fs/iomap/buffered-io.c
···
601
601
iter.status = iomap_read_folio_iter(&iter, ctx,
602
602
&bytes_submitted);
603
603
604
-
if (ctx->ops->submit_read)
605
-
ctx->ops->submit_read(ctx);
604
+
if (ctx->read_ctx && ctx->ops->submit_read)
605
+
ctx->ops->submit_read(&iter, ctx);
606
606
607
607
if (ctx->cur_folio)
608
608
iomap_read_end(ctx->cur_folio, bytes_submitted);
···
668
668
iter.status = iomap_readahead_iter(&iter, ctx,
669
669
&cur_bytes_submitted);
670
670
671
-
if (ctx->ops->submit_read)
672
-
ctx->ops->submit_read(ctx);
671
+
if (ctx->read_ctx && ctx->ops->submit_read)
672
+
ctx->ops->submit_read(&iter, ctx);
673
673
674
674
if (ctx->cur_folio)
675
675
iomap_read_end(ctx->cur_folio, cur_bytes_submitted);
+14
-1
fs/iomap/direct-io.c
+14
-1
fs/iomap/direct-io.c
···
3
3
* Copyright (C) 2010 Red Hat, Inc.
4
4
* Copyright (c) 2016-2025 Christoph Hellwig.
5
5
*/
6
+
#include <linux/bio-integrity.h>
6
7
#include <linux/blk-crypto.h>
7
8
#include <linux/fscrypt.h>
8
9
#include <linux/pagemap.h>
···
241
240
{
242
241
struct iomap_dio *dio = bio->bi_private;
243
242
243
+
if (bio_integrity(bio))
244
+
fs_bio_integrity_free(bio);
245
+
244
246
if (dio->flags & IOMAP_DIO_BOUNCE) {
245
247
bio_iov_iter_unbounce(bio, !!dio->error,
246
248
dio->flags & IOMAP_DIO_USER_BACKED);
···
354
350
bio->bi_private = dio;
355
351
bio->bi_end_io = iomap_dio_bio_end_io;
356
352
353
+
357
354
if (dio->flags & IOMAP_DIO_BOUNCE)
358
-
ret = bio_iov_iter_bounce(bio, dio->submit.iter);
355
+
ret = bio_iov_iter_bounce(bio, dio->submit.iter,
356
+
iomap_max_bio_size(&iter->iomap));
359
357
else
360
358
ret = bio_iov_iter_get_pages(bio, dio->submit.iter,
361
359
alignment - 1);
···
372
366
if ((op & REQ_ATOMIC) && WARN_ON_ONCE(ret != iomap_length(iter))) {
373
367
ret = -EINVAL;
374
368
goto out_put_bio;
369
+
}
370
+
371
+
if (iter->iomap.flags & IOMAP_F_INTEGRITY) {
372
+
if (dio->flags & IOMAP_DIO_WRITE)
373
+
fs_bio_integrity_generate(bio);
374
+
else
375
+
fs_bio_integrity_alloc(bio);
375
376
}
376
377
377
378
if (dio->flags & IOMAP_DIO_WRITE)
+14
fs/iomap/internal.h
+14
fs/iomap/internal.h
···
4
4
5
5
#define IOEND_BATCH_SIZE 4096
6
6
7
+
/*
8
+
* Normally we can build bios as big as the data structure supports.
9
+
*
10
+
* But for integrity protected I/O we need to respect the maximum size of the
11
+
* single contiguous allocation for the integrity buffer.
12
+
*/
13
+
static inline size_t iomap_max_bio_size(const struct iomap *iomap)
14
+
{
15
+
if (iomap->flags & IOMAP_F_INTEGRITY)
16
+
return max_integrity_io_size(bdev_limits(iomap->bdev));
17
+
return BIO_MAX_SIZE;
18
+
}
19
+
20
+
u32 iomap_finish_ioend_buffered_read(struct iomap_ioend *ioend);
7
21
u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend);
8
22
9
23
#ifdef CONFIG_BLOCK
+24
-6
fs/iomap/ioend.c
+24
-6
fs/iomap/ioend.c
···
2
2
/*
3
3
* Copyright (c) 2016-2025 Christoph Hellwig.
4
4
*/
5
+
#include <linux/bio-integrity.h>
5
6
#include <linux/iomap.h>
6
7
#include <linux/list_sort.h>
7
8
#include <linux/pagemap.h>
···
38
37
* state, release holds on bios, and finally free up memory. Do not use the
39
38
* ioend after this.
40
39
*/
41
-
static u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend)
40
+
static u32 iomap_finish_ioend_buffered_write(struct iomap_ioend *ioend)
42
41
{
43
42
struct inode *inode = ioend->io_inode;
44
43
struct bio *bio = &ioend->io_bio;
···
66
65
folio_count++;
67
66
}
68
67
68
+
if (bio_integrity(bio))
69
+
fs_bio_integrity_free(bio);
69
70
bio_put(bio); /* frees the ioend */
70
71
return folio_count;
71
72
}
···
90
87
while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
91
88
io_list))) {
92
89
list_del_init(&ioend->io_list);
93
-
iomap_finish_ioend_buffered(ioend);
90
+
iomap_finish_ioend_buffered_write(ioend);
94
91
cond_resched();
95
92
}
96
93
}
···
123
120
return;
124
121
}
125
122
126
-
iomap_finish_ioend_buffered(ioend);
123
+
iomap_finish_ioend_buffered_write(ioend);
127
124
}
128
125
129
126
/*
···
147
144
return error;
148
145
}
149
146
147
+
if (wpc->iomap.flags & IOMAP_F_INTEGRITY)
148
+
fs_bio_integrity_generate(&ioend->io_bio);
150
149
submit_bio(&ioend->io_bio);
151
150
return 0;
152
151
}
···
170
165
}
171
166
172
167
static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos,
173
-
u16 ioend_flags)
168
+
unsigned int map_len, u16 ioend_flags)
174
169
{
175
170
struct iomap_ioend *ioend = wpc->wb_ctx;
176
171
172
+
if (ioend->io_bio.bi_iter.bi_size >
173
+
iomap_max_bio_size(&wpc->iomap) - map_len)
174
+
return false;
177
175
if (ioend_flags & IOMAP_IOEND_BOUNDARY)
178
176
return false;
179
177
if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
···
242
234
if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY))
243
235
ioend_flags |= IOMAP_IOEND_BOUNDARY;
244
236
245
-
if (!ioend || !iomap_can_add_to_ioend(wpc, pos, ioend_flags)) {
237
+
if (!ioend || !iomap_can_add_to_ioend(wpc, pos, map_len, ioend_flags)) {
246
238
new_ioend:
247
239
if (ioend) {
248
240
error = wpc->ops->writeback_submit(wpc, 0);
···
319
311
320
312
if (!atomic_dec_and_test(&ioend->io_remaining))
321
313
return 0;
314
+
315
+
if (!ioend->io_error &&
316
+
bio_integrity(&ioend->io_bio) &&
317
+
bio_op(&ioend->io_bio) == REQ_OP_READ) {
318
+
ioend->io_error = fs_bio_integrity_verify(&ioend->io_bio,
319
+
ioend->io_sector, ioend->io_size);
320
+
}
321
+
322
322
if (ioend->io_flags & IOMAP_IOEND_DIRECT)
323
323
return iomap_finish_ioend_direct(ioend);
324
-
return iomap_finish_ioend_buffered(ioend);
324
+
if (bio_op(&ioend->io_bio) == REQ_OP_READ)
325
+
return iomap_finish_ioend_buffered_read(ioend);
326
+
return iomap_finish_ioend_buffered_write(ioend);
325
327
}
326
328
327
329
/*
+6
-51
fs/ntfs3/inode.c
+6
-51
fs/ntfs3/inode.c
···
605
605
bio_put(bio);
606
606
}
607
607
608
-
/*
609
-
* Copied from iomap/bio.c.
610
-
*/
611
-
static int ntfs_iomap_bio_read_folio_range(const struct iomap_iter *iter,
612
-
struct iomap_read_folio_ctx *ctx,
613
-
size_t plen)
614
-
{
615
-
struct folio *folio = ctx->cur_folio;
616
-
const struct iomap *iomap = &iter->iomap;
617
-
loff_t pos = iter->pos;
618
-
size_t poff = offset_in_folio(folio, pos);
619
-
loff_t length = iomap_length(iter);
620
-
sector_t sector;
621
-
struct bio *bio = ctx->read_ctx;
622
-
623
-
sector = iomap_sector(iomap, pos);
624
-
if (!bio || bio_end_sector(bio) != sector ||
625
-
!bio_add_folio(bio, folio, plen, poff)) {
626
-
gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
627
-
gfp_t orig_gfp = gfp;
628
-
unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE);
629
-
630
-
if (bio)
631
-
submit_bio(bio);
632
-
633
-
if (ctx->rac) /* same as readahead_gfp_mask */
634
-
gfp |= __GFP_NORETRY | __GFP_NOWARN;
635
-
bio = bio_alloc(iomap->bdev, bio_max_segs(nr_vecs), REQ_OP_READ,
636
-
gfp);
637
-
/*
638
-
* If the bio_alloc fails, try it again for a single page to
639
-
* avoid having to deal with partial page reads. This emulates
640
-
* what do_mpage_read_folio does.
641
-
*/
642
-
if (!bio)
643
-
bio = bio_alloc(iomap->bdev, 1, REQ_OP_READ, orig_gfp);
644
-
if (ctx->rac)
645
-
bio->bi_opf |= REQ_RAHEAD;
646
-
bio->bi_iter.bi_sector = sector;
647
-
bio->bi_end_io = ntfs_iomap_read_end_io;
648
-
bio_add_folio_nofail(bio, folio, plen, poff);
649
-
ctx->read_ctx = bio;
650
-
}
651
-
return 0;
652
-
}
653
-
654
-
static void ntfs_iomap_bio_submit_read(struct iomap_read_folio_ctx *ctx)
608
+
static void ntfs_iomap_bio_submit_read(const struct iomap_iter *iter,
609
+
struct iomap_read_folio_ctx *ctx)
655
610
{
656
611
struct bio *bio = ctx->read_ctx;
657
612
658
-
if (bio)
659
-
submit_bio(bio);
613
+
bio->bi_end_io = ntfs_iomap_read_end_io;
614
+
submit_bio(bio);
660
615
}
661
616
662
617
static const struct iomap_read_ops ntfs_iomap_bio_read_ops = {
663
-
.read_folio_range = ntfs_iomap_bio_read_folio_range,
664
-
.submit_read = ntfs_iomap_bio_submit_read,
618
+
.read_folio_range = iomap_bio_read_folio_range,
619
+
.submit_read = ntfs_iomap_bio_submit_read,
665
620
};
666
621
667
622
static int ntfs_read_folio(struct file *file, struct folio *folio)
+43
-4
fs/xfs/xfs_aops.c
+43
-4
fs/xfs/xfs_aops.c
···
22
22
#include "xfs_icache.h"
23
23
#include "xfs_zone_alloc.h"
24
24
#include "xfs_rtgroup.h"
25
+
#include <linux/bio-integrity.h>
25
26
26
27
struct xfs_writepage_ctx {
27
28
struct iomap_writepage_ctx ctx;
···
662
661
bio_endio(&ioend->io_bio);
663
662
return error;
664
663
}
664
+
if (wpc->iomap.flags & IOMAP_F_INTEGRITY)
665
+
fs_bio_integrity_generate(&ioend->io_bio);
665
666
xfs_zone_alloc_and_submit(ioend, &XFS_ZWPC(wpc)->open_zone);
666
667
return 0;
667
668
}
···
744
741
return iomap_bmap(mapping, block, &xfs_read_iomap_ops);
745
742
}
746
743
744
+
static void
745
+
xfs_bio_submit_read(
746
+
const struct iomap_iter *iter,
747
+
struct iomap_read_folio_ctx *ctx)
748
+
{
749
+
struct bio *bio = ctx->read_ctx;
750
+
751
+
/* defer read completions to the ioend workqueue */
752
+
iomap_init_ioend(iter->inode, bio, ctx->read_ctx_file_offset, 0);
753
+
bio->bi_end_io = xfs_end_bio;
754
+
submit_bio(bio);
755
+
}
756
+
757
+
static const struct iomap_read_ops xfs_iomap_read_ops = {
758
+
.read_folio_range = iomap_bio_read_folio_range,
759
+
.submit_read = xfs_bio_submit_read,
760
+
.bio_set = &iomap_ioend_bioset,
761
+
};
762
+
763
+
static inline const struct iomap_read_ops *
764
+
xfs_get_iomap_read_ops(
765
+
const struct address_space *mapping)
766
+
{
767
+
struct xfs_inode *ip = XFS_I(mapping->host);
768
+
769
+
if (bdev_has_integrity_csum(xfs_inode_buftarg(ip)->bt_bdev))
770
+
return &xfs_iomap_read_ops;
771
+
return &iomap_bio_read_ops;
772
+
}
773
+
747
774
STATIC int
748
775
xfs_vm_read_folio(
749
-
struct file *unused,
750
-
struct folio *folio)
776
+
struct file *file,
777
+
struct folio *folio)
751
778
{
752
-
iomap_bio_read_folio(folio, &xfs_read_iomap_ops);
779
+
struct iomap_read_folio_ctx ctx = { .cur_folio = folio };
780
+
781
+
ctx.ops = xfs_get_iomap_read_ops(folio->mapping);
782
+
iomap_read_folio(&xfs_read_iomap_ops, &ctx, NULL);
753
783
return 0;
754
784
}
755
785
···
790
754
xfs_vm_readahead(
791
755
struct readahead_control *rac)
792
756
{
793
-
iomap_bio_readahead(rac, &xfs_read_iomap_ops);
757
+
struct iomap_read_folio_ctx ctx = { .rac = rac };
758
+
759
+
ctx.ops = xfs_get_iomap_read_ops(rac->mapping),
760
+
iomap_readahead(&xfs_read_iomap_ops, &ctx, NULL);
794
761
}
795
762
796
763
static int
+7
-4
fs/xfs/xfs_iomap.c
+7
-4
fs/xfs/xfs_iomap.c
···
143
143
}
144
144
iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff);
145
145
iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount);
146
-
if (mapping_flags & IOMAP_DAX)
147
-
iomap->dax_dev = target->bt_daxdev;
148
-
else
149
-
iomap->bdev = target->bt_bdev;
150
146
iomap->flags = iomap_flags;
147
+
if (mapping_flags & IOMAP_DAX) {
148
+
iomap->dax_dev = target->bt_daxdev;
149
+
} else {
150
+
iomap->bdev = target->bt_bdev;
151
+
if (bdev_has_integrity_csum(iomap->bdev))
152
+
iomap->flags |= IOMAP_F_INTEGRITY;
153
+
}
151
154
152
155
/*
153
156
* If the inode is dirty for datasync purposes, let iomap know so it
+9
-3
include/linux/bio-integrity.h
+9
-3
include/linux/bio-integrity.h
···
78
78
int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter);
79
79
int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta);
80
80
void bio_integrity_unmap_user(struct bio *bio);
81
-
bool bio_integrity_prep(struct bio *bio);
81
+
void bio_integrity_prep(struct bio *bio, unsigned int action);
82
82
void bio_integrity_advance(struct bio *bio, unsigned int bytes_done);
83
83
void bio_integrity_trim(struct bio *bio);
84
84
int bio_integrity_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp_mask);
···
104
104
{
105
105
}
106
106
107
-
static inline bool bio_integrity_prep(struct bio *bio)
107
+
static inline void bio_integrity_prep(struct bio *bio, unsigned int action)
108
108
{
109
-
return true;
110
109
}
111
110
112
111
static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
···
143
144
144
145
void bio_integrity_alloc_buf(struct bio *bio, bool zero_buffer);
145
146
void bio_integrity_free_buf(struct bio_integrity_payload *bip);
147
+
void bio_integrity_setup_default(struct bio *bio);
148
+
149
+
unsigned int fs_bio_integrity_alloc(struct bio *bio);
150
+
void fs_bio_integrity_free(struct bio *bio);
151
+
void fs_bio_integrity_generate(struct bio *bio);
152
+
int fs_bio_integrity_verify(struct bio *bio, sector_t sector,
153
+
unsigned int size);
146
154
147
155
#endif /* _LINUX_BIO_INTEGRITY_H */
+1
-1
include/linux/bio.h
+1
-1
include/linux/bio.h
···
474
474
extern void bio_set_pages_dirty(struct bio *bio);
475
475
extern void bio_check_pages_dirty(struct bio *bio);
476
476
477
-
int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter);
477
+
int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen);
478
478
void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty);
479
479
480
480
extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
+23
-5
include/linux/blk-integrity.h
+23
-5
include/linux/blk-integrity.h
···
8
8
9
9
struct request;
10
10
11
-
/*
12
-
* Maximum contiguous integrity buffer allocation.
13
-
*/
14
-
#define BLK_INTEGRITY_MAX_SIZE SZ_2M
15
-
16
11
enum blk_integrity_flags {
17
12
BLK_INTEGRITY_NOVERIFY = 1 << 0,
18
13
BLK_INTEGRITY_NOGENERATE = 1 << 1,
···
174
179
return (struct bio_vec){ };
175
180
}
176
181
#endif /* CONFIG_BLK_DEV_INTEGRITY */
182
+
183
+
enum bio_integrity_action {
184
+
BI_ACT_BUFFER = (1u << 0), /* allocate buffer */
185
+
BI_ACT_CHECK = (1u << 1), /* generate / verify PI */
186
+
BI_ACT_ZERO = (1u << 2), /* zero buffer */
187
+
};
188
+
189
+
/**
190
+
* bio_integrity_action - return the integrity action needed for a bio
191
+
* @bio: bio to operate on
192
+
*
193
+
* Returns the mask of integrity actions (BI_ACT_*) that need to be performed
194
+
* for @bio.
195
+
*/
196
+
unsigned int __bio_integrity_action(struct bio *bio);
197
+
static inline unsigned int bio_integrity_action(struct bio *bio)
198
+
{
199
+
if (!blk_get_integrity(bio->bi_bdev->bd_disk))
200
+
return 0;
201
+
if (bio_integrity(bio))
202
+
return 0;
203
+
return __bio_integrity_action(bio);
204
+
}
177
205
178
206
#endif /* _LINUX_BLK_INTEGRITY_H */
+28
-6
include/linux/blkdev.h
+28
-6
include/linux/blkdev.h
···
1477
1477
return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS;
1478
1478
}
1479
1479
1480
+
static inline bool bdev_has_integrity_csum(struct block_device *bdev)
1481
+
{
1482
+
struct queue_limits *lim = bdev_limits(bdev);
1483
+
1484
+
return IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1485
+
lim->integrity.csum_type != BLK_INTEGRITY_CSUM_NONE;
1486
+
}
1487
+
1480
1488
static inline bool bdev_stable_writes(struct block_device *bdev)
1481
1489
{
1482
-
struct request_queue *q = bdev_get_queue(bdev);
1483
-
1484
-
if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1485
-
q->limits.integrity.csum_type != BLK_INTEGRITY_CSUM_NONE)
1486
-
return true;
1487
-
return q->limits.features & BLK_FEAT_STABLE_WRITES;
1490
+
return bdev_has_integrity_csum(bdev) ||
1491
+
(bdev_limits(bdev)->features & BLK_FEAT_STABLE_WRITES);
1488
1492
}
1489
1493
1490
1494
static inline bool blk_queue_write_cache(struct request_queue *q)
···
1879
1875
unsigned *segs, unsigned max_bytes)
1880
1876
{
1881
1877
return bio_split_io_at(bio, lim, segs, max_bytes, lim->dma_alignment);
1878
+
}
1879
+
1880
+
/*
1881
+
* Maximum contiguous integrity buffer allocation.
1882
+
*/
1883
+
#define BLK_INTEGRITY_MAX_SIZE SZ_2M
1884
+
1885
+
/*
1886
+
* Maximum size of I/O that needs a block layer integrity buffer. Limited
1887
+
* by the number of intervals for which we can fit the integrity buffer into
1888
+
* the buffer size. Because the buffer is a single segment it is also limited
1889
+
* by the maximum segment size.
1890
+
*/
1891
+
static inline unsigned int max_integrity_io_size(struct queue_limits *lim)
1892
+
{
1893
+
return min_t(unsigned int, lim->max_segment_size,
1894
+
(BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size) <<
1895
+
lim->integrity.interval_exp);
1882
1896
}
1883
1897
1884
1898
#define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { }
+19
-1
include/linux/iomap.h
+19
-1
include/linux/iomap.h
···
65
65
*
66
66
* IOMAP_F_ATOMIC_BIO indicates that (write) I/O will be issued as an atomic
67
67
* bio, i.e. set REQ_ATOMIC.
68
+
*
69
+
* IOMAP_F_INTEGRITY indicates that the filesystems handles integrity metadata.
68
70
*/
69
71
#define IOMAP_F_NEW (1U << 0)
70
72
#define IOMAP_F_DIRTY (1U << 1)
···
81
79
#define IOMAP_F_BOUNDARY (1U << 6)
82
80
#define IOMAP_F_ANON_WRITE (1U << 7)
83
81
#define IOMAP_F_ATOMIC_BIO (1U << 8)
82
+
#ifdef CONFIG_BLK_DEV_INTEGRITY
83
+
#define IOMAP_F_INTEGRITY (1U << 9)
84
+
#else
85
+
#define IOMAP_F_INTEGRITY 0
86
+
#endif /* CONFIG_BLK_DEV_INTEGRITY */
84
87
85
88
/*
86
89
* Flag reserved for file system specific usage
···
500
493
struct folio *cur_folio;
501
494
struct readahead_control *rac;
502
495
void *read_ctx;
496
+
loff_t read_ctx_file_offset;
503
497
};
504
498
505
499
struct iomap_read_ops {
···
520
512
*
521
513
* This is optional.
522
514
*/
523
-
void (*submit_read)(struct iomap_read_folio_ctx *ctx);
515
+
void (*submit_read)(const struct iomap_iter *iter,
516
+
struct iomap_read_folio_ctx *ctx);
517
+
518
+
/*
519
+
* Optional, allows filesystem to specify own bio_set, so new bio's
520
+
* can be allocated from the provided bio_set.
521
+
*/
522
+
struct bio_set *bio_set;
524
523
};
525
524
526
525
/*
···
613
598
extern struct bio_set iomap_ioend_bioset;
614
599
615
600
#ifdef CONFIG_BLOCK
601
+
int iomap_bio_read_folio_range(const struct iomap_iter *iter,
602
+
struct iomap_read_folio_ctx *ctx, size_t plen);
603
+
616
604
extern const struct iomap_read_ops iomap_bio_read_ops;
617
605
618
606
static inline void iomap_bio_read_folio(struct folio *folio,