Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'fsverity-for-linus' of git://git.kernel.org/pub/scm/fs/fsverity/linux
Pull fsverity updates from Eric Biggers:
"fsverity cleanups, speedup, and memory usage optimization from
Christoph Hellwig:
- Move some logic into common code
- Fix btrfs to reject truncates of fsverity files
- Improve the readahead implementation
- Store each inode's fsverity_info in a hash table instead of using a
pointer in the filesystem-specific part of the inode.
This optimizes for memory usage in the usual case where most files
don't have fsverity enabled.
- Look up the fsverity_info fewer times during verification, to
amortize the hash table overhead"
* tag 'fsverity-for-linus' of git://git.kernel.org/pub/scm/fs/fsverity/linux:
fsverity: remove inode from fsverity_verification_ctx
fsverity: use a hashtable to find the fsverity_info
btrfs: consolidate fsverity_info lookup
f2fs: consolidate fsverity_info lookup
ext4: consolidate fsverity_info lookup
fs: consolidate fsverity_info lookup in buffer.c
fsverity: push out fsverity_info lookup
fsverity: deconstify the inode pointer in struct fsverity_info
fsverity: kick off hash readahead at data I/O submission time
ext4: move ->read_folio and ->readahead to readpage.c
readahead: push invalidate_lock out of page_cache_ra_unbounded
fsverity: don't issue readahead for non-ENOENT errors from __filemap_get_folio
fsverity: start consolidating pagecache code
fsverity: pass struct file to ->write_merkle_tree_block
f2fs: don't build the fsverity work handler for !CONFIG_FS_VERITY
ext4: don't build the fsverity work handler for !CONFIG_FS_VERITY
fs,fsverity: clear out fsverity_info from common code
fs,fsverity: reject size changes on fsverity files in setattr_prepare
+519
-437
+11
-1
fs/attr.c
+11
-1
fs/attr.c
···
169
169
* ATTR_FORCE.
170
170
*/
171
171
if (ia_valid & ATTR_SIZE) {
172
-
int error = inode_newsize_ok(inode, attr->ia_size);
172
+
int error;
173
+
174
+
/*
175
+
* Verity files are immutable, so deny truncates. This isn't
176
+
* covered by the open-time check because sys_truncate() takes a
177
+
* path, not an open file.
178
+
*/
179
+
if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
180
+
return -EPERM;
181
+
182
+
error = inode_newsize_ok(inode, attr->ia_size);
173
183
if (error)
174
184
return error;
175
185
}
-4
fs/btrfs/btrfs_inode.h
-4
fs/btrfs/btrfs_inode.h
+33
-20
fs/btrfs/extent_io.c
+33
-20
fs/btrfs/extent_io.c
···
475
475
end, page_ops);
476
476
}
477
477
478
-
static bool btrfs_verify_folio(struct folio *folio, u64 start, u32 len)
478
+
static bool btrfs_verify_folio(struct fsverity_info *vi, struct folio *folio,
479
+
u64 start, u32 len)
479
480
{
480
481
struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
481
482
482
-
if (!fsverity_active(folio->mapping->host) ||
483
-
btrfs_folio_test_uptodate(fs_info, folio, start, len) ||
484
-
start >= i_size_read(folio->mapping->host))
483
+
if (!vi || btrfs_folio_test_uptodate(fs_info, folio, start, len))
485
484
return true;
486
-
return fsverity_verify_folio(folio);
485
+
return fsverity_verify_folio(vi, folio);
487
486
}
488
487
489
-
static void end_folio_read(struct folio *folio, bool uptodate, u64 start, u32 len)
488
+
static void end_folio_read(struct fsverity_info *vi, struct folio *folio,
489
+
bool uptodate, u64 start, u32 len)
490
490
{
491
491
struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
492
492
493
493
ASSERT(folio_pos(folio) <= start &&
494
494
start + len <= folio_next_pos(folio));
495
495
496
-
if (uptodate && btrfs_verify_folio(folio, start, len))
496
+
if (uptodate && btrfs_verify_folio(vi, folio, start, len))
497
497
btrfs_folio_set_uptodate(fs_info, folio, start, len);
498
498
else
499
499
btrfs_folio_clear_uptodate(fs_info, folio, start, len);
···
573
573
static void end_bbio_data_read(struct btrfs_bio *bbio)
574
574
{
575
575
struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;
576
+
struct inode *inode = &bbio->inode->vfs_inode;
576
577
struct bio *bio = &bbio->bio;
578
+
struct fsverity_info *vi = NULL;
577
579
struct folio_iter fi;
578
580
579
581
ASSERT(!bio_flagged(bio, BIO_CLONED));
582
+
583
+
if (bbio->file_offset < i_size_read(inode))
584
+
vi = fsverity_get_info(inode);
585
+
580
586
bio_for_each_folio_all(fi, &bbio->bio) {
581
587
bool uptodate = !bio->bi_status;
582
588
struct folio *folio = fi.folio;
583
-
struct inode *inode = folio->mapping->host;
584
589
u64 start = folio_pos(folio) + fi.offset;
585
590
586
591
btrfs_debug(fs_info,
···
620
615
}
621
616
622
617
/* Update page status and unlock. */
623
-
end_folio_read(folio, uptodate, start, fi.length);
618
+
end_folio_read(vi, folio, uptodate, start, fi.length);
624
619
}
625
620
bio_put(bio);
626
621
}
···
995
990
* return 0 on success, otherwise return error
996
991
*/
997
992
static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
998
-
struct btrfs_bio_ctrl *bio_ctrl)
993
+
struct btrfs_bio_ctrl *bio_ctrl,
994
+
struct fsverity_info *vi)
999
995
{
1000
996
struct inode *inode = folio->mapping->host;
1001
997
struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
···
1040
1034
ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
1041
1035
if (cur >= last_byte) {
1042
1036
folio_zero_range(folio, pg_offset, end - cur + 1);
1043
-
end_folio_read(folio, true, cur, end - cur + 1);
1037
+
end_folio_read(vi, folio, true, cur, end - cur + 1);
1044
1038
break;
1045
1039
}
1046
1040
if (btrfs_folio_test_uptodate(fs_info, folio, cur, blocksize)) {
1047
-
end_folio_read(folio, true, cur, blocksize);
1041
+
end_folio_read(vi, folio, true, cur, blocksize);
1048
1042
continue;
1049
1043
}
1050
1044
/*
···
1056
1050
*/
1057
1051
em = get_extent_map(BTRFS_I(inode), folio, cur, locked_end - cur + 1, em_cached);
1058
1052
if (IS_ERR(em)) {
1059
-
end_folio_read(folio, false, cur, end + 1 - cur);
1053
+
end_folio_read(vi, folio, false, cur, end + 1 - cur);
1060
1054
return PTR_ERR(em);
1061
1055
}
1062
1056
extent_offset = cur - em->start;
···
1133
1127
/* we've found a hole, just zero and go on */
1134
1128
if (block_start == EXTENT_MAP_HOLE) {
1135
1129
folio_zero_range(folio, pg_offset, blocksize);
1136
-
end_folio_read(folio, true, cur, blocksize);
1130
+
end_folio_read(vi, folio, true, cur, blocksize);
1137
1131
continue;
1138
1132
}
1139
1133
/* the get_extent function already copied into the folio */
1140
1134
if (block_start == EXTENT_MAP_INLINE) {
1141
-
end_folio_read(folio, true, cur, blocksize);
1135
+
end_folio_read(vi, folio, true, cur, blocksize);
1142
1136
continue;
1143
1137
}
1144
1138
···
1335
1329
1336
1330
int btrfs_read_folio(struct file *file, struct folio *folio)
1337
1331
{
1338
-
struct btrfs_inode *inode = folio_to_inode(folio);
1332
+
struct inode *vfs_inode = folio->mapping->host;
1333
+
struct btrfs_inode *inode = BTRFS_I(vfs_inode);
1339
1334
const u64 start = folio_pos(folio);
1340
1335
const u64 end = start + folio_size(folio) - 1;
1341
1336
struct extent_state *cached_state = NULL;
···
1345
1338
.last_em_start = U64_MAX,
1346
1339
};
1347
1340
struct extent_map *em_cached = NULL;
1341
+
struct fsverity_info *vi = NULL;
1348
1342
int ret;
1349
1343
1350
1344
lock_extents_for_read(inode, start, end, &cached_state);
1351
-
ret = btrfs_do_readpage(folio, &em_cached, &bio_ctrl);
1345
+
if (folio_pos(folio) < i_size_read(vfs_inode))
1346
+
vi = fsverity_get_info(vfs_inode);
1347
+
ret = btrfs_do_readpage(folio, &em_cached, &bio_ctrl, vi);
1352
1348
btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state);
1353
1349
1354
1350
btrfs_free_extent_map(em_cached);
···
2724
2714
.last_em_start = U64_MAX,
2725
2715
};
2726
2716
struct folio *folio;
2727
-
struct btrfs_inode *inode = BTRFS_I(rac->mapping->host);
2717
+
struct inode *vfs_inode = rac->mapping->host;
2718
+
struct btrfs_inode *inode = BTRFS_I(vfs_inode);
2728
2719
const u64 start = readahead_pos(rac);
2729
2720
const u64 end = start + readahead_length(rac) - 1;
2730
2721
struct extent_state *cached_state = NULL;
2731
2722
struct extent_map *em_cached = NULL;
2723
+
struct fsverity_info *vi = NULL;
2732
2724
2733
2725
lock_extents_for_read(inode, start, end, &cached_state);
2734
-
2726
+
if (start < i_size_read(vfs_inode))
2727
+
vi = fsverity_get_info(vfs_inode);
2735
2728
while ((folio = readahead_folio(rac)) != NULL)
2736
-
btrfs_do_readpage(folio, &em_cached, &bio_ctrl);
2729
+
btrfs_do_readpage(folio, &em_cached, &bio_ctrl, vi);
2737
2730
2738
2731
btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state);
2739
2732
+3
-10
fs/btrfs/inode.c
+3
-10
fs/btrfs/inode.c
···
34
34
#include <linux/sched/mm.h>
35
35
#include <linux/iomap.h>
36
36
#include <linux/unaligned.h>
37
-
#include <linux/fsverity.h>
38
37
#include "misc.h"
39
38
#include "ctree.h"
40
39
#include "disk-io.h"
···
5615
5616
5616
5617
trace_btrfs_inode_evict(inode);
5617
5618
5618
-
if (!root) {
5619
-
fsverity_cleanup_inode(inode);
5620
-
clear_inode(inode);
5621
-
return;
5622
-
}
5619
+
if (!root)
5620
+
goto clear_inode;
5623
5621
5624
5622
fs_info = inode_to_fs_info(inode);
5625
5623
evict_inode_truncate_pages(inode);
···
5716
5720
* to retry these periodically in the future.
5717
5721
*/
5718
5722
btrfs_remove_delayed_node(BTRFS_I(inode));
5719
-
fsverity_cleanup_inode(inode);
5723
+
clear_inode:
5720
5724
clear_inode(inode);
5721
5725
}
5722
5726
···
8147
8151
struct btrfs_inode *ei = foo;
8148
8152
8149
8153
inode_init_once(&ei->vfs_inode);
8150
-
#ifdef CONFIG_FS_VERITY
8151
-
ei->i_verity_info = NULL;
8152
-
#endif
8153
8154
}
8154
8155
8155
8156
void __cold btrfs_destroy_cachep(void)
+4
-7
fs/btrfs/verity.c
+4
-7
fs/btrfs/verity.c
···
694
694
*
695
695
* @inode: inode to read a merkle tree page for
696
696
* @index: page index relative to the start of the merkle tree
697
-
* @num_ra_pages: number of pages to readahead. Optional, we ignore it
698
697
*
699
698
* The Merkle tree is stored in the filesystem btree, but its pages are cached
700
699
* with a logical position past EOF in the inode's mapping.
···
701
702
* Returns the page we read, or an ERR_PTR on error.
702
703
*/
703
704
static struct page *btrfs_read_merkle_tree_page(struct inode *inode,
704
-
pgoff_t index,
705
-
unsigned long num_ra_pages)
705
+
pgoff_t index)
706
706
{
707
707
struct folio *folio;
708
708
u64 off = (u64)index << PAGE_SHIFT;
···
769
771
/*
770
772
* fsverity op that writes a Merkle tree block into the btree.
771
773
*
772
-
* @inode: inode to write a Merkle tree block for
774
+
* @file: file to write a Merkle tree block for
773
775
* @buf: Merkle tree block to write
774
776
* @pos: the position of the block in the Merkle tree (in bytes)
775
777
* @size: the Merkle tree block size (in bytes)
776
778
*
777
779
* Returns 0 on success or negative error code on failure
778
780
*/
779
-
static int btrfs_write_merkle_tree_block(struct inode *inode, const void *buf,
781
+
static int btrfs_write_merkle_tree_block(struct file *file, const void *buf,
780
782
u64 pos, unsigned int size)
781
783
{
784
+
struct inode *inode = file_inode(file);
782
785
loff_t merkle_pos = merkle_file_pos(inode);
783
786
784
787
if (merkle_pos < 0)
···
792
793
}
793
794
794
795
const struct fsverity_operations btrfs_verityops = {
795
-
.inode_info_offs = (int)offsetof(struct btrfs_inode, i_verity_info) -
796
-
(int)offsetof(struct btrfs_inode, vfs_inode),
797
796
.begin_enable_verity = btrfs_begin_enable_verity,
798
797
.end_enable_verity = btrfs_end_enable_verity,
799
798
.get_verity_descriptor = btrfs_get_verity_descriptor,
+11
-14
fs/buffer.c
+11
-14
fs/buffer.c
···
303
303
struct postprocess_bh_ctx {
304
304
struct work_struct work;
305
305
struct buffer_head *bh;
306
+
struct fsverity_info *vi;
306
307
};
307
308
308
309
static void verify_bh(struct work_struct *work)
···
313
312
struct buffer_head *bh = ctx->bh;
314
313
bool valid;
315
314
316
-
valid = fsverity_verify_blocks(bh->b_folio, bh->b_size, bh_offset(bh));
315
+
valid = fsverity_verify_blocks(ctx->vi, bh->b_folio, bh->b_size,
316
+
bh_offset(bh));
317
317
end_buffer_async_read(bh, valid);
318
318
kfree(ctx);
319
-
}
320
-
321
-
static bool need_fsverity(struct buffer_head *bh)
322
-
{
323
-
struct folio *folio = bh->b_folio;
324
-
struct inode *inode = folio->mapping->host;
325
-
326
-
return fsverity_active(inode) &&
327
-
/* needed by ext4 */
328
-
folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
329
319
}
330
320
331
321
static void decrypt_bh(struct work_struct *work)
···
328
336
329
337
err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size,
330
338
bh_offset(bh));
331
-
if (err == 0 && need_fsverity(bh)) {
339
+
if (err == 0 && ctx->vi) {
332
340
/*
333
341
* We use different work queues for decryption and for verity
334
342
* because verity may require reading metadata pages that need
···
350
358
{
351
359
struct inode *inode = bh->b_folio->mapping->host;
352
360
bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode);
353
-
bool verify = need_fsverity(bh);
361
+
struct fsverity_info *vi = NULL;
362
+
363
+
/* needed by ext4 */
364
+
if (bh->b_folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE))
365
+
vi = fsverity_get_info(inode);
354
366
355
367
/* Decrypt (with fscrypt) and/or verify (with fsverity) if needed. */
356
-
if (uptodate && (decrypt || verify)) {
368
+
if (uptodate && (decrypt || vi)) {
357
369
struct postprocess_bh_ctx *ctx =
358
370
kmalloc(sizeof(*ctx), GFP_ATOMIC);
359
371
360
372
if (ctx) {
361
373
ctx->bh = bh;
374
+
ctx->vi = vi;
362
375
if (decrypt) {
363
376
INIT_WORK(&ctx->work, decrypt_bh);
364
377
fscrypt_enqueue_decrypt_work(&ctx->work);
+2
-6
fs/ext4/ext4.h
+2
-6
fs/ext4/ext4.h
···
1196
1196
#ifdef CONFIG_FS_ENCRYPTION
1197
1197
struct fscrypt_inode_info *i_crypt_info;
1198
1198
#endif
1199
-
1200
-
#ifdef CONFIG_FS_VERITY
1201
-
struct fsverity_info *i_verity_info;
1202
-
#endif
1203
1199
};
1204
1200
1205
1201
/*
···
3740
3744
}
3741
3745
3742
3746
/* readpages.c */
3743
-
extern int ext4_mpage_readpages(struct inode *inode,
3744
-
struct readahead_control *rac, struct folio *folio);
3747
+
int ext4_read_folio(struct file *file, struct folio *folio);
3748
+
void ext4_readahead(struct readahead_control *rac);
3745
3749
extern int __init ext4_init_post_read_processing(void);
3746
3750
extern void ext4_exit_post_read_processing(void);
3747
3751
-31
fs/ext4/inode.c
-31
fs/ext4/inode.c
···
3373
3373
return ret;
3374
3374
}
3375
3375
3376
-
static int ext4_read_folio(struct file *file, struct folio *folio)
3377
-
{
3378
-
int ret = -EAGAIN;
3379
-
struct inode *inode = folio->mapping->host;
3380
-
3381
-
trace_ext4_read_folio(inode, folio);
3382
-
3383
-
if (ext4_has_inline_data(inode))
3384
-
ret = ext4_readpage_inline(inode, folio);
3385
-
3386
-
if (ret == -EAGAIN)
3387
-
return ext4_mpage_readpages(inode, NULL, folio);
3388
-
3389
-
return ret;
3390
-
}
3391
-
3392
-
static void ext4_readahead(struct readahead_control *rac)
3393
-
{
3394
-
struct inode *inode = rac->mapping->host;
3395
-
3396
-
/* If the file has inline data, no need to do readahead. */
3397
-
if (ext4_has_inline_data(inode))
3398
-
return;
3399
-
3400
-
ext4_mpage_readpages(inode, rac, NULL);
3401
-
}
3402
-
3403
3376
static void ext4_invalidate_folio(struct folio *folio, size_t offset,
3404
3377
size_t length)
3405
3378
{
···
5785
5812
return error;
5786
5813
5787
5814
error = fscrypt_prepare_setattr(dentry, attr);
5788
-
if (error)
5789
-
return error;
5790
-
5791
-
error = fsverity_prepare_setattr(dentry, attr);
5792
5815
if (error)
5793
5816
return error;
5794
5817
+50
-14
fs/ext4/readpage.c
+50
-14
fs/ext4/readpage.c
···
46
46
#include <linux/pagevec.h>
47
47
48
48
#include "ext4.h"
49
+
#include <trace/events/ext4.h>
49
50
50
51
#define NUM_PREALLOC_POST_READ_CTXS 128
51
52
···
63
62
64
63
struct bio_post_read_ctx {
65
64
struct bio *bio;
65
+
struct fsverity_info *vi;
66
66
struct work_struct work;
67
67
unsigned int cur_step;
68
68
unsigned int enabled_steps;
···
99
97
struct bio_post_read_ctx *ctx =
100
98
container_of(work, struct bio_post_read_ctx, work);
101
99
struct bio *bio = ctx->bio;
100
+
struct fsverity_info *vi = ctx->vi;
102
101
103
102
/*
104
103
* fsverity_verify_bio() may call readahead() again, and although verity
···
112
109
mempool_free(ctx, bio_post_read_ctx_pool);
113
110
bio->bi_private = NULL;
114
111
115
-
fsverity_verify_bio(bio);
112
+
fsverity_verify_bio(vi, bio);
116
113
117
114
__read_end_io(bio);
118
115
}
···
134
131
ctx->cur_step++;
135
132
fallthrough;
136
133
case STEP_VERITY:
137
-
if (ctx->enabled_steps & (1 << STEP_VERITY)) {
134
+
if (IS_ENABLED(CONFIG_FS_VERITY) &&
135
+
ctx->enabled_steps & (1 << STEP_VERITY)) {
138
136
INIT_WORK(&ctx->work, verity_work);
139
137
fsverity_enqueue_verify_work(&ctx->work);
140
138
return;
···
176
172
__read_end_io(bio);
177
173
}
178
174
179
-
static inline bool ext4_need_verity(const struct inode *inode, pgoff_t idx)
180
-
{
181
-
return fsverity_active(inode) &&
182
-
idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
183
-
}
184
-
185
175
static void ext4_set_bio_post_read_ctx(struct bio *bio,
186
176
const struct inode *inode,
187
-
pgoff_t first_idx)
177
+
struct fsverity_info *vi)
188
178
{
189
179
unsigned int post_read_steps = 0;
190
180
191
181
if (fscrypt_inode_uses_fs_layer_crypto(inode))
192
182
post_read_steps |= 1 << STEP_DECRYPT;
193
183
194
-
if (ext4_need_verity(inode, first_idx))
184
+
if (vi)
195
185
post_read_steps |= 1 << STEP_VERITY;
196
186
197
187
if (post_read_steps) {
···
194
196
mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
195
197
196
198
ctx->bio = bio;
199
+
ctx->vi = vi;
197
200
ctx->enabled_steps = post_read_steps;
198
201
bio->bi_private = ctx;
199
202
}
···
208
209
return i_size_read(inode);
209
210
}
210
211
211
-
int ext4_mpage_readpages(struct inode *inode,
212
+
static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi,
212
213
struct readahead_control *rac, struct folio *folio)
213
214
{
214
215
struct bio *bio = NULL;
···
328
329
folio_zero_segment(folio, first_hole << blkbits,
329
330
folio_size(folio));
330
331
if (first_hole == 0) {
331
-
if (ext4_need_verity(inode, folio->index) &&
332
-
!fsverity_verify_folio(folio))
332
+
if (vi && !fsverity_verify_folio(vi, folio))
333
333
goto set_error_page;
334
334
folio_end_read(folio, true);
335
335
continue;
···
356
358
REQ_OP_READ, GFP_KERNEL);
357
359
fscrypt_set_bio_crypt_ctx(bio, inode, next_block,
358
360
GFP_KERNEL);
359
-
ext4_set_bio_post_read_ctx(bio, inode, folio->index);
361
+
ext4_set_bio_post_read_ctx(bio, inode, vi);
360
362
bio->bi_iter.bi_sector = first_block << (blkbits - 9);
361
363
bio->bi_end_io = mpage_end_io;
362
364
if (rac)
···
390
392
if (bio)
391
393
blk_crypto_submit_bio(bio);
392
394
return 0;
395
+
}
396
+
397
+
int ext4_read_folio(struct file *file, struct folio *folio)
398
+
{
399
+
struct inode *inode = folio->mapping->host;
400
+
struct fsverity_info *vi = NULL;
401
+
int ret;
402
+
403
+
trace_ext4_read_folio(inode, folio);
404
+
405
+
if (ext4_has_inline_data(inode)) {
406
+
ret = ext4_readpage_inline(inode, folio);
407
+
if (ret != -EAGAIN)
408
+
return ret;
409
+
}
410
+
411
+
if (folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE))
412
+
vi = fsverity_get_info(inode);
413
+
if (vi)
414
+
fsverity_readahead(vi, folio->index, folio_nr_pages(folio));
415
+
return ext4_mpage_readpages(inode, vi, NULL, folio);
416
+
}
417
+
418
+
void ext4_readahead(struct readahead_control *rac)
419
+
{
420
+
struct inode *inode = rac->mapping->host;
421
+
struct fsverity_info *vi = NULL;
422
+
423
+
/* If the file has inline data, no need to do readahead. */
424
+
if (ext4_has_inline_data(inode))
425
+
return;
426
+
427
+
if (readahead_index(rac) < DIV_ROUND_UP(inode->i_size, PAGE_SIZE))
428
+
vi = fsverity_get_info(inode);
429
+
if (vi)
430
+
fsverity_readahead(vi, readahead_index(rac),
431
+
readahead_count(rac));
432
+
ext4_mpage_readpages(inode, vi, rac, NULL);
393
433
}
394
434
395
435
int __init ext4_init_post_read_processing(void)
-4
fs/ext4/super.c
-4
fs/ext4/super.c
···
1489
1489
#ifdef CONFIG_FS_ENCRYPTION
1490
1490
ei->i_crypt_info = NULL;
1491
1491
#endif
1492
-
#ifdef CONFIG_FS_VERITY
1493
-
ei->i_verity_info = NULL;
1494
-
#endif
1495
1492
}
1496
1493
1497
1494
static int __init init_inodecache(void)
···
1536
1539
EXT4_I(inode)->jinode = NULL;
1537
1540
}
1538
1541
fscrypt_put_encryption_info(inode);
1539
-
fsverity_cleanup_inode(inode);
1540
1542
}
1541
1543
1542
1544
static struct inode *ext4_nfs_get_inode(struct super_block *sb,
+13
-23
fs/ext4/verity.c
+13
-23
fs/ext4/verity.c
···
360
360
}
361
361
362
362
static struct page *ext4_read_merkle_tree_page(struct inode *inode,
363
-
pgoff_t index,
364
-
unsigned long num_ra_pages)
363
+
pgoff_t index)
365
364
{
366
-
struct folio *folio;
367
-
368
365
index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT;
369
-
370
-
folio = __filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0);
371
-
if (IS_ERR(folio) || !folio_test_uptodate(folio)) {
372
-
DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index);
373
-
374
-
if (!IS_ERR(folio))
375
-
folio_put(folio);
376
-
else if (num_ra_pages > 1)
377
-
page_cache_ra_unbounded(&ractl, num_ra_pages, 0);
378
-
folio = read_mapping_folio(inode->i_mapping, index, NULL);
379
-
if (IS_ERR(folio))
380
-
return ERR_CAST(folio);
381
-
}
382
-
return folio_file_page(folio, index);
366
+
return generic_read_merkle_tree_page(inode, index);
383
367
}
384
368
385
-
static int ext4_write_merkle_tree_block(struct inode *inode, const void *buf,
369
+
static void ext4_readahead_merkle_tree(struct inode *inode, pgoff_t index,
370
+
unsigned long nr_pages)
371
+
{
372
+
index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT;
373
+
generic_readahead_merkle_tree(inode, index, nr_pages);
374
+
}
375
+
376
+
static int ext4_write_merkle_tree_block(struct file *file, const void *buf,
386
377
u64 pos, unsigned int size)
387
378
{
388
-
pos += ext4_verity_metadata_pos(inode);
379
+
pos += ext4_verity_metadata_pos(file_inode(file));
389
380
390
-
return pagecache_write(inode, buf, size, pos);
381
+
return pagecache_write(file_inode(file), buf, size, pos);
391
382
}
392
383
393
384
const struct fsverity_operations ext4_verityops = {
394
-
.inode_info_offs = (int)offsetof(struct ext4_inode_info, i_verity_info) -
395
-
(int)offsetof(struct ext4_inode_info, vfs_inode),
396
385
.begin_enable_verity = ext4_begin_enable_verity,
397
386
.end_enable_verity = ext4_end_enable_verity,
398
387
.get_verity_descriptor = ext4_get_verity_descriptor,
399
388
.read_merkle_tree_page = ext4_read_merkle_tree_page,
389
+
.readahead_merkle_tree = ext4_readahead_merkle_tree,
400
390
.write_merkle_tree_block = ext4_write_merkle_tree_block,
401
391
};
+4
-3
fs/f2fs/compress.c
+4
-3
fs/f2fs/compress.c
···
1181
1181
.cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
1182
1182
.rpages = NULL,
1183
1183
.nr_rpages = 0,
1184
+
.vi = NULL, /* can't write to fsverity files */
1184
1185
};
1185
1186
1186
1187
return prepare_compress_overwrite(&cc, pagep, index, fsdata);
···
1717
1716
dic->nr_cpages = cc->nr_cpages;
1718
1717
refcount_set(&dic->refcnt, 1);
1719
1718
dic->failed = false;
1720
-
dic->need_verity = f2fs_need_verity(cc->inode, start_idx);
1719
+
dic->vi = cc->vi;
1721
1720
1722
1721
for (i = 0; i < dic->cluster_size; i++)
1723
1722
dic->rpages[i] = cc->rpages[i];
···
1815
1814
if (!rpage)
1816
1815
continue;
1817
1816
1818
-
if (fsverity_verify_page(rpage))
1817
+
if (fsverity_verify_page(dic->vi, rpage))
1819
1818
SetPageUptodate(rpage);
1820
1819
else
1821
1820
ClearPageUptodate(rpage);
···
1834
1833
{
1835
1834
int i;
1836
1835
1837
-
if (!failed && dic->need_verity) {
1836
+
if (IS_ENABLED(CONFIG_FS_VERITY) && !failed && dic->vi) {
1838
1837
/*
1839
1838
* Note that to avoid deadlocks, the verity work can't be done
1840
1839
* on the decompression workqueue. This is because verifying
+63
-37
fs/f2fs/data.c
+63
-37
fs/f2fs/data.c
···
109
109
struct bio_post_read_ctx {
110
110
struct bio *bio;
111
111
struct f2fs_sb_info *sbi;
112
+
struct fsverity_info *vi;
112
113
struct work_struct work;
113
114
unsigned int enabled_steps;
114
115
/*
···
166
165
container_of(work, struct bio_post_read_ctx, work);
167
166
struct bio *bio = ctx->bio;
168
167
bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS);
168
+
struct fsverity_info *vi = ctx->vi;
169
169
170
170
/*
171
171
* fsverity_verify_bio() may call readahead() again, and while verity
···
189
187
struct folio *folio = fi.folio;
190
188
191
189
if (!f2fs_is_compressed_page(folio) &&
192
-
!fsverity_verify_page(&folio->page)) {
190
+
!fsverity_verify_page(vi, &folio->page)) {
193
191
bio->bi_status = BLK_STS_IOERR;
194
192
break;
195
193
}
196
194
}
197
195
} else {
198
-
fsverity_verify_bio(bio);
196
+
fsverity_verify_bio(vi, bio);
199
197
}
200
198
201
199
f2fs_finish_read_bio(bio, true);
···
1038
1036
f2fs_up_write(&io->io_rwsem);
1039
1037
}
1040
1038
1041
-
static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
1039
+
static struct bio *f2fs_grab_read_bio(struct inode *inode,
1040
+
struct fsverity_info *vi, block_t blkaddr,
1042
1041
unsigned nr_pages, blk_opf_t op_flag,
1043
1042
pgoff_t first_idx, bool for_write)
1044
1043
{
···
1060
1057
if (fscrypt_inode_uses_fs_layer_crypto(inode))
1061
1058
post_read_steps |= STEP_DECRYPT;
1062
1059
1063
-
if (f2fs_need_verity(inode, first_idx))
1060
+
if (vi)
1064
1061
post_read_steps |= STEP_VERITY;
1065
1062
1066
1063
/*
···
1075
1072
ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
1076
1073
ctx->bio = bio;
1077
1074
ctx->sbi = sbi;
1075
+
ctx->vi = vi;
1078
1076
ctx->enabled_steps = post_read_steps;
1079
1077
ctx->fs_blkaddr = blkaddr;
1080
1078
ctx->decompression_attempted = false;
···
1087
1083
}
1088
1084
1089
1085
/* This can handle encryption stuffs */
1090
-
static void f2fs_submit_page_read(struct inode *inode, struct folio *folio,
1091
-
block_t blkaddr, blk_opf_t op_flags,
1092
-
bool for_write)
1086
+
static void f2fs_submit_page_read(struct inode *inode, struct fsverity_info *vi,
1087
+
struct folio *folio, block_t blkaddr,
1088
+
blk_opf_t op_flags, bool for_write)
1093
1089
{
1094
1090
struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1095
1091
struct bio *bio;
1096
1092
1097
-
bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags,
1098
-
folio->index, for_write);
1093
+
bio = f2fs_grab_read_bio(inode, vi, blkaddr, 1, op_flags, folio->index,
1094
+
for_write);
1099
1095
1100
1096
/* wait for GCed page writeback via META_MAPPING */
1101
1097
f2fs_wait_on_block_writeback(inode, blkaddr);
···
1197
1193
return err;
1198
1194
}
1199
1195
1196
+
static inline struct fsverity_info *f2fs_need_verity(const struct inode *inode,
1197
+
pgoff_t idx)
1198
+
{
1199
+
if (idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE))
1200
+
return fsverity_get_info(inode);
1201
+
return NULL;
1202
+
}
1203
+
1200
1204
struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index,
1201
1205
blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs)
1202
1206
{
···
1270
1258
return folio;
1271
1259
}
1272
1260
1273
-
f2fs_submit_page_read(inode, folio, dn.data_blkaddr,
1274
-
op_flags, for_write);
1261
+
f2fs_submit_page_read(inode, f2fs_need_verity(inode, folio->index),
1262
+
folio, dn.data_blkaddr, op_flags, for_write);
1275
1263
return folio;
1276
1264
1277
1265
put_err:
···
2075
2063
return rac ? REQ_RAHEAD : 0;
2076
2064
}
2077
2065
2078
-
static int f2fs_read_single_page(struct inode *inode, struct folio *folio,
2079
-
unsigned nr_pages,
2080
-
struct f2fs_map_blocks *map,
2081
-
struct bio **bio_ret,
2082
-
sector_t *last_block_in_bio,
2083
-
struct readahead_control *rac)
2066
+
static int f2fs_read_single_page(struct inode *inode, struct fsverity_info *vi,
2067
+
struct folio *folio, unsigned int nr_pages,
2068
+
struct f2fs_map_blocks *map,
2069
+
struct bio **bio_ret,
2070
+
sector_t *last_block_in_bio,
2071
+
struct readahead_control *rac)
2084
2072
{
2085
2073
struct bio *bio = *bio_ret;
2086
2074
const unsigned int blocksize = F2FS_BLKSIZE;
···
2132
2120
} else {
2133
2121
zero_out:
2134
2122
folio_zero_segment(folio, 0, folio_size(folio));
2135
-
if (f2fs_need_verity(inode, index) &&
2136
-
!fsverity_verify_folio(folio)) {
2123
+
if (vi && !fsverity_verify_folio(vi, folio)) {
2137
2124
ret = -EIO;
2138
2125
goto out;
2139
2126
}
···
2154
2143
bio = NULL;
2155
2144
}
2156
2145
if (bio == NULL)
2157
-
bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
2158
-
f2fs_ra_op_flags(rac), index,
2159
-
false);
2146
+
bio = f2fs_grab_read_bio(inode, vi, block_nr, nr_pages,
2147
+
f2fs_ra_op_flags(rac), index, false);
2160
2148
2161
2149
/*
2162
2150
* If the page is under writeback, we need to wait for
···
2305
2295
}
2306
2296
2307
2297
if (!bio)
2308
-
bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages - i,
2309
-
f2fs_ra_op_flags(rac),
2310
-
folio->index, for_write);
2298
+
bio = f2fs_grab_read_bio(inode, cc->vi, blkaddr,
2299
+
nr_pages - i,
2300
+
f2fs_ra_op_flags(rac),
2301
+
folio->index, for_write);
2311
2302
2312
2303
if (!bio_add_folio(bio, folio, blocksize, 0))
2313
2304
goto submit_and_realloc;
···
2347
2336
* This function was originally taken from fs/mpage.c, and customized for f2fs.
2348
2337
* Major change was from block_size == page_size in f2fs by default.
2349
2338
*/
2350
-
static int f2fs_mpage_readpages(struct inode *inode,
2339
+
static int f2fs_mpage_readpages(struct inode *inode, struct fsverity_info *vi,
2351
2340
struct readahead_control *rac, struct folio *folio)
2352
2341
{
2353
2342
struct bio *bio = NULL;
···
2402
2391
2403
2392
/* there are remained compressed pages, submit them */
2404
2393
if (!f2fs_cluster_can_merge_page(&cc, index)) {
2394
+
cc.vi = vi;
2405
2395
ret = f2fs_read_multi_pages(&cc, &bio,
2406
2396
max_nr_pages,
2407
2397
&last_block_in_bio,
···
2436
2424
read_single_page:
2437
2425
#endif
2438
2426
2439
-
ret = f2fs_read_single_page(inode, folio, max_nr_pages, &map,
2440
-
&bio, &last_block_in_bio, rac);
2427
+
ret = f2fs_read_single_page(inode, vi, folio, max_nr_pages,
2428
+
&map, &bio, &last_block_in_bio,
2429
+
rac);
2441
2430
if (ret) {
2442
2431
#ifdef CONFIG_F2FS_FS_COMPRESSION
2443
2432
set_error_page:
···
2454
2441
if (f2fs_compressed_file(inode)) {
2455
2442
/* last page */
2456
2443
if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) {
2444
+
cc.vi = vi;
2457
2445
ret = f2fs_read_multi_pages(&cc, &bio,
2458
2446
max_nr_pages,
2459
2447
&last_block_in_bio,
···
2472
2458
static int f2fs_read_data_folio(struct file *file, struct folio *folio)
2473
2459
{
2474
2460
struct inode *inode = folio->mapping->host;
2475
-
int ret = -EAGAIN;
2461
+
struct fsverity_info *vi = NULL;
2462
+
int ret;
2476
2463
2477
2464
trace_f2fs_readpage(folio, DATA);
2478
2465
···
2483
2468
}
2484
2469
2485
2470
/* If the file has inline data, try to read it directly */
2486
-
if (f2fs_has_inline_data(inode))
2471
+
if (f2fs_has_inline_data(inode)) {
2487
2472
ret = f2fs_read_inline_data(inode, folio);
2488
-
if (ret == -EAGAIN)
2489
-
ret = f2fs_mpage_readpages(inode, NULL, folio);
2490
-
return ret;
2473
+
if (ret != -EAGAIN)
2474
+
return ret;
2475
+
}
2476
+
2477
+
vi = f2fs_need_verity(inode, folio->index);
2478
+
if (vi)
2479
+
fsverity_readahead(vi, folio->index, folio_nr_pages(folio));
2480
+
return f2fs_mpage_readpages(inode, vi, NULL, folio);
2491
2481
}
2492
2482
2493
2483
static void f2fs_readahead(struct readahead_control *rac)
2494
2484
{
2495
2485
struct inode *inode = rac->mapping->host;
2486
+
struct fsverity_info *vi = NULL;
2496
2487
2497
2488
trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac));
2498
2489
···
2509
2488
if (f2fs_has_inline_data(inode))
2510
2489
return;
2511
2490
2512
-
f2fs_mpage_readpages(inode, rac, NULL);
2491
+
vi = f2fs_need_verity(inode, readahead_index(rac));
2492
+
if (vi)
2493
+
fsverity_readahead(vi, readahead_index(rac),
2494
+
readahead_count(rac));
2495
+
f2fs_mpage_readpages(inode, vi, rac, NULL);
2513
2496
}
2514
2497
2515
2498
int f2fs_encrypt_one_page(struct f2fs_io_info *fio)
···
3662
3637
err = -EFSCORRUPTED;
3663
3638
goto put_folio;
3664
3639
}
3665
-
f2fs_submit_page_read(use_cow ?
3666
-
F2FS_I(inode)->cow_inode : inode,
3667
-
folio, blkaddr, 0, true);
3640
+
f2fs_submit_page_read(use_cow ? F2FS_I(inode)->cow_inode :
3641
+
inode,
3642
+
NULL, /* can't write to fsverity files */
3643
+
folio, blkaddr, 0, true);
3668
3644
3669
3645
folio_lock(folio);
3670
3646
if (unlikely(folio->mapping != mapping)) {
+2
-10
fs/f2fs/f2fs.h
+2
-10
fs/f2fs/f2fs.h
···
974
974
#ifdef CONFIG_FS_ENCRYPTION
975
975
struct fscrypt_inode_info *i_crypt_info; /* filesystem encryption info */
976
976
#endif
977
-
#ifdef CONFIG_FS_VERITY
978
-
struct fsverity_info *i_verity_info; /* filesystem verity info */
979
-
#endif
980
977
};
981
978
982
979
static inline void get_read_extent_info(struct extent_info *ext,
···
1600
1603
size_t clen; /* valid data length in cbuf */
1601
1604
void *private; /* payload buffer for specified compression algorithm */
1602
1605
void *private2; /* extra payload buffer */
1606
+
struct fsverity_info *vi; /* verity info if needed */
1603
1607
};
1604
1608
1605
1609
/* compress context for write IO path */
···
1656
1658
refcount_t refcnt;
1657
1659
1658
1660
bool failed; /* IO error occurred before decompression? */
1659
-
bool need_verity; /* need fs-verity verification after decompression? */
1661
+
struct fsverity_info *vi; /* fs-verity context if needed */
1660
1662
unsigned char compress_algorithm; /* backup algorithm type */
1661
1663
void *private; /* payload buffer for specified decompression algorithm */
1662
1664
void *private2; /* extra payload buffer */
···
4882
4884
if (flag != F2FS_GET_BLOCK_DIO)
4883
4885
return false;
4884
4886
return sbi->aligned_blksize;
4885
-
}
4886
-
4887
-
static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
4888
-
{
4889
-
return fsverity_active(inode) &&
4890
-
idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
4891
4887
}
4892
4888
4893
4889
#ifdef CONFIG_F2FS_FAULT_INJECTION
+2
-4
fs/f2fs/file.c
+2
-4
fs/f2fs/file.c
···
1076
1076
if (err)
1077
1077
return err;
1078
1078
1079
-
err = fsverity_prepare_setattr(dentry, attr);
1080
-
if (err)
1081
-
return err;
1082
-
1083
1079
if (unlikely(IS_IMMUTABLE(inode)))
1084
1080
return -EPERM;
1085
1081
···
4420
4424
pgoff_t redirty_idx = page_idx;
4421
4425
int page_len = 0, ret = 0;
4422
4426
4427
+
filemap_invalidate_lock_shared(mapping);
4423
4428
page_cache_ra_unbounded(&ractl, len, 0);
4429
+
filemap_invalidate_unlock_shared(mapping);
4424
4430
4425
4431
do {
4426
4432
folio = read_cache_folio(mapping, page_idx, NULL, NULL);
-1
fs/f2fs/inode.c
-1
fs/f2fs/inode.c
-3
fs/f2fs/super.c
-3
fs/f2fs/super.c
+13
-23
fs/f2fs/verity.c
+13
-23
fs/f2fs/verity.c
···
256
256
}
257
257
258
258
static struct page *f2fs_read_merkle_tree_page(struct inode *inode,
259
-
pgoff_t index,
260
-
unsigned long num_ra_pages)
259
+
pgoff_t index)
261
260
{
262
-
struct folio *folio;
263
-
264
261
index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT;
265
-
266
-
folio = f2fs_filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0);
267
-
if (IS_ERR(folio) || !folio_test_uptodate(folio)) {
268
-
DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index);
269
-
270
-
if (!IS_ERR(folio))
271
-
folio_put(folio);
272
-
else if (num_ra_pages > 1)
273
-
page_cache_ra_unbounded(&ractl, num_ra_pages, 0);
274
-
folio = read_mapping_folio(inode->i_mapping, index, NULL);
275
-
if (IS_ERR(folio))
276
-
return ERR_CAST(folio);
277
-
}
278
-
return folio_file_page(folio, index);
262
+
return generic_read_merkle_tree_page(inode, index);
279
263
}
280
264
281
-
static int f2fs_write_merkle_tree_block(struct inode *inode, const void *buf,
265
+
static void f2fs_readahead_merkle_tree(struct inode *inode, pgoff_t index,
266
+
unsigned long nr_pages)
267
+
{
268
+
index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT;
269
+
generic_readahead_merkle_tree(inode, index, nr_pages);
270
+
}
271
+
272
+
static int f2fs_write_merkle_tree_block(struct file *file, const void *buf,
282
273
u64 pos, unsigned int size)
283
274
{
284
-
pos += f2fs_verity_metadata_pos(inode);
275
+
pos += f2fs_verity_metadata_pos(file_inode(file));
285
276
286
-
return pagecache_write(inode, buf, size, pos);
277
+
return pagecache_write(file_inode(file), buf, size, pos);
287
278
}
288
279
289
280
const struct fsverity_operations f2fs_verityops = {
290
-
.inode_info_offs = (int)offsetof(struct f2fs_inode_info, i_verity_info) -
291
-
(int)offsetof(struct f2fs_inode_info, vfs_inode),
292
281
.begin_enable_verity = f2fs_begin_enable_verity,
293
282
.end_enable_verity = f2fs_end_enable_verity,
294
283
.get_verity_descriptor = f2fs_get_verity_descriptor,
295
284
.read_merkle_tree_page = f2fs_read_merkle_tree_page,
285
+
.readahead_merkle_tree = f2fs_readahead_merkle_tree,
296
286
.write_merkle_tree_block = f2fs_write_merkle_tree_block,
297
287
};
+9
fs/inode.c
+9
fs/inode.c
···
14
14
#include <linux/cdev.h>
15
15
#include <linux/memblock.h>
16
16
#include <linux/fsnotify.h>
17
+
#include <linux/fsverity.h>
17
18
#include <linux/mount.h>
18
19
#include <linux/posix_acl.h>
19
20
#include <linux/buffer_head.h> /* for inode_has_buffers */
···
774
773
775
774
void clear_inode(struct inode *inode)
776
775
{
776
+
/*
777
+
* Only IS_VERITY() inodes can have verity info, so start by checking
778
+
* for IS_VERITY() (which is faster than retrieving the pointer to the
779
+
* verity info). This minimizes overhead for non-verity inodes.
780
+
*/
781
+
if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
782
+
fsverity_cleanup_inode(inode);
783
+
777
784
/*
778
785
* We have to cycle the i_pages lock here because reclaim can be in the
779
786
* process of removing the last page (in __filemap_remove_folio())
+1
fs/verity/Makefile
+1
fs/verity/Makefile
+25
-16
fs/verity/enable.c
+25
-16
fs/verity/enable.c
···
41
41
return 0;
42
42
}
43
43
44
-
static int write_merkle_tree_block(struct inode *inode, const u8 *buf,
44
+
static int write_merkle_tree_block(struct file *file, const u8 *buf,
45
45
unsigned long index,
46
46
const struct merkle_tree_params *params)
47
47
{
48
+
struct inode *inode = file_inode(file);
48
49
u64 pos = (u64)index << params->log_blocksize;
49
50
int err;
50
51
51
-
err = inode->i_sb->s_vop->write_merkle_tree_block(inode, buf, pos,
52
+
err = inode->i_sb->s_vop->write_merkle_tree_block(file, buf, pos,
52
53
params->block_size);
53
54
if (err)
54
55
fsverity_err(inode, "Error %d writing Merkle tree block %lu",
···
136
135
err = hash_one_block(params, &buffers[level]);
137
136
if (err)
138
137
goto out;
139
-
err = write_merkle_tree_block(inode,
138
+
err = write_merkle_tree_block(filp,
140
139
buffers[level].data,
141
140
level_offset[level],
142
141
params);
···
156
155
err = hash_one_block(params, &buffers[level]);
157
156
if (err)
158
157
goto out;
159
-
err = write_merkle_tree_block(inode,
158
+
err = write_merkle_tree_block(filp,
160
159
buffers[level].data,
161
160
level_offset[level],
162
161
params);
···
266
265
}
267
266
268
267
/*
268
+
* Add the fsverity_info into the hash table before finishing the
269
+
* initialization so that we don't have to undo the enabling when memory
270
+
* allocation for the hash table fails. This is safe because looking up
271
+
* the fsverity_info always first checks the S_VERITY flag on the inode,
272
+
* which will only be set at the very end of the ->end_enable_verity
273
+
* method.
274
+
*/
275
+
err = fsverity_set_info(vi);
276
+
if (err) {
277
+
fsverity_free_info(vi);
278
+
goto rollback;
279
+
}
280
+
281
+
/*
269
282
* Tell the filesystem to finish enabling verity on the file.
270
-
* Serialized with ->begin_enable_verity() by the inode lock.
283
+
* Serialized with ->begin_enable_verity() by the inode lock. The file
284
+
* system needs to set the S_VERITY flag on the inode at the very end of
285
+
* the method, at which point the fsverity information can be accessed
286
+
* by other threads.
271
287
*/
272
288
inode_lock(inode);
273
289
err = vops->end_enable_verity(filp, desc, desc_size, params.tree_size);
···
292
274
if (err) {
293
275
fsverity_err(inode, "%ps() failed with err %d",
294
276
vops->end_enable_verity, err);
295
-
fsverity_free_info(vi);
277
+
fsverity_remove_info(vi);
296
278
} else if (WARN_ON_ONCE(!IS_VERITY(inode))) {
279
+
fsverity_remove_info(vi);
297
280
err = -EINVAL;
298
-
fsverity_free_info(vi);
299
-
} else {
300
-
/* Successfully enabled verity */
301
-
302
-
/*
303
-
* Readers can start using the inode's verity info immediately,
304
-
* so it can't be rolled back once set. So don't set it until
305
-
* just after the filesystem has successfully enabled verity.
306
-
*/
307
-
fsverity_set_info(inode, vi);
308
281
}
309
282
out:
310
283
kfree(params.hashstate);
+11
-9
fs/verity/fsverity_private.h
+11
-9
fs/verity/fsverity_private.h
···
11
11
#define pr_fmt(fmt) "fs-verity: " fmt
12
12
13
13
#include <linux/fsverity.h>
14
+
#include <linux/rhashtable.h>
14
15
15
16
/*
16
17
* Implementation limit: maximum depth of the Merkle tree. For now 8 is plenty;
···
64
63
* fsverity_info - cached verity metadata for an inode
65
64
*
66
65
* When a verity file is first opened, an instance of this struct is allocated
67
-
* and a pointer to it is stored in the file's in-memory inode. It remains
68
-
* until the inode is evicted. It caches information about the Merkle tree
69
-
* that's needed to efficiently verify data read from the file. It also caches
70
-
* the file digest. The Merkle tree pages themselves are not cached here, but
71
-
* the filesystem may cache them.
66
+
* and a pointer to it is stored in the global hash table, indexed by the inode
67
+
* pointer value. It remains alive until the inode is evicted. It caches
68
+
* information about the Merkle tree that's needed to efficiently verify data
69
+
* read from the file. It also caches the file digest. The Merkle tree pages
70
+
* themselves are not cached here, but the filesystem may cache them.
72
71
*/
73
72
struct fsverity_info {
73
+
struct rhash_head rhash_head;
74
74
struct merkle_tree_params tree_params;
75
75
u8 root_hash[FS_VERITY_MAX_DIGEST_SIZE];
76
76
u8 file_digest[FS_VERITY_MAX_DIGEST_SIZE];
77
-
const struct inode *inode;
77
+
struct inode *inode;
78
78
unsigned long *hash_block_verified;
79
79
};
80
80
···
126
124
unsigned int log_blocksize,
127
125
const u8 *salt, size_t salt_size);
128
126
129
-
struct fsverity_info *fsverity_create_info(const struct inode *inode,
127
+
struct fsverity_info *fsverity_create_info(struct inode *inode,
130
128
struct fsverity_descriptor *desc);
131
129
132
-
void fsverity_set_info(struct inode *inode, struct fsverity_info *vi);
133
-
130
+
int fsverity_set_info(struct fsverity_info *vi);
134
131
void fsverity_free_info(struct fsverity_info *vi);
132
+
void fsverity_remove_info(struct fsverity_info *vi);
135
133
136
134
int fsverity_get_descriptor(struct inode *inode,
137
135
struct fsverity_descriptor **desc_ret);
+49
-37
fs/verity/open.c
+49
-37
fs/verity/open.c
···
12
12
#include <linux/slab.h>
13
13
14
14
static struct kmem_cache *fsverity_info_cachep;
15
+
static struct rhashtable fsverity_info_hash;
16
+
17
+
static const struct rhashtable_params fsverity_info_hash_params = {
18
+
.key_len = sizeof_field(struct fsverity_info, inode),
19
+
.key_offset = offsetof(struct fsverity_info, inode),
20
+
.head_offset = offsetof(struct fsverity_info, rhash_head),
21
+
.automatic_shrinking = true,
22
+
};
15
23
16
24
/**
17
25
* fsverity_init_merkle_tree_params() - initialize Merkle tree parameters
···
183
175
* appended builtin signature), and check the signature if present. The
184
176
* fsverity_descriptor must have already undergone basic validation.
185
177
*/
186
-
struct fsverity_info *fsverity_create_info(const struct inode *inode,
178
+
struct fsverity_info *fsverity_create_info(struct inode *inode,
187
179
struct fsverity_descriptor *desc)
188
180
{
189
181
struct fsverity_info *vi;
···
249
241
return ERR_PTR(err);
250
242
}
251
243
252
-
void fsverity_set_info(struct inode *inode, struct fsverity_info *vi)
244
+
int fsverity_set_info(struct fsverity_info *vi)
253
245
{
254
-
/*
255
-
* Multiple tasks may race to set the inode's verity info pointer, so
256
-
* use cmpxchg_release(). This pairs with the smp_load_acquire() in
257
-
* fsverity_get_info(). I.e., publish the pointer with a RELEASE
258
-
* barrier so that other tasks can ACQUIRE it.
259
-
*/
260
-
if (cmpxchg_release(fsverity_info_addr(inode), NULL, vi) != NULL) {
261
-
/* Lost the race, so free the verity info we allocated. */
262
-
fsverity_free_info(vi);
263
-
/*
264
-
* Afterwards, the caller may access the inode's verity info
265
-
* directly, so make sure to ACQUIRE the winning verity info.
266
-
*/
267
-
(void)fsverity_get_info(inode);
268
-
}
246
+
return rhashtable_lookup_insert_fast(&fsverity_info_hash,
247
+
&vi->rhash_head,
248
+
fsverity_info_hash_params);
269
249
}
270
250
271
-
void fsverity_free_info(struct fsverity_info *vi)
251
+
struct fsverity_info *__fsverity_get_info(const struct inode *inode)
272
252
{
273
-
if (!vi)
274
-
return;
275
-
kfree(vi->tree_params.hashstate);
276
-
kvfree(vi->hash_block_verified);
277
-
kmem_cache_free(fsverity_info_cachep, vi);
253
+
return rhashtable_lookup_fast(&fsverity_info_hash, &inode,
254
+
fsverity_info_hash_params);
278
255
}
256
+
EXPORT_SYMBOL_GPL(__fsverity_get_info);
279
257
280
258
static bool validate_fsverity_descriptor(struct inode *inode,
281
259
const struct fsverity_descriptor *desc,
···
346
352
347
353
static int ensure_verity_info(struct inode *inode)
348
354
{
349
-
struct fsverity_info *vi = fsverity_get_info(inode);
355
+
struct fsverity_info *vi = fsverity_get_info(inode), *found;
350
356
struct fsverity_descriptor *desc;
351
357
int err;
352
358
···
363
369
goto out_free_desc;
364
370
}
365
371
366
-
fsverity_set_info(inode, vi);
367
-
err = 0;
372
+
/*
373
+
* Multiple tasks may race to set the inode's verity info, in which case
374
+
* we might find an existing fsverity_info in the hash table.
375
+
*/
376
+
found = rhashtable_lookup_get_insert_fast(&fsverity_info_hash,
377
+
&vi->rhash_head,
378
+
fsverity_info_hash_params);
379
+
if (found) {
380
+
fsverity_free_info(vi);
381
+
if (IS_ERR(found))
382
+
err = PTR_ERR(found);
383
+
}
384
+
368
385
out_free_desc:
369
386
kfree(desc);
370
387
return err;
···
389
384
}
390
385
EXPORT_SYMBOL_GPL(__fsverity_file_open);
391
386
392
-
int __fsverity_prepare_setattr(struct dentry *dentry, struct iattr *attr)
387
+
void fsverity_free_info(struct fsverity_info *vi)
393
388
{
394
-
if (attr->ia_valid & ATTR_SIZE)
395
-
return -EPERM;
396
-
return 0;
389
+
kfree(vi->tree_params.hashstate);
390
+
kvfree(vi->hash_block_verified);
391
+
kmem_cache_free(fsverity_info_cachep, vi);
397
392
}
398
-
EXPORT_SYMBOL_GPL(__fsverity_prepare_setattr);
399
393
400
-
void __fsverity_cleanup_inode(struct inode *inode)
394
+
void fsverity_remove_info(struct fsverity_info *vi)
401
395
{
402
-
struct fsverity_info **vi_addr = fsverity_info_addr(inode);
403
-
404
-
fsverity_free_info(*vi_addr);
405
-
*vi_addr = NULL;
396
+
rhashtable_remove_fast(&fsverity_info_hash, &vi->rhash_head,
397
+
fsverity_info_hash_params);
398
+
fsverity_free_info(vi);
406
399
}
407
-
EXPORT_SYMBOL_GPL(__fsverity_cleanup_inode);
400
+
401
+
void fsverity_cleanup_inode(struct inode *inode)
402
+
{
403
+
struct fsverity_info *vi = fsverity_get_info(inode);
404
+
405
+
if (vi)
406
+
fsverity_remove_info(vi);
407
+
}
408
408
409
409
void __init fsverity_init_info_cache(void)
410
410
{
411
+
if (rhashtable_init(&fsverity_info_hash, &fsverity_info_hash_params))
412
+
panic("failed to initialize fsverity hash\n");
411
413
fsverity_info_cachep = KMEM_CACHE_USERCOPY(
412
414
fsverity_info,
413
415
SLAB_RECLAIM_ACCOUNT | SLAB_PANIC,
+58
fs/verity/pagecache.c
+58
fs/verity/pagecache.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* Copyright 2019 Google LLC
4
+
*/
5
+
6
+
#include <linux/export.h>
7
+
#include <linux/fsverity.h>
8
+
#include <linux/pagemap.h>
9
+
10
+
/**
11
+
* generic_read_merkle_tree_page - generic ->read_merkle_tree_page helper
12
+
* @inode: inode containing the Merkle tree
13
+
* @index: 0-based index of the Merkle tree page in the inode
14
+
*
15
+
* The caller needs to adjust @index from the Merkle-tree relative index passed
16
+
* to ->read_merkle_tree_page to the actual index where the Merkle tree is
17
+
* stored in the page cache for @inode.
18
+
*/
19
+
struct page *generic_read_merkle_tree_page(struct inode *inode, pgoff_t index)
20
+
{
21
+
struct folio *folio;
22
+
23
+
folio = read_mapping_folio(inode->i_mapping, index, NULL);
24
+
if (IS_ERR(folio))
25
+
return ERR_CAST(folio);
26
+
return folio_file_page(folio, index);
27
+
}
28
+
EXPORT_SYMBOL_GPL(generic_read_merkle_tree_page);
29
+
30
+
/**
31
+
* generic_readahead_merkle_tree() - generic ->readahead_merkle_tree helper
32
+
* @inode: inode containing the Merkle tree
33
+
* @index: 0-based index of the first Merkle tree page to read ahead in the
34
+
* inode
35
+
* @nr_pages: the number of Merkle tree pages that should be read ahead
36
+
*
37
+
* The caller needs to adjust @index from the Merkle-tree relative index passed
38
+
* to ->read_merkle_tree_page to the actual index where the Merkle tree is
39
+
* stored in the page cache for @inode.
40
+
*/
41
+
void generic_readahead_merkle_tree(struct inode *inode, pgoff_t index,
42
+
unsigned long nr_pages)
43
+
{
44
+
struct folio *folio;
45
+
46
+
lockdep_assert_held(&inode->i_mapping->invalidate_lock);
47
+
48
+
folio = __filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0);
49
+
if (folio == ERR_PTR(-ENOENT) ||
50
+
(!IS_ERR(folio) && !folio_test_uptodate(folio))) {
51
+
DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index);
52
+
53
+
page_cache_ra_unbounded(&ractl, nr_pages, 0);
54
+
}
55
+
if (!IS_ERR(folio))
56
+
folio_put(folio);
57
+
}
58
+
EXPORT_SYMBOL_GPL(generic_readahead_merkle_tree);
+14
-5
fs/verity/read_metadata.c
+14
-5
fs/verity/read_metadata.c
···
28
28
if (offset >= end_offset)
29
29
return 0;
30
30
offs_in_page = offset_in_page(offset);
31
+
index = offset >> PAGE_SHIFT;
31
32
last_index = (end_offset - 1) >> PAGE_SHIFT;
33
+
34
+
/*
35
+
* Kick off readahead for the range we are going to read to ensure a
36
+
* single large sequential read instead of lots of small ones.
37
+
*/
38
+
if (inode->i_sb->s_vop->readahead_merkle_tree) {
39
+
filemap_invalidate_lock_shared(inode->i_mapping);
40
+
inode->i_sb->s_vop->readahead_merkle_tree(
41
+
inode, index, last_index - index + 1);
42
+
filemap_invalidate_unlock_shared(inode->i_mapping);
43
+
}
32
44
33
45
/*
34
46
* Iterate through each Merkle tree page in the requested range and copy
···
48
36
* size isn't important here, as we are returning a byte stream; i.e.,
49
37
* we can just work with pages even if the tree block size != PAGE_SIZE.
50
38
*/
51
-
for (index = offset >> PAGE_SHIFT; index <= last_index; index++) {
52
-
unsigned long num_ra_pages =
53
-
min_t(unsigned long, last_index - index + 1,
54
-
inode->i_sb->s_bdi->io_pages);
39
+
for (; index <= last_index; index++) {
55
40
unsigned int bytes_to_copy = min_t(u64, end_offset - offset,
56
41
PAGE_SIZE - offs_in_page);
57
42
struct page *page;
58
43
const void *virt;
59
44
60
-
page = vops->read_merkle_tree_page(inode, index, num_ra_pages);
45
+
page = vops->read_merkle_tree_page(inode, index);
61
46
if (IS_ERR(page)) {
62
47
err = PTR_ERR(page);
63
48
fsverity_err(inode,
+57
-34
fs/verity/verify.c
+57
-34
fs/verity/verify.c
···
19
19
};
20
20
21
21
struct fsverity_verification_context {
22
-
struct inode *inode;
23
22
struct fsverity_info *vi;
24
-
unsigned long max_ra_pages;
25
23
26
24
/*
27
25
* This is the queue of data blocks that are pending verification. When
···
34
36
};
35
37
36
38
static struct workqueue_struct *fsverity_read_workqueue;
39
+
40
+
/**
41
+
* fsverity_readahead() - kick off readahead on fsverity hashes
42
+
* @vi: fsverity_info for the inode to be read
43
+
* @index: first file data page index that is being read
44
+
* @nr_pages: number of file data pages to be read
45
+
*
46
+
* Start readahead on the fsverity hashes that are needed to verify the file
47
+
* data in the range from @index to @index + @nr_pages (exclusive upper bound).
48
+
*
49
+
* To be called from the file systems' ->read_folio and ->readahead methods to
50
+
* ensure that the hashes are already cached on completion of the file data
51
+
* read if possible.
52
+
*/
53
+
void fsverity_readahead(struct fsverity_info *vi, pgoff_t index,
54
+
unsigned long nr_pages)
55
+
{
56
+
struct inode *inode = vi->inode;
57
+
const struct merkle_tree_params *params = &vi->tree_params;
58
+
u64 start_hidx = (u64)index << params->log_blocks_per_page;
59
+
u64 end_hidx =
60
+
(((u64)index + nr_pages) << params->log_blocks_per_page) - 1;
61
+
int level;
62
+
63
+
if (!inode->i_sb->s_vop->readahead_merkle_tree)
64
+
return;
65
+
66
+
for (level = 0; level < params->num_levels; level++) {
67
+
unsigned long level_start = params->level_start[level];
68
+
unsigned long next_start_hidx = start_hidx >> params->log_arity;
69
+
unsigned long next_end_hidx = end_hidx >> params->log_arity;
70
+
pgoff_t start_idx = (level_start + next_start_hidx) >>
71
+
params->log_blocks_per_page;
72
+
pgoff_t end_idx = (level_start + next_end_hidx) >>
73
+
params->log_blocks_per_page;
74
+
75
+
inode->i_sb->s_vop->readahead_merkle_tree(
76
+
inode, start_idx, end_idx - start_idx + 1);
77
+
78
+
start_hidx = next_start_hidx;
79
+
end_hidx = next_end_hidx;
80
+
}
81
+
}
82
+
EXPORT_SYMBOL_GPL(fsverity_readahead);
37
83
38
84
/*
39
85
* Returns true if the hash block with index @hblock_idx in the tree, located in
···
155
113
*
156
114
* Return: %true if the data block is valid, else %false.
157
115
*/
158
-
static bool verify_data_block(struct inode *inode, struct fsverity_info *vi,
159
-
const struct fsverity_pending_block *dblock,
160
-
unsigned long max_ra_pages)
116
+
static bool verify_data_block(struct fsverity_info *vi,
117
+
const struct fsverity_pending_block *dblock)
161
118
{
119
+
struct inode *inode = vi->inode;
162
120
const u64 data_pos = dblock->pos;
163
121
const struct merkle_tree_params *params = &vi->tree_params;
164
122
const unsigned int hsize = params->digest_size;
···
242
200
(params->block_size - 1);
243
201
244
202
hpage = inode->i_sb->s_vop->read_merkle_tree_page(inode,
245
-
hpage_idx, level == 0 ? min(max_ra_pages,
246
-
params->tree_pages - hpage_idx) : 0);
203
+
hpage_idx);
247
204
if (IS_ERR(hpage)) {
248
205
fsverity_err(inode,
249
206
"Error %ld reading Merkle tree page %lu",
···
313
272
314
273
static void
315
274
fsverity_init_verification_context(struct fsverity_verification_context *ctx,
316
-
struct inode *inode,
317
-
unsigned long max_ra_pages)
275
+
struct fsverity_info *vi)
318
276
{
319
-
struct fsverity_info *vi = *fsverity_info_addr(inode);
320
-
321
-
ctx->inode = inode;
322
277
ctx->vi = vi;
323
-
ctx->max_ra_pages = max_ra_pages;
324
278
ctx->num_pending = 0;
325
279
if (vi->tree_params.hash_alg->algo_id == HASH_ALGO_SHA256 &&
326
280
sha256_finup_2x_is_optimized())
···
358
322
}
359
323
360
324
for (i = 0; i < ctx->num_pending; i++) {
361
-
if (!verify_data_block(ctx->inode, vi, &ctx->pending_blocks[i],
362
-
ctx->max_ra_pages))
325
+
if (!verify_data_block(vi, &ctx->pending_blocks[i]))
363
326
return false;
364
327
}
365
328
fsverity_clear_pending_blocks(ctx);
···
394
359
395
360
/**
396
361
* fsverity_verify_blocks() - verify data in a folio
362
+
* @vi: fsverity_info for the inode to be read
397
363
* @folio: the folio containing the data to verify
398
364
* @len: the length of the data to verify in the folio
399
365
* @offset: the offset of the data to verify in the folio
···
405
369
*
406
370
* Return: %true if the data is valid, else %false.
407
371
*/
408
-
bool fsverity_verify_blocks(struct folio *folio, size_t len, size_t offset)
372
+
bool fsverity_verify_blocks(struct fsverity_info *vi, struct folio *folio,
373
+
size_t len, size_t offset)
409
374
{
410
375
struct fsverity_verification_context ctx;
411
376
412
-
fsverity_init_verification_context(&ctx, folio->mapping->host, 0);
377
+
fsverity_init_verification_context(&ctx, vi);
413
378
414
379
if (fsverity_add_data_blocks(&ctx, folio, len, offset) &&
415
380
fsverity_verify_pending_blocks(&ctx))
···
423
386
#ifdef CONFIG_BLOCK
424
387
/**
425
388
* fsverity_verify_bio() - verify a 'read' bio that has just completed
389
+
* @vi: fsverity_info for the inode to be read
426
390
* @bio: the bio to verify
427
391
*
428
392
* Verify the bio's data against the file's Merkle tree. All bio data segments
···
436
398
* filesystems) must instead call fsverity_verify_page() directly on each page.
437
399
* All filesystems must also call fsverity_verify_page() on holes.
438
400
*/
439
-
void fsverity_verify_bio(struct bio *bio)
401
+
void fsverity_verify_bio(struct fsverity_info *vi, struct bio *bio)
440
402
{
441
-
struct inode *inode = bio_first_folio_all(bio)->mapping->host;
442
403
struct fsverity_verification_context ctx;
443
404
struct folio_iter fi;
444
-
unsigned long max_ra_pages = 0;
445
405
446
-
if (bio->bi_opf & REQ_RAHEAD) {
447
-
/*
448
-
* If this bio is for data readahead, then we also do readahead
449
-
* of the first (largest) level of the Merkle tree. Namely,
450
-
* when a Merkle tree page is read, we also try to piggy-back on
451
-
* some additional pages -- up to 1/4 the number of data pages.
452
-
*
453
-
* This improves sequential read performance, as it greatly
454
-
* reduces the number of I/O requests made to the Merkle tree.
455
-
*/
456
-
max_ra_pages = bio->bi_iter.bi_size >> (PAGE_SHIFT + 2);
457
-
}
458
-
459
-
fsverity_init_verification_context(&ctx, inode, max_ra_pages);
406
+
fsverity_init_verification_context(&ctx, vi);
460
407
461
408
bio_for_each_folio_all(fi, bio) {
462
409
if (!fsverity_add_data_blocks(&ctx, fi.folio, fi.length,
+75
-115
include/linux/fsverity.h
+75
-115
include/linux/fsverity.h
···
31
31
/* Verity operations for filesystems */
32
32
struct fsverity_operations {
33
33
/**
34
-
* The offset of the pointer to struct fsverity_info in the
35
-
* filesystem-specific part of the inode, relative to the beginning of
36
-
* the common part of the inode (the 'struct inode').
37
-
*/
38
-
ptrdiff_t inode_info_offs;
39
-
40
-
/**
41
34
* Begin enabling verity on the given file.
42
35
*
43
36
* @filp: a readonly file descriptor for the file
···
90
97
*
91
98
* @inode: the inode
92
99
* @index: 0-based index of the page within the Merkle tree
93
-
* @num_ra_pages: The number of Merkle tree pages that should be
94
-
* prefetched starting at @index if the page at @index
95
-
* isn't already cached. Implementations may ignore this
96
-
* argument; it's only a performance optimization.
97
100
*
98
101
* This can be called at any time on an open verity file. It may be
99
102
* called by multiple processes concurrently, even with the same page.
···
99
110
* Return: the page on success, ERR_PTR() on failure
100
111
*/
101
112
struct page *(*read_merkle_tree_page)(struct inode *inode,
102
-
pgoff_t index,
103
-
unsigned long num_ra_pages);
113
+
pgoff_t index);
104
114
105
115
/**
106
-
* Write a Merkle tree block to the given inode.
116
+
* Perform readahead of a Merkle tree for the given inode.
107
117
*
108
-
* @inode: the inode for which the Merkle tree is being built
118
+
* @inode: the inode
119
+
* @index: 0-based index of the first page within the Merkle tree
120
+
* @nr_pages: number of pages to be read ahead.
121
+
*
122
+
* This can be called at any time on an open verity file. It may be
123
+
* called by multiple processes concurrently, even with the same range.
124
+
*
125
+
* Optional method so that ->read_merkle_tree_page preferably finds
126
+
* cached data instead of issuing dependent I/O.
127
+
*/
128
+
void (*readahead_merkle_tree)(struct inode *inode, pgoff_t index,
129
+
unsigned long nr_pages);
130
+
131
+
/**
132
+
* Write a Merkle tree block to the given file.
133
+
*
134
+
* @file: the file for which the Merkle tree is being built
109
135
* @buf: the Merkle tree block to write
110
136
* @pos: the position of the block in the Merkle tree (in bytes)
111
137
* @size: the Merkle tree block size (in bytes)
···
130
126
*
131
127
* Return: 0 on success, -errno on failure
132
128
*/
133
-
int (*write_merkle_tree_block)(struct inode *inode, const void *buf,
129
+
int (*write_merkle_tree_block)(struct file *file, const void *buf,
134
130
u64 pos, unsigned int size);
135
131
};
136
132
137
133
#ifdef CONFIG_FS_VERITY
138
-
139
-
/*
140
-
* Returns the address of the verity info pointer within the filesystem-specific
141
-
* part of the inode. (To save memory on filesystems that don't support
142
-
* fsverity, a field in 'struct inode' itself is no longer used.)
134
+
/**
135
+
* fsverity_active() - do reads from the inode need to go through fs-verity?
136
+
* @inode: inode to check
137
+
*
138
+
* This checks whether the inode's verity info has been set, and reads need
139
+
* to verify the file data.
140
+
*
141
+
* Return: true if reads need to go through fs-verity, otherwise false
143
142
*/
144
-
static inline struct fsverity_info **
145
-
fsverity_info_addr(const struct inode *inode)
143
+
static inline bool fsverity_active(const struct inode *inode)
146
144
{
147
-
VFS_WARN_ON_ONCE(inode->i_sb->s_vop->inode_info_offs == 0);
148
-
return (void *)inode + inode->i_sb->s_vop->inode_info_offs;
145
+
if (IS_VERITY(inode)) {
146
+
/*
147
+
* This pairs with the try_cmpxchg in set_mask_bits()
148
+
* used to set the S_VERITY bit in i_flags.
149
+
*/
150
+
smp_mb();
151
+
return true;
152
+
}
153
+
154
+
return false;
149
155
}
150
156
157
+
struct fsverity_info *__fsverity_get_info(const struct inode *inode);
158
+
/**
159
+
* fsverity_get_info - get fsverity information for an inode
160
+
* @inode: inode to operate on.
161
+
*
162
+
* This gets the fsverity_info for @inode if it exists. Safe to call without
163
+
* knowin that a fsverity_info exist for @inode, including on file systems that
164
+
* do not support fsverity.
165
+
*/
151
166
static inline struct fsverity_info *fsverity_get_info(const struct inode *inode)
152
167
{
153
-
/*
154
-
* Since this function can be called on inodes belonging to filesystems
155
-
* that don't support fsverity at all, and fsverity_info_addr() doesn't
156
-
* work on such filesystems, we have to start with an IS_VERITY() check.
157
-
* Checking IS_VERITY() here is also useful to minimize the overhead of
158
-
* fsverity_active() on non-verity files.
159
-
*/
160
-
if (!IS_VERITY(inode))
168
+
if (!fsverity_active(inode))
161
169
return NULL;
162
-
163
-
/*
164
-
* Pairs with the cmpxchg_release() in fsverity_set_info(). I.e.,
165
-
* another task may publish the inode's verity info concurrently,
166
-
* executing a RELEASE barrier. Use smp_load_acquire() here to safely
167
-
* ACQUIRE the memory the other task published.
168
-
*/
169
-
return smp_load_acquire(fsverity_info_addr(inode));
170
+
return __fsverity_get_info(inode);
170
171
}
171
172
172
173
/* enable.c */
···
188
179
/* open.c */
189
180
190
181
int __fsverity_file_open(struct inode *inode, struct file *filp);
191
-
int __fsverity_prepare_setattr(struct dentry *dentry, struct iattr *attr);
192
-
void __fsverity_cleanup_inode(struct inode *inode);
193
-
194
-
/**
195
-
* fsverity_cleanup_inode() - free the inode's verity info, if present
196
-
* @inode: an inode being evicted
197
-
*
198
-
* Filesystems must call this on inode eviction to free the inode's verity info.
199
-
*/
200
-
static inline void fsverity_cleanup_inode(struct inode *inode)
201
-
{
202
-
/*
203
-
* Only IS_VERITY() inodes can have verity info, so start by checking
204
-
* for IS_VERITY() (which is faster than retrieving the pointer to the
205
-
* verity info). This minimizes overhead for non-verity inodes.
206
-
*/
207
-
if (IS_VERITY(inode))
208
-
__fsverity_cleanup_inode(inode);
209
-
else
210
-
VFS_WARN_ON_ONCE(*fsverity_info_addr(inode) != NULL);
211
-
}
212
182
213
183
/* read_metadata.c */
214
184
···
195
207
196
208
/* verify.c */
197
209
198
-
bool fsverity_verify_blocks(struct folio *folio, size_t len, size_t offset);
199
-
void fsverity_verify_bio(struct bio *bio);
210
+
bool fsverity_verify_blocks(struct fsverity_info *vi, struct folio *folio,
211
+
size_t len, size_t offset);
212
+
void fsverity_verify_bio(struct fsverity_info *vi, struct bio *bio);
200
213
void fsverity_enqueue_verify_work(struct work_struct *work);
201
214
202
215
#else /* !CONFIG_FS_VERITY */
216
+
217
+
static inline bool fsverity_active(const struct inode *inode)
218
+
{
219
+
return false;
220
+
}
203
221
204
222
static inline struct fsverity_info *fsverity_get_info(const struct inode *inode)
205
223
{
···
245
251
return -EOPNOTSUPP;
246
252
}
247
253
248
-
static inline int __fsverity_prepare_setattr(struct dentry *dentry,
249
-
struct iattr *attr)
250
-
{
251
-
return -EOPNOTSUPP;
252
-
}
253
-
254
-
static inline void fsverity_cleanup_inode(struct inode *inode)
255
-
{
256
-
}
257
-
258
254
/* read_metadata.c */
259
255
260
256
static inline int fsverity_ioctl_read_metadata(struct file *filp,
···
255
271
256
272
/* verify.c */
257
273
258
-
static inline bool fsverity_verify_blocks(struct folio *folio, size_t len,
274
+
static inline bool fsverity_verify_blocks(struct fsverity_info *vi,
275
+
struct folio *folio, size_t len,
259
276
size_t offset)
260
277
{
261
278
WARN_ON_ONCE(1);
262
279
return false;
263
280
}
264
281
265
-
static inline void fsverity_verify_bio(struct bio *bio)
282
+
static inline void fsverity_verify_bio(struct fsverity_info *vi,
283
+
struct bio *bio)
266
284
{
267
285
WARN_ON_ONCE(1);
268
286
}
···
276
290
277
291
#endif /* !CONFIG_FS_VERITY */
278
292
279
-
static inline bool fsverity_verify_folio(struct folio *folio)
293
+
static inline bool fsverity_verify_folio(struct fsverity_info *vi,
294
+
struct folio *folio)
280
295
{
281
-
return fsverity_verify_blocks(folio, folio_size(folio), 0);
296
+
return fsverity_verify_blocks(vi, folio, folio_size(folio), 0);
282
297
}
283
298
284
-
static inline bool fsverity_verify_page(struct page *page)
299
+
static inline bool fsverity_verify_page(struct fsverity_info *vi,
300
+
struct page *page)
285
301
{
286
-
return fsverity_verify_blocks(page_folio(page), PAGE_SIZE, 0);
287
-
}
288
-
289
-
/**
290
-
* fsverity_active() - do reads from the inode need to go through fs-verity?
291
-
* @inode: inode to check
292
-
*
293
-
* This checks whether the inode's verity info has been set.
294
-
*
295
-
* Filesystems call this from ->readahead() to check whether the pages need to
296
-
* be verified or not. Don't use IS_VERITY() for this purpose; it's subject to
297
-
* a race condition where the file is being read concurrently with
298
-
* FS_IOC_ENABLE_VERITY completing. (S_VERITY is set before the verity info.)
299
-
*
300
-
* Return: true if reads need to go through fs-verity, otherwise false
301
-
*/
302
-
static inline bool fsverity_active(const struct inode *inode)
303
-
{
304
-
return fsverity_get_info(inode) != NULL;
302
+
return fsverity_verify_blocks(vi, page_folio(page), PAGE_SIZE, 0);
305
303
}
306
304
307
305
/**
···
308
338
return 0;
309
339
}
310
340
311
-
/**
312
-
* fsverity_prepare_setattr() - prepare to change a verity inode's attributes
313
-
* @dentry: dentry through which the inode is being changed
314
-
* @attr: attributes to change
315
-
*
316
-
* Verity files are immutable, so deny truncates. This isn't covered by the
317
-
* open-time check because sys_truncate() takes a path, not a file descriptor.
318
-
*
319
-
* Return: 0 on success, -errno on failure
320
-
*/
321
-
static inline int fsverity_prepare_setattr(struct dentry *dentry,
322
-
struct iattr *attr)
323
-
{
324
-
if (IS_VERITY(d_inode(dentry)))
325
-
return __fsverity_prepare_setattr(dentry, attr);
326
-
return 0;
327
-
}
341
+
void fsverity_cleanup_inode(struct inode *inode);
342
+
void fsverity_readahead(struct fsverity_info *vi, pgoff_t index,
343
+
unsigned long nr_pages);
344
+
345
+
struct page *generic_read_merkle_tree_page(struct inode *inode, pgoff_t index);
346
+
void generic_readahead_merkle_tree(struct inode *inode, pgoff_t index,
347
+
unsigned long nr_pages);
328
348
329
349
#endif /* _LINUX_FSVERITY_H */
+9
-6
mm/readahead.c
+9
-6
mm/readahead.c
···
204
204
* not the function you want to call. Use page_cache_async_readahead()
205
205
* or page_cache_sync_readahead() instead.
206
206
*
207
-
* Context: File is referenced by caller. Mutexes may be held by caller.
208
-
* May sleep, but will not reenter filesystem to reclaim memory.
207
+
* Context: File is referenced by caller, and ractl->mapping->invalidate_lock
208
+
* must be held by the caller at least in shared mode. Mutexes may be held by
209
+
* caller. May sleep, but will not reenter filesystem to reclaim memory.
209
210
*/
210
211
void page_cache_ra_unbounded(struct readahead_control *ractl,
211
212
unsigned long nr_to_read, unsigned long lookahead_size)
···
229
228
*/
230
229
unsigned int nofs = memalloc_nofs_save();
231
230
231
+
lockdep_assert_held(&mapping->invalidate_lock);
232
+
232
233
trace_page_cache_ra_unbounded(mapping->host, index, nr_to_read,
233
234
lookahead_size);
234
-
filemap_invalidate_lock_shared(mapping);
235
235
index = mapping_align_index(mapping, index);
236
236
237
237
/*
···
302
300
* will then handle the error.
303
301
*/
304
302
read_pages(ractl);
305
-
filemap_invalidate_unlock_shared(mapping);
306
303
memalloc_nofs_restore(nofs);
307
304
}
308
305
EXPORT_SYMBOL_GPL(page_cache_ra_unbounded);
···
315
314
static void do_page_cache_ra(struct readahead_control *ractl,
316
315
unsigned long nr_to_read, unsigned long lookahead_size)
317
316
{
318
-
struct inode *inode = ractl->mapping->host;
317
+
struct address_space *mapping = ractl->mapping;
319
318
unsigned long index = readahead_index(ractl);
320
-
loff_t isize = i_size_read(inode);
319
+
loff_t isize = i_size_read(mapping->host);
321
320
pgoff_t end_index; /* The last page we want to read */
322
321
323
322
if (isize == 0)
···
330
329
if (nr_to_read > end_index - index)
331
330
nr_to_read = end_index - index + 1;
332
331
332
+
filemap_invalidate_lock_shared(mapping);
333
333
page_cache_ra_unbounded(ractl, nr_to_read, lookahead_size);
334
+
filemap_invalidate_unlock_shared(mapping);
334
335
}
335
336
336
337
/*