Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge patch series "fuse: use iomap for buffered reads + readahead"
Joanne Koong <joannelkoong@gmail.com> says:
This series adds fuse iomap support for buffered reads and readahead.
This is needed so that granular uptodate tracking can be used in fuse when
large folios are enabled so that only the non-uptodate portions of the folio
need to be read in instead of having to read in the entire folio. It also is
needed in order to turn on large folios for servers that use the writeback
cache since otherwise there is a race condition that may lead to data
corruption if there is a partial write, then a read and the read happens
before the write has undergone writeback, since otherwise the folio will not
be marked uptodate from the partial write so the read will read in the entire
folio from disk, which will overwrite the partial write.
This is on top of two locally-patched iomap patches [1] [2] patched on top of
commit f1c864be6e88 ("Merge branch 'vfs-6.18.async' into vfs.all") in
Christian's vfs.all tree.
This series was run through fstests on fuse passthrough_hp with an
out-of kernel patch enabling fuse large folios.
This patchset does not enable large folios on fuse yet. That will be part
of a different patchset.
* patches from https://lore.kernel.org/20250926002609.1302233-1-joannelkoong@gmail.com:
fuse: remove fc->blkbits workaround for partial writes
fuse: use iomap for readahead
fuse: use iomap for read_folio
iomap: make iomap_read_folio() a void return
iomap: move buffered io bio logic into new file
iomap: add caller-provided callbacks for read and readahead
iomap: set accurate iter->pos when reading folio ranges
iomap: track pending read bytes more optimally
iomap: rename iomap_readpage_ctx struct to iomap_read_folio_ctx
iomap: rename iomap_readpage_iter() to iomap_read_folio_iter()
iomap: iterate over folio mapping in iomap_readpage_iter()
iomap: store read/readahead bio generically
iomap: move read/readahead bio submission logic into helper function
iomap: move bio read logic into helper function
Signed-off-by: Christian Brauner <brauner@kernel.org>
+544
-291
+44
Documentation/filesystems/iomap/operations.rst
+44
Documentation/filesystems/iomap/operations.rst
···
135
135
136
136
* ``IOCB_DONTCACHE``: Turns on ``IOMAP_DONTCACHE``.
137
137
138
+
``struct iomap_read_ops``
139
+
--------------------------
140
+
141
+
.. code-block:: c
142
+
143
+
struct iomap_read_ops {
144
+
int (*read_folio_range)(const struct iomap_iter *iter,
145
+
struct iomap_read_folio_ctx *ctx, size_t len);
146
+
void (*submit_read)(struct iomap_read_folio_ctx *ctx);
147
+
};
148
+
149
+
iomap calls these functions:
150
+
151
+
- ``read_folio_range``: Called to read in the range. This must be provided
152
+
by the caller. The caller is responsible for calling
153
+
iomap_finish_folio_read() after reading in the folio range. This should be
154
+
done even if an error is encountered during the read. This returns 0 on
155
+
success or a negative error on failure.
156
+
157
+
- ``submit_read``: Submit any pending read requests. This function is
158
+
optional.
159
+
138
160
Internal per-Folio State
139
161
------------------------
140
162
···
203
181
The ``flags`` argument to ``->iomap_begin`` will be set to zero.
204
182
The pagecache takes whatever locks it needs before calling the
205
183
filesystem.
184
+
185
+
Both ``iomap_readahead`` and ``iomap_read_folio`` pass in a ``struct
186
+
iomap_read_folio_ctx``:
187
+
188
+
.. code-block:: c
189
+
190
+
struct iomap_read_folio_ctx {
191
+
const struct iomap_read_ops *ops;
192
+
struct folio *cur_folio;
193
+
struct readahead_control *rac;
194
+
void *read_ctx;
195
+
};
196
+
197
+
``iomap_readahead`` must set:
198
+
* ``ops->read_folio_range()`` and ``rac``
199
+
200
+
``iomap_read_folio`` must set:
201
+
* ``ops->read_folio_range()`` and ``cur_folio``
202
+
203
+
``ops->submit_read()`` and ``read_ctx`` are optional. ``read_ctx`` is used to
204
+
pass in any custom data the caller needs accessible in the ops callbacks for
205
+
fulfilling reads.
206
206
207
207
Buffered Writes
208
208
---------------
+3
-2
block/fops.c
+3
-2
block/fops.c
···
540
540
#else /* CONFIG_BUFFER_HEAD */
541
541
static int blkdev_read_folio(struct file *file, struct folio *folio)
542
542
{
543
-
return iomap_read_folio(folio, &blkdev_iomap_ops);
543
+
iomap_bio_read_folio(folio, &blkdev_iomap_ops);
544
+
return 0;
544
545
}
545
546
546
547
static void blkdev_readahead(struct readahead_control *rac)
547
548
{
548
-
iomap_readahead(rac, &blkdev_iomap_ops);
549
+
iomap_bio_readahead(rac, &blkdev_iomap_ops);
549
550
}
550
551
551
552
static ssize_t blkdev_writeback_range(struct iomap_writepage_ctx *wpc,
+3
-2
fs/erofs/data.c
+3
-2
fs/erofs/data.c
···
371
371
{
372
372
trace_erofs_read_folio(folio, true);
373
373
374
-
return iomap_read_folio(folio, &erofs_iomap_ops);
374
+
iomap_bio_read_folio(folio, &erofs_iomap_ops);
375
+
return 0;
375
376
}
376
377
377
378
static void erofs_readahead(struct readahead_control *rac)
···
380
379
trace_erofs_readahead(rac->mapping->host, readahead_index(rac),
381
380
readahead_count(rac), true);
382
381
383
-
return iomap_readahead(rac, &erofs_iomap_ops);
382
+
iomap_bio_readahead(rac, &erofs_iomap_ops);
384
383
}
385
384
386
385
static sector_t erofs_bmap(struct address_space *mapping, sector_t block)
+1
-1
fs/fuse/dir.c
+1
-1
fs/fuse/dir.c
+174
-114
fs/fuse/file.c
+174
-114
fs/fuse/file.c
···
834
834
return 0;
835
835
}
836
836
837
+
static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
838
+
unsigned int flags, struct iomap *iomap,
839
+
struct iomap *srcmap)
840
+
{
841
+
iomap->type = IOMAP_MAPPED;
842
+
iomap->length = length;
843
+
iomap->offset = offset;
844
+
return 0;
845
+
}
846
+
847
+
static const struct iomap_ops fuse_iomap_ops = {
848
+
.iomap_begin = fuse_iomap_begin,
849
+
};
850
+
851
+
struct fuse_fill_read_data {
852
+
struct file *file;
853
+
854
+
/* Fields below are used if sending the read request asynchronously */
855
+
struct fuse_conn *fc;
856
+
struct fuse_io_args *ia;
857
+
unsigned int nr_bytes;
858
+
};
859
+
860
+
/* forward declarations */
861
+
static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos,
862
+
unsigned len, struct fuse_args_pages *ap,
863
+
unsigned cur_bytes, bool write);
864
+
static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file,
865
+
unsigned int count, bool async);
866
+
867
+
static int fuse_handle_readahead(struct folio *folio,
868
+
struct readahead_control *rac,
869
+
struct fuse_fill_read_data *data, loff_t pos,
870
+
size_t len)
871
+
{
872
+
struct fuse_io_args *ia = data->ia;
873
+
size_t off = offset_in_folio(folio, pos);
874
+
struct fuse_conn *fc = data->fc;
875
+
struct fuse_args_pages *ap;
876
+
unsigned int nr_pages;
877
+
878
+
if (ia && fuse_folios_need_send(fc, pos, len, &ia->ap, data->nr_bytes,
879
+
false)) {
880
+
fuse_send_readpages(ia, data->file, data->nr_bytes,
881
+
fc->async_read);
882
+
data->nr_bytes = 0;
883
+
data->ia = NULL;
884
+
ia = NULL;
885
+
}
886
+
if (!ia) {
887
+
if (fc->num_background >= fc->congestion_threshold &&
888
+
rac->ra->async_size >= readahead_count(rac))
889
+
/*
890
+
* Congested and only async pages left, so skip the
891
+
* rest.
892
+
*/
893
+
return -EAGAIN;
894
+
895
+
nr_pages = min(fc->max_pages, readahead_count(rac));
896
+
data->ia = fuse_io_alloc(NULL, nr_pages);
897
+
if (!data->ia)
898
+
return -ENOMEM;
899
+
ia = data->ia;
900
+
}
901
+
folio_get(folio);
902
+
ap = &ia->ap;
903
+
ap->folios[ap->num_folios] = folio;
904
+
ap->descs[ap->num_folios].offset = off;
905
+
ap->descs[ap->num_folios].length = len;
906
+
data->nr_bytes += len;
907
+
ap->num_folios++;
908
+
909
+
return 0;
910
+
}
911
+
912
+
static int fuse_iomap_read_folio_range_async(const struct iomap_iter *iter,
913
+
struct iomap_read_folio_ctx *ctx,
914
+
size_t len)
915
+
{
916
+
struct fuse_fill_read_data *data = ctx->read_ctx;
917
+
struct folio *folio = ctx->cur_folio;
918
+
loff_t pos = iter->pos;
919
+
size_t off = offset_in_folio(folio, pos);
920
+
struct file *file = data->file;
921
+
int ret;
922
+
923
+
if (ctx->rac) {
924
+
ret = fuse_handle_readahead(folio, ctx->rac, data, pos, len);
925
+
/*
926
+
* If fuse_handle_readahead was successful, fuse_readpages_end
927
+
* will do the iomap_finish_folio_read, else we need to call it
928
+
* here
929
+
*/
930
+
if (ret)
931
+
iomap_finish_folio_read(folio, off, len, ret);
932
+
} else {
933
+
/*
934
+
* for non-readahead read requests, do reads synchronously
935
+
* since it's not guaranteed that the server can handle
936
+
* out-of-order reads
937
+
*/
938
+
ret = fuse_do_readfolio(file, folio, off, len);
939
+
iomap_finish_folio_read(folio, off, len, ret);
940
+
}
941
+
return ret;
942
+
}
943
+
944
+
static void fuse_iomap_read_submit(struct iomap_read_folio_ctx *ctx)
945
+
{
946
+
struct fuse_fill_read_data *data = ctx->read_ctx;
947
+
948
+
if (data->ia)
949
+
fuse_send_readpages(data->ia, data->file, data->nr_bytes,
950
+
data->fc->async_read);
951
+
}
952
+
953
+
static const struct iomap_read_ops fuse_iomap_read_ops = {
954
+
.read_folio_range = fuse_iomap_read_folio_range_async,
955
+
.submit_read = fuse_iomap_read_submit,
956
+
};
957
+
837
958
static int fuse_read_folio(struct file *file, struct folio *folio)
838
959
{
839
960
struct inode *inode = folio->mapping->host;
840
-
int err;
961
+
struct fuse_fill_read_data data = {
962
+
.file = file,
963
+
};
964
+
struct iomap_read_folio_ctx ctx = {
965
+
.cur_folio = folio,
966
+
.ops = &fuse_iomap_read_ops,
967
+
.read_ctx = &data,
841
968
842
-
err = -EIO;
843
-
if (fuse_is_bad(inode))
844
-
goto out;
969
+
};
845
970
846
-
err = fuse_do_readfolio(file, folio, 0, folio_size(folio));
847
-
if (!err)
848
-
folio_mark_uptodate(folio);
971
+
if (fuse_is_bad(inode)) {
972
+
folio_unlock(folio);
973
+
return -EIO;
974
+
}
849
975
976
+
iomap_read_folio(&fuse_iomap_ops, &ctx);
850
977
fuse_invalidate_atime(inode);
851
-
out:
852
-
folio_unlock(folio);
853
-
return err;
978
+
return 0;
854
979
}
855
980
856
981
static int fuse_iomap_read_folio_range(const struct iomap_iter *iter,
···
1012
887
fuse_invalidate_atime(inode);
1013
888
1014
889
for (i = 0; i < ap->num_folios; i++) {
1015
-
folio_end_read(ap->folios[i], !err);
890
+
iomap_finish_folio_read(ap->folios[i], ap->descs[i].offset,
891
+
ap->descs[i].length, err);
1016
892
folio_put(ap->folios[i]);
1017
893
}
1018
894
if (ia->ff)
···
1023
897
}
1024
898
1025
899
static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file,
1026
-
unsigned int count)
900
+
unsigned int count, bool async)
1027
901
{
1028
902
struct fuse_file *ff = file->private_data;
1029
903
struct fuse_mount *fm = ff->fm;
···
1045
919
1046
920
fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
1047
921
ia->read.attr_ver = fuse_get_attr_version(fm->fc);
1048
-
if (fm->fc->async_read) {
922
+
if (async) {
1049
923
ia->ff = fuse_file_get(ff);
1050
924
ap->args.end = fuse_readpages_end;
1051
925
err = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
···
1062
936
{
1063
937
struct inode *inode = rac->mapping->host;
1064
938
struct fuse_conn *fc = get_fuse_conn(inode);
1065
-
unsigned int max_pages, nr_pages;
1066
-
struct folio *folio = NULL;
939
+
struct fuse_fill_read_data data = {
940
+
.file = rac->file,
941
+
.fc = fc,
942
+
};
943
+
struct iomap_read_folio_ctx ctx = {
944
+
.ops = &fuse_iomap_read_ops,
945
+
.rac = rac,
946
+
.read_ctx = &data
947
+
};
1067
948
1068
949
if (fuse_is_bad(inode))
1069
950
return;
1070
951
1071
-
max_pages = min_t(unsigned int, fc->max_pages,
1072
-
fc->max_read / PAGE_SIZE);
1073
-
1074
-
/*
1075
-
* This is only accurate the first time through, since readahead_folio()
1076
-
* doesn't update readahead_count() from the previous folio until the
1077
-
* next call. Grab nr_pages here so we know how many pages we're going
1078
-
* to have to process. This means that we will exit here with
1079
-
* readahead_count() == folio_nr_pages(last_folio), but we will have
1080
-
* consumed all of the folios, and read_pages() will call
1081
-
* readahead_folio() again which will clean up the rac.
1082
-
*/
1083
-
nr_pages = readahead_count(rac);
1084
-
1085
-
while (nr_pages) {
1086
-
struct fuse_io_args *ia;
1087
-
struct fuse_args_pages *ap;
1088
-
unsigned cur_pages = min(max_pages, nr_pages);
1089
-
unsigned int pages = 0;
1090
-
1091
-
if (fc->num_background >= fc->congestion_threshold &&
1092
-
rac->ra->async_size >= readahead_count(rac))
1093
-
/*
1094
-
* Congested and only async pages left, so skip the
1095
-
* rest.
1096
-
*/
1097
-
break;
1098
-
1099
-
ia = fuse_io_alloc(NULL, cur_pages);
1100
-
if (!ia)
1101
-
break;
1102
-
ap = &ia->ap;
1103
-
1104
-
while (pages < cur_pages) {
1105
-
unsigned int folio_pages;
1106
-
1107
-
/*
1108
-
* This returns a folio with a ref held on it.
1109
-
* The ref needs to be held until the request is
1110
-
* completed, since the splice case (see
1111
-
* fuse_try_move_page()) drops the ref after it's
1112
-
* replaced in the page cache.
1113
-
*/
1114
-
if (!folio)
1115
-
folio = __readahead_folio(rac);
1116
-
1117
-
folio_pages = folio_nr_pages(folio);
1118
-
if (folio_pages > cur_pages - pages) {
1119
-
/*
1120
-
* Large folios belonging to fuse will never
1121
-
* have more pages than max_pages.
1122
-
*/
1123
-
WARN_ON(!pages);
1124
-
break;
1125
-
}
1126
-
1127
-
ap->folios[ap->num_folios] = folio;
1128
-
ap->descs[ap->num_folios].length = folio_size(folio);
1129
-
ap->num_folios++;
1130
-
pages += folio_pages;
1131
-
folio = NULL;
1132
-
}
1133
-
fuse_send_readpages(ia, rac->file, pages << PAGE_SHIFT);
1134
-
nr_pages -= pages;
1135
-
}
1136
-
if (folio) {
1137
-
folio_end_read(folio, false);
1138
-
folio_put(folio);
1139
-
}
952
+
iomap_readahead(&fuse_iomap_ops, &ctx);
1140
953
}
1141
954
1142
955
static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
···
1460
1395
1461
1396
static const struct iomap_write_ops fuse_iomap_write_ops = {
1462
1397
.read_folio_range = fuse_iomap_read_folio_range,
1463
-
};
1464
-
1465
-
static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
1466
-
unsigned int flags, struct iomap *iomap,
1467
-
struct iomap *srcmap)
1468
-
{
1469
-
iomap->type = IOMAP_MAPPED;
1470
-
iomap->length = length;
1471
-
iomap->offset = offset;
1472
-
return 0;
1473
-
}
1474
-
1475
-
static const struct iomap_ops fuse_iomap_ops = {
1476
-
.iomap_begin = fuse_iomap_begin,
1477
1398
};
1478
1399
1479
1400
static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
···
2098
2047
struct fuse_file *ff;
2099
2048
unsigned int max_folios;
2100
2049
/*
2101
-
* nr_bytes won't overflow since fuse_writepage_need_send() caps
2050
+
* nr_bytes won't overflow since fuse_folios_need_send() caps
2102
2051
* wb requests to never exceed fc->max_pages (which has an upper bound
2103
2052
* of U16_MAX).
2104
2053
*/
···
2143
2092
spin_unlock(&fi->lock);
2144
2093
}
2145
2094
2146
-
static bool fuse_writepage_need_send(struct fuse_conn *fc, loff_t pos,
2147
-
unsigned len, struct fuse_args_pages *ap,
2148
-
struct fuse_fill_wb_data *data)
2095
+
static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos,
2096
+
unsigned len, struct fuse_args_pages *ap,
2097
+
unsigned cur_bytes, bool write)
2149
2098
{
2150
2099
struct folio *prev_folio;
2151
2100
struct fuse_folio_desc prev_desc;
2152
-
unsigned bytes = data->nr_bytes + len;
2101
+
unsigned bytes = cur_bytes + len;
2153
2102
loff_t prev_pos;
2103
+
size_t max_bytes = write ? fc->max_write : fc->max_read;
2154
2104
2155
2105
WARN_ON(!ap->num_folios);
2156
2106
···
2159
2107
if ((bytes + PAGE_SIZE - 1) >> PAGE_SHIFT > fc->max_pages)
2160
2108
return true;
2161
2109
2162
-
/* Reached max write bytes */
2163
-
if (bytes > fc->max_write)
2110
+
if (bytes > max_bytes)
2164
2111
return true;
2165
2112
2166
2113
/* Discontinuity */
···
2167
2116
prev_desc = ap->descs[ap->num_folios - 1];
2168
2117
prev_pos = folio_pos(prev_folio) + prev_desc.offset + prev_desc.length;
2169
2118
if (prev_pos != pos)
2170
-
return true;
2171
-
2172
-
/* Need to grow the pages array? If so, did the expansion fail? */
2173
-
if (ap->num_folios == data->max_folios &&
2174
-
!fuse_pages_realloc(data, fc->max_pages))
2175
2119
return true;
2176
2120
2177
2121
return false;
···
2192
2146
return -EIO;
2193
2147
}
2194
2148
2195
-
if (wpa && fuse_writepage_need_send(fc, pos, len, ap, data)) {
2196
-
fuse_writepages_send(inode, data);
2197
-
data->wpa = NULL;
2198
-
data->nr_bytes = 0;
2149
+
if (wpa) {
2150
+
bool send = fuse_folios_need_send(fc, pos, len, ap,
2151
+
data->nr_bytes, true);
2152
+
2153
+
if (!send) {
2154
+
/*
2155
+
* Need to grow the pages array? If so, did the
2156
+
* expansion fail?
2157
+
*/
2158
+
send = (ap->num_folios == data->max_folios) &&
2159
+
!fuse_pages_realloc(data, fc->max_pages);
2160
+
}
2161
+
2162
+
if (send) {
2163
+
fuse_writepages_send(inode, data);
2164
+
data->wpa = NULL;
2165
+
data->nr_bytes = 0;
2166
+
}
2199
2167
}
2200
2168
2201
2169
if (data->wpa == NULL) {
-8
fs/fuse/fuse_i.h
-8
fs/fuse/fuse_i.h
···
981
981
/* Request timeout (in jiffies). 0 = no timeout */
982
982
unsigned int req_timeout;
983
983
} timeout;
984
-
985
-
/*
986
-
* This is a workaround until fuse uses iomap for reads.
987
-
* For fuseblk servers, this represents the blocksize passed in at
988
-
* mount time and for regular fuse servers, this is equivalent to
989
-
* inode->i_blkbits.
990
-
*/
991
-
u8 blkbits;
992
984
};
993
985
994
986
/*
+1
-12
fs/fuse/inode.c
+1
-12
fs/fuse/inode.c
···
291
291
if (attr->blksize)
292
292
fi->cached_i_blkbits = ilog2(attr->blksize);
293
293
else
294
-
fi->cached_i_blkbits = fc->blkbits;
294
+
fi->cached_i_blkbits = inode->i_sb->s_blocksize_bits;
295
295
296
296
/*
297
297
* Don't set the sticky bit in i_mode, unless we want the VFS
···
1838
1838
err = -EINVAL;
1839
1839
if (!sb_set_blocksize(sb, ctx->blksize))
1840
1840
goto err;
1841
-
/*
1842
-
* This is a workaround until fuse hooks into iomap for reads.
1843
-
* Use PAGE_SIZE for the blocksize else if the writeback cache
1844
-
* is enabled, buffered writes go through iomap and a read may
1845
-
* overwrite partially written data if blocksize < PAGE_SIZE
1846
-
*/
1847
-
fc->blkbits = sb->s_blocksize_bits;
1848
-
if (ctx->blksize != PAGE_SIZE &&
1849
-
!sb_set_blocksize(sb, PAGE_SIZE))
1850
-
goto err;
1851
1841
#endif
1852
1842
fc->sync_fs = 1;
1853
1843
} else {
1854
1844
sb->s_blocksize = PAGE_SIZE;
1855
1845
sb->s_blocksize_bits = PAGE_SHIFT;
1856
-
fc->blkbits = sb->s_blocksize_bits;
1857
1846
}
1858
1847
1859
1848
sb->s_subtype = ctx->subtype;
+3
-3
fs/gfs2/aops.c
+3
-3
fs/gfs2/aops.c
···
424
424
struct inode *inode = folio->mapping->host;
425
425
struct gfs2_inode *ip = GFS2_I(inode);
426
426
struct gfs2_sbd *sdp = GFS2_SB(inode);
427
-
int error;
427
+
int error = 0;
428
428
429
429
if (!gfs2_is_jdata(ip) ||
430
430
(i_blocksize(inode) == PAGE_SIZE && !folio_buffers(folio))) {
431
-
error = iomap_read_folio(folio, &gfs2_iomap_ops);
431
+
iomap_bio_read_folio(folio, &gfs2_iomap_ops);
432
432
} else if (gfs2_is_stuffed(ip)) {
433
433
error = stuffed_read_folio(ip, folio);
434
434
} else {
···
503
503
else if (gfs2_is_jdata(ip))
504
504
mpage_readahead(rac, gfs2_block_map);
505
505
else
506
-
iomap_readahead(rac, &gfs2_iomap_ops);
506
+
iomap_bio_readahead(rac, &gfs2_iomap_ops);
507
507
}
508
508
509
509
/**
+2
-1
fs/iomap/Makefile
+2
-1
fs/iomap/Makefile
+88
fs/iomap/bio.c
+88
fs/iomap/bio.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* Copyright (C) 2010 Red Hat, Inc.
4
+
* Copyright (C) 2016-2023 Christoph Hellwig.
5
+
*/
6
+
#include <linux/iomap.h>
7
+
#include <linux/pagemap.h>
8
+
#include "internal.h"
9
+
#include "trace.h"
10
+
11
+
static void iomap_read_end_io(struct bio *bio)
12
+
{
13
+
int error = blk_status_to_errno(bio->bi_status);
14
+
struct folio_iter fi;
15
+
16
+
bio_for_each_folio_all(fi, bio)
17
+
iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error);
18
+
bio_put(bio);
19
+
}
20
+
21
+
static void iomap_bio_submit_read(struct iomap_read_folio_ctx *ctx)
22
+
{
23
+
struct bio *bio = ctx->read_ctx;
24
+
25
+
if (bio)
26
+
submit_bio(bio);
27
+
}
28
+
29
+
static int iomap_bio_read_folio_range(const struct iomap_iter *iter,
30
+
struct iomap_read_folio_ctx *ctx, size_t plen)
31
+
{
32
+
struct folio *folio = ctx->cur_folio;
33
+
const struct iomap *iomap = &iter->iomap;
34
+
loff_t pos = iter->pos;
35
+
size_t poff = offset_in_folio(folio, pos);
36
+
loff_t length = iomap_length(iter);
37
+
sector_t sector;
38
+
struct bio *bio = ctx->read_ctx;
39
+
40
+
sector = iomap_sector(iomap, pos);
41
+
if (!bio || bio_end_sector(bio) != sector ||
42
+
!bio_add_folio(bio, folio, plen, poff)) {
43
+
gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
44
+
gfp_t orig_gfp = gfp;
45
+
unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE);
46
+
47
+
if (bio)
48
+
submit_bio(bio);
49
+
50
+
if (ctx->rac) /* same as readahead_gfp_mask */
51
+
gfp |= __GFP_NORETRY | __GFP_NOWARN;
52
+
bio = bio_alloc(iomap->bdev, bio_max_segs(nr_vecs), REQ_OP_READ,
53
+
gfp);
54
+
/*
55
+
* If the bio_alloc fails, try it again for a single page to
56
+
* avoid having to deal with partial page reads. This emulates
57
+
* what do_mpage_read_folio does.
58
+
*/
59
+
if (!bio)
60
+
bio = bio_alloc(iomap->bdev, 1, REQ_OP_READ, orig_gfp);
61
+
if (ctx->rac)
62
+
bio->bi_opf |= REQ_RAHEAD;
63
+
bio->bi_iter.bi_sector = sector;
64
+
bio->bi_end_io = iomap_read_end_io;
65
+
bio_add_folio_nofail(bio, folio, plen, poff);
66
+
ctx->read_ctx = bio;
67
+
}
68
+
return 0;
69
+
}
70
+
71
+
const struct iomap_read_ops iomap_bio_read_ops = {
72
+
.read_folio_range = iomap_bio_read_folio_range,
73
+
.submit_read = iomap_bio_submit_read,
74
+
};
75
+
EXPORT_SYMBOL_GPL(iomap_bio_read_ops);
76
+
77
+
int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter,
78
+
struct folio *folio, loff_t pos, size_t len)
79
+
{
80
+
const struct iomap *srcmap = iomap_iter_srcmap(iter);
81
+
struct bio_vec bvec;
82
+
struct bio bio;
83
+
84
+
bio_init(&bio, srcmap->bdev, &bvec, 1, REQ_OP_READ);
85
+
bio.bi_iter.bi_sector = iomap_sector(srcmap, pos);
86
+
bio_add_folio_nofail(&bio, folio, len, offset_in_folio(folio, pos));
87
+
return submit_bio_wait(&bio);
88
+
}
+146
-142
fs/iomap/buffered-io.c
+146
-142
fs/iomap/buffered-io.c
···
8
8
#include <linux/writeback.h>
9
9
#include <linux/swap.h>
10
10
#include <linux/migrate.h>
11
+
#include "internal.h"
11
12
#include "trace.h"
12
13
13
14
#include "../internal.h"
···
328
327
return 0;
329
328
}
330
329
331
-
#ifdef CONFIG_BLOCK
332
-
static void iomap_finish_folio_read(struct folio *folio, size_t off,
333
-
size_t len, int error)
330
+
void iomap_finish_folio_read(struct folio *folio, size_t off, size_t len,
331
+
int error)
334
332
{
335
333
struct iomap_folio_state *ifs = folio->private;
336
334
bool uptodate = !error;
···
349
349
if (finished)
350
350
folio_end_read(folio, uptodate);
351
351
}
352
+
EXPORT_SYMBOL_GPL(iomap_finish_folio_read);
352
353
353
-
static void iomap_read_end_io(struct bio *bio)
354
+
static void iomap_read_init(struct folio *folio)
354
355
{
355
-
int error = blk_status_to_errno(bio->bi_status);
356
-
struct folio_iter fi;
356
+
struct iomap_folio_state *ifs = folio->private;
357
357
358
-
bio_for_each_folio_all(fi, bio)
359
-
iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error);
360
-
bio_put(bio);
358
+
if (ifs) {
359
+
size_t len = folio_size(folio);
360
+
361
+
/*
362
+
* ifs->read_bytes_pending is used to track how many bytes are
363
+
* read in asynchronously by the IO helper. We need to track
364
+
* this so that we can know when the IO helper has finished
365
+
* reading in all the necessary ranges of the folio and can end
366
+
* the read.
367
+
*
368
+
* Increase ->read_bytes_pending by the folio size to start, and
369
+
* add a +1 bias. We'll subtract the bias and any uptodate /
370
+
* zeroed ranges that did not require IO in iomap_read_end()
371
+
* after we're done processing the folio.
372
+
*
373
+
* We do this because otherwise, we would have to increment
374
+
* ifs->read_bytes_pending every time a range in the folio needs
375
+
* to be read in, which can get expensive since the spinlock
376
+
* needs to be held whenever modifying ifs->read_bytes_pending.
377
+
*
378
+
* We add the bias to ensure the read has not been ended on the
379
+
* folio when iomap_read_end() is called, even if the IO helper
380
+
* has already finished reading in the entire folio.
381
+
*/
382
+
spin_lock_irq(&ifs->state_lock);
383
+
ifs->read_bytes_pending += len + 1;
384
+
spin_unlock_irq(&ifs->state_lock);
385
+
}
361
386
}
362
387
363
-
struct iomap_readpage_ctx {
364
-
struct folio *cur_folio;
365
-
bool cur_folio_in_bio;
366
-
struct bio *bio;
367
-
struct readahead_control *rac;
368
-
};
388
+
/*
389
+
* This ends IO if no bytes were submitted to an IO helper.
390
+
*
391
+
* Otherwise, this calibrates ifs->read_bytes_pending to represent only the
392
+
* submitted bytes (see comment in iomap_read_init()). If all bytes submitted
393
+
* have already been completed by the IO helper, then this will end the read.
394
+
* Else the IO helper will end the read after all submitted ranges have been
395
+
* read.
396
+
*/
397
+
static void iomap_read_end(struct folio *folio, size_t bytes_pending)
398
+
{
399
+
struct iomap_folio_state *ifs;
369
400
370
-
static int iomap_readpage_iter(struct iomap_iter *iter,
371
-
struct iomap_readpage_ctx *ctx)
401
+
/*
402
+
* If there are no bytes pending, this means we are responsible for
403
+
* unlocking the folio here, since no IO helper has taken ownership of
404
+
* it.
405
+
*/
406
+
if (!bytes_pending) {
407
+
folio_unlock(folio);
408
+
return;
409
+
}
410
+
411
+
ifs = folio->private;
412
+
if (ifs) {
413
+
bool end_read, uptodate;
414
+
/*
415
+
* Subtract any bytes that were initially accounted to
416
+
* read_bytes_pending but skipped for IO.
417
+
* The +1 accounts for the bias we added in iomap_read_init().
418
+
*/
419
+
size_t bytes_accounted = folio_size(folio) + 1 -
420
+
bytes_pending;
421
+
422
+
spin_lock_irq(&ifs->state_lock);
423
+
ifs->read_bytes_pending -= bytes_accounted;
424
+
/*
425
+
* If !ifs->read_bytes_pending, this means all pending reads
426
+
* by the IO helper have already completed, which means we need
427
+
* to end the folio read here. If ifs->read_bytes_pending != 0,
428
+
* the IO helper will end the folio read.
429
+
*/
430
+
end_read = !ifs->read_bytes_pending;
431
+
if (end_read)
432
+
uptodate = ifs_is_fully_uptodate(folio, ifs);
433
+
spin_unlock_irq(&ifs->state_lock);
434
+
if (end_read)
435
+
folio_end_read(folio, uptodate);
436
+
}
437
+
}
438
+
439
+
static int iomap_read_folio_iter(struct iomap_iter *iter,
440
+
struct iomap_read_folio_ctx *ctx, size_t *bytes_pending)
372
441
{
373
442
const struct iomap *iomap = &iter->iomap;
374
443
loff_t pos = iter->pos;
375
444
loff_t length = iomap_length(iter);
376
445
struct folio *folio = ctx->cur_folio;
377
-
struct iomap_folio_state *ifs;
378
446
size_t poff, plen;
379
-
sector_t sector;
447
+
loff_t pos_diff;
380
448
int ret;
381
449
382
450
if (iomap->type == IOMAP_INLINE) {
···
454
386
return iomap_iter_advance(iter, length);
455
387
}
456
388
457
-
/* zero post-eof blocks as the page may be mapped */
458
-
ifs = ifs_alloc(iter->inode, folio, iter->flags);
459
-
iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff, &plen);
460
-
if (plen == 0)
461
-
goto done;
389
+
ifs_alloc(iter->inode, folio, iter->flags);
462
390
463
-
if (iomap_block_needs_zeroing(iter, pos)) {
464
-
folio_zero_range(folio, poff, plen);
465
-
iomap_set_range_uptodate(folio, poff, plen);
466
-
goto done;
467
-
}
391
+
length = min_t(loff_t, length,
392
+
folio_size(folio) - offset_in_folio(folio, pos));
393
+
while (length) {
394
+
iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff,
395
+
&plen);
468
396
469
-
ctx->cur_folio_in_bio = true;
470
-
if (ifs) {
471
-
spin_lock_irq(&ifs->state_lock);
472
-
ifs->read_bytes_pending += plen;
473
-
spin_unlock_irq(&ifs->state_lock);
474
-
}
397
+
pos_diff = pos - iter->pos;
398
+
if (WARN_ON_ONCE(pos_diff + plen > length))
399
+
return -EIO;
475
400
476
-
sector = iomap_sector(iomap, pos);
477
-
if (!ctx->bio ||
478
-
bio_end_sector(ctx->bio) != sector ||
479
-
!bio_add_folio(ctx->bio, folio, plen, poff)) {
480
-
gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
481
-
gfp_t orig_gfp = gfp;
482
-
unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE);
483
-
484
-
if (ctx->bio)
485
-
submit_bio(ctx->bio);
486
-
487
-
if (ctx->rac) /* same as readahead_gfp_mask */
488
-
gfp |= __GFP_NORETRY | __GFP_NOWARN;
489
-
ctx->bio = bio_alloc(iomap->bdev, bio_max_segs(nr_vecs),
490
-
REQ_OP_READ, gfp);
491
-
/*
492
-
* If the bio_alloc fails, try it again for a single page to
493
-
* avoid having to deal with partial page reads. This emulates
494
-
* what do_mpage_read_folio does.
495
-
*/
496
-
if (!ctx->bio) {
497
-
ctx->bio = bio_alloc(iomap->bdev, 1, REQ_OP_READ,
498
-
orig_gfp);
499
-
}
500
-
if (ctx->rac)
501
-
ctx->bio->bi_opf |= REQ_RAHEAD;
502
-
ctx->bio->bi_iter.bi_sector = sector;
503
-
ctx->bio->bi_end_io = iomap_read_end_io;
504
-
bio_add_folio_nofail(ctx->bio, folio, plen, poff);
505
-
}
506
-
507
-
done:
508
-
/*
509
-
* Move the caller beyond our range so that it keeps making progress.
510
-
* For that, we have to include any leading non-uptodate ranges, but
511
-
* we can skip trailing ones as they will be handled in the next
512
-
* iteration.
513
-
*/
514
-
length = pos - iter->pos + plen;
515
-
return iomap_iter_advance(iter, length);
516
-
}
517
-
518
-
static int iomap_read_folio_iter(struct iomap_iter *iter,
519
-
struct iomap_readpage_ctx *ctx)
520
-
{
521
-
int ret;
522
-
523
-
while (iomap_length(iter)) {
524
-
ret = iomap_readpage_iter(iter, ctx);
401
+
ret = iomap_iter_advance(iter, pos_diff);
525
402
if (ret)
526
403
return ret;
527
-
}
528
404
405
+
if (plen == 0)
406
+
return 0;
407
+
408
+
/* zero post-eof blocks as the page may be mapped */
409
+
if (iomap_block_needs_zeroing(iter, pos)) {
410
+
folio_zero_range(folio, poff, plen);
411
+
iomap_set_range_uptodate(folio, poff, plen);
412
+
} else {
413
+
if (!*bytes_pending)
414
+
iomap_read_init(folio);
415
+
*bytes_pending += plen;
416
+
ret = ctx->ops->read_folio_range(iter, ctx, plen);
417
+
if (ret)
418
+
return ret;
419
+
}
420
+
421
+
ret = iomap_iter_advance(iter, plen);
422
+
if (ret)
423
+
return ret;
424
+
length -= pos_diff + plen;
425
+
pos = iter->pos;
426
+
}
529
427
return 0;
530
428
}
531
429
532
-
int iomap_read_folio(struct folio *folio, const struct iomap_ops *ops)
430
+
void iomap_read_folio(const struct iomap_ops *ops,
431
+
struct iomap_read_folio_ctx *ctx)
533
432
{
433
+
struct folio *folio = ctx->cur_folio;
534
434
struct iomap_iter iter = {
535
435
.inode = folio->mapping->host,
536
436
.pos = folio_pos(folio),
537
437
.len = folio_size(folio),
538
438
};
539
-
struct iomap_readpage_ctx ctx = {
540
-
.cur_folio = folio,
541
-
};
439
+
size_t bytes_pending = 0;
542
440
int ret;
543
441
544
442
trace_iomap_readpage(iter.inode, 1);
545
443
546
444
while ((ret = iomap_iter(&iter, ops)) > 0)
547
-
iter.status = iomap_read_folio_iter(&iter, &ctx);
445
+
iter.status = iomap_read_folio_iter(&iter, ctx, &bytes_pending);
548
446
549
-
if (ctx.bio) {
550
-
submit_bio(ctx.bio);
551
-
WARN_ON_ONCE(!ctx.cur_folio_in_bio);
552
-
} else {
553
-
WARN_ON_ONCE(ctx.cur_folio_in_bio);
554
-
folio_unlock(folio);
555
-
}
447
+
if (ctx->ops->submit_read)
448
+
ctx->ops->submit_read(ctx);
556
449
557
-
/*
558
-
* Just like mpage_readahead and block_read_full_folio, we always
559
-
* return 0 and just set the folio error flag on errors. This
560
-
* should be cleaned up throughout the stack eventually.
561
-
*/
562
-
return 0;
450
+
iomap_read_end(folio, bytes_pending);
563
451
}
564
452
EXPORT_SYMBOL_GPL(iomap_read_folio);
565
453
566
454
static int iomap_readahead_iter(struct iomap_iter *iter,
567
-
struct iomap_readpage_ctx *ctx)
455
+
struct iomap_read_folio_ctx *ctx, size_t *cur_bytes_pending)
568
456
{
569
457
int ret;
570
458
571
459
while (iomap_length(iter)) {
572
460
if (ctx->cur_folio &&
573
461
offset_in_folio(ctx->cur_folio, iter->pos) == 0) {
574
-
if (!ctx->cur_folio_in_bio)
575
-
folio_unlock(ctx->cur_folio);
462
+
iomap_read_end(ctx->cur_folio, *cur_bytes_pending);
576
463
ctx->cur_folio = NULL;
577
464
}
578
465
if (!ctx->cur_folio) {
579
466
ctx->cur_folio = readahead_folio(ctx->rac);
580
-
ctx->cur_folio_in_bio = false;
467
+
if (WARN_ON_ONCE(!ctx->cur_folio))
468
+
return -EINVAL;
469
+
*cur_bytes_pending = 0;
581
470
}
582
-
ret = iomap_readpage_iter(iter, ctx);
471
+
ret = iomap_read_folio_iter(iter, ctx, cur_bytes_pending);
583
472
if (ret)
584
473
return ret;
585
474
}
···
546
521
547
522
/**
548
523
* iomap_readahead - Attempt to read pages from a file.
549
-
* @rac: Describes the pages to be read.
550
524
* @ops: The operations vector for the filesystem.
525
+
* @ctx: The ctx used for issuing readahead.
551
526
*
552
527
* This function is for filesystems to call to implement their readahead
553
528
* address_space operation.
···
559
534
* function is called with memalloc_nofs set, so allocations will not cause
560
535
* the filesystem to be reentered.
561
536
*/
562
-
void iomap_readahead(struct readahead_control *rac, const struct iomap_ops *ops)
537
+
void iomap_readahead(const struct iomap_ops *ops,
538
+
struct iomap_read_folio_ctx *ctx)
563
539
{
540
+
struct readahead_control *rac = ctx->rac;
564
541
struct iomap_iter iter = {
565
542
.inode = rac->mapping->host,
566
543
.pos = readahead_pos(rac),
567
544
.len = readahead_length(rac),
568
545
};
569
-
struct iomap_readpage_ctx ctx = {
570
-
.rac = rac,
571
-
};
546
+
size_t cur_bytes_pending;
572
547
573
548
trace_iomap_readahead(rac->mapping->host, readahead_count(rac));
574
549
575
550
while (iomap_iter(&iter, ops) > 0)
576
-
iter.status = iomap_readahead_iter(&iter, &ctx);
551
+
iter.status = iomap_readahead_iter(&iter, ctx,
552
+
&cur_bytes_pending);
577
553
578
-
if (ctx.bio)
579
-
submit_bio(ctx.bio);
580
-
if (ctx.cur_folio) {
581
-
if (!ctx.cur_folio_in_bio)
582
-
folio_unlock(ctx.cur_folio);
583
-
}
554
+
if (ctx->ops->submit_read)
555
+
ctx->ops->submit_read(ctx);
556
+
557
+
if (ctx->cur_folio)
558
+
iomap_read_end(ctx->cur_folio, cur_bytes_pending);
584
559
}
585
560
EXPORT_SYMBOL_GPL(iomap_readahead);
586
-
587
-
static int iomap_read_folio_range(const struct iomap_iter *iter,
588
-
struct folio *folio, loff_t pos, size_t len)
589
-
{
590
-
const struct iomap *srcmap = iomap_iter_srcmap(iter);
591
-
struct bio_vec bvec;
592
-
struct bio bio;
593
-
594
-
bio_init(&bio, srcmap->bdev, &bvec, 1, REQ_OP_READ);
595
-
bio.bi_iter.bi_sector = iomap_sector(srcmap, pos);
596
-
bio_add_folio_nofail(&bio, folio, len, offset_in_folio(folio, pos));
597
-
return submit_bio_wait(&bio);
598
-
}
599
-
#else
600
-
static int iomap_read_folio_range(const struct iomap_iter *iter,
601
-
struct folio *folio, loff_t pos, size_t len)
602
-
{
603
-
WARN_ON_ONCE(1);
604
-
return -EIO;
605
-
}
606
-
#endif /* CONFIG_BLOCK */
607
561
608
562
/*
609
563
* iomap_is_partially_uptodate checks whether blocks within a folio are
···
753
749
status = write_ops->read_folio_range(iter,
754
750
folio, block_start, plen);
755
751
else
756
-
status = iomap_read_folio_range(iter,
752
+
status = iomap_bio_read_folio_range_sync(iter,
757
753
folio, block_start, plen);
758
754
if (status)
759
755
return status;
+12
fs/iomap/internal.h
+12
fs/iomap/internal.h
···
6
6
7
7
u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend);
8
8
9
+
#ifdef CONFIG_BLOCK
10
+
int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter,
11
+
struct folio *folio, loff_t pos, size_t len);
12
+
#else
13
+
static inline int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter,
14
+
struct folio *folio, loff_t pos, size_t len)
15
+
{
16
+
WARN_ON_ONCE(1);
17
+
return -EIO;
18
+
}
19
+
#endif /* CONFIG_BLOCK */
20
+
9
21
#endif /* _IOMAP_INTERNAL_H */
+3
-2
fs/xfs/xfs_aops.c
+3
-2
fs/xfs/xfs_aops.c
···
742
742
struct file *unused,
743
743
struct folio *folio)
744
744
{
745
-
return iomap_read_folio(folio, &xfs_read_iomap_ops);
745
+
iomap_bio_read_folio(folio, &xfs_read_iomap_ops);
746
+
return 0;
746
747
}
747
748
748
749
STATIC void
749
750
xfs_vm_readahead(
750
751
struct readahead_control *rac)
751
752
{
752
-
iomap_readahead(rac, &xfs_read_iomap_ops);
753
+
iomap_bio_readahead(rac, &xfs_read_iomap_ops);
753
754
}
754
755
755
756
static int
+3
-2
fs/zonefs/file.c
+3
-2
fs/zonefs/file.c
···
112
112
113
113
static int zonefs_read_folio(struct file *unused, struct folio *folio)
114
114
{
115
-
return iomap_read_folio(folio, &zonefs_read_iomap_ops);
115
+
iomap_bio_read_folio(folio, &zonefs_read_iomap_ops);
116
+
return 0;
116
117
}
117
118
118
119
static void zonefs_readahead(struct readahead_control *rac)
119
120
{
120
-
iomap_readahead(rac, &zonefs_read_iomap_ops);
121
+
iomap_bio_readahead(rac, &zonefs_read_iomap_ops);
121
122
}
122
123
123
124
/*
+61
-2
include/linux/iomap.h
+61
-2
include/linux/iomap.h
···
16
16
struct iomap_iter;
17
17
struct iomap_dio;
18
18
struct iomap_writepage_ctx;
19
+
struct iomap_read_folio_ctx;
19
20
struct iov_iter;
20
21
struct kiocb;
21
22
struct page;
···
338
337
ssize_t iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *from,
339
338
const struct iomap_ops *ops,
340
339
const struct iomap_write_ops *write_ops, void *private);
341
-
int iomap_read_folio(struct folio *folio, const struct iomap_ops *ops);
342
-
void iomap_readahead(struct readahead_control *, const struct iomap_ops *ops);
340
+
void iomap_read_folio(const struct iomap_ops *ops,
341
+
struct iomap_read_folio_ctx *ctx);
342
+
void iomap_readahead(const struct iomap_ops *ops,
343
+
struct iomap_read_folio_ctx *ctx);
343
344
bool iomap_is_partially_uptodate(struct folio *, size_t from, size_t count);
344
345
struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len);
345
346
bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags);
···
468
465
loff_t pos, loff_t end_pos, unsigned int dirty_len);
469
466
int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error);
470
467
468
+
void iomap_finish_folio_read(struct folio *folio, size_t off, size_t len,
469
+
int error);
471
470
void iomap_start_folio_write(struct inode *inode, struct folio *folio,
472
471
size_t len);
473
472
void iomap_finish_folio_write(struct inode *inode, struct folio *folio,
···
477
472
478
473
int iomap_writeback_folio(struct iomap_writepage_ctx *wpc, struct folio *folio);
479
474
int iomap_writepages(struct iomap_writepage_ctx *wpc);
475
+
476
+
struct iomap_read_folio_ctx {
477
+
const struct iomap_read_ops *ops;
478
+
struct folio *cur_folio;
479
+
struct readahead_control *rac;
480
+
void *read_ctx;
481
+
};
482
+
483
+
struct iomap_read_ops {
484
+
/*
485
+
* Read in a folio range.
486
+
*
487
+
* The caller is responsible for calling iomap_finish_folio_read() after
488
+
* reading in the folio range. This should be done even if an error is
489
+
* encountered during the read.
490
+
*
491
+
* Returns 0 on success or a negative error on failure.
492
+
*/
493
+
int (*read_folio_range)(const struct iomap_iter *iter,
494
+
struct iomap_read_folio_ctx *ctx, size_t len);
495
+
496
+
/*
497
+
* Submit any pending read requests.
498
+
*
499
+
* This is optional.
500
+
*/
501
+
void (*submit_read)(struct iomap_read_folio_ctx *ctx);
502
+
};
480
503
481
504
/*
482
505
* Flags for direct I/O ->end_io:
···
570
537
#endif /* CONFIG_SWAP */
571
538
572
539
extern struct bio_set iomap_ioend_bioset;
540
+
541
+
#ifdef CONFIG_BLOCK
542
+
extern const struct iomap_read_ops iomap_bio_read_ops;
543
+
544
+
static inline void iomap_bio_read_folio(struct folio *folio,
545
+
const struct iomap_ops *ops)
546
+
{
547
+
struct iomap_read_folio_ctx ctx = {
548
+
.ops = &iomap_bio_read_ops,
549
+
.cur_folio = folio,
550
+
};
551
+
552
+
iomap_read_folio(ops, &ctx);
553
+
}
554
+
555
+
static inline void iomap_bio_readahead(struct readahead_control *rac,
556
+
const struct iomap_ops *ops)
557
+
{
558
+
struct iomap_read_folio_ctx ctx = {
559
+
.ops = &iomap_bio_read_ops,
560
+
.rac = rac,
561
+
};
562
+
563
+
iomap_readahead(ops, &ctx);
564
+
}
565
+
#endif /* CONFIG_BLOCK */
573
566
574
567
#endif /* LINUX_IOMAP_H */