Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'iomap-5.17' of git://git.infradead.org/users/willy/linux
Pull iomap updates from Matthew Wilcox:
"Convert xfs/iomap to use folios.
This should be all that is needed for XFS to use large folios. There
is no code in this pull request to create large folios, but no
additional changes should be needed to XFS or iomap once they are
created.
Usually this would have come from Darrick, and we had intended that it
would come that route. Between the holidays and various things which
Darrick needed to work on, he asked if I could send things directly.
There weren't any other iomap patches pending for this release, which
probably also played a role"
* tag 'iomap-5.17' of git://git.infradead.org/users/willy/linux: (26 commits)
iomap: Inline __iomap_zero_iter into its caller
xfs: Support large folios
iomap: Support large folios in invalidatepage
iomap: Convert iomap_migrate_page() to use folios
iomap: Convert iomap_add_to_ioend() to take a folio
iomap: Simplify iomap_do_writepage()
iomap: Simplify iomap_writepage_map()
iomap,xfs: Convert ->discard_page to ->discard_folio
iomap: Convert iomap_write_end_inline to take a folio
iomap: Convert iomap_write_begin() and iomap_write_end() to folios
iomap: Convert __iomap_zero_iter to use a folio
iomap: Allow iomap_write_begin() to be called with the full length
iomap: Convert iomap_page_mkwrite to use a folio
iomap: Convert readahead and readpage to use a folio
iomap: Convert iomap_read_inline_data to take a folio
iomap: Use folio offsets instead of page offsets
iomap: Convert bio completions to use folios
iomap: Pass the iomap_page into iomap_set_range_uptodate
iomap: Add iomap_invalidate_folio
iomap: Convert iomap_releasepage to use a folio
...
+389
-295
+1
Documentation/core-api/kernel-api.rst
+1
Documentation/core-api/kernel-api.rst
+22
block/bio.c
+22
block/bio.c
···
1035
1035
}
1036
1036
EXPORT_SYMBOL(bio_add_page);
1037
1037
1038
+
/**
1039
+
* bio_add_folio - Attempt to add part of a folio to a bio.
1040
+
* @bio: BIO to add to.
1041
+
* @folio: Folio to add.
1042
+
* @len: How many bytes from the folio to add.
1043
+
* @off: First byte in this folio to add.
1044
+
*
1045
+
* Filesystems that use folios can call this function instead of calling
1046
+
* bio_add_page() for each page in the folio. If @off is bigger than
1047
+
* PAGE_SIZE, this function can create a bio_vec that starts in a page
1048
+
* after the bv_page. BIOs do not support folios that are 4GiB or larger.
1049
+
*
1050
+
* Return: Whether the addition was successful.
1051
+
*/
1052
+
bool bio_add_folio(struct bio *bio, struct folio *folio, size_t len,
1053
+
size_t off)
1054
+
{
1055
+
if (len > UINT_MAX || off > UINT_MAX)
1056
+
return 0;
1057
+
return bio_add_page(bio, &folio->page, len, off) > 0;
1058
+
}
1059
+
1038
1060
void __bio_release_pages(struct bio *bio, bool mark_dirty)
1039
1061
{
1040
1062
struct bvec_iter_all iter_all;
+12
-11
fs/buffer.c
+12
-11
fs/buffer.c
···
1969
1969
}
1970
1970
}
1971
1971
1972
-
int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
1972
+
int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len,
1973
1973
get_block_t *get_block, const struct iomap *iomap)
1974
1974
{
1975
1975
unsigned from = pos & (PAGE_SIZE - 1);
1976
1976
unsigned to = from + len;
1977
-
struct inode *inode = page->mapping->host;
1977
+
struct inode *inode = folio->mapping->host;
1978
1978
unsigned block_start, block_end;
1979
1979
sector_t block;
1980
1980
int err = 0;
1981
1981
unsigned blocksize, bbits;
1982
1982
struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1983
1983
1984
-
BUG_ON(!PageLocked(page));
1984
+
BUG_ON(!folio_test_locked(folio));
1985
1985
BUG_ON(from > PAGE_SIZE);
1986
1986
BUG_ON(to > PAGE_SIZE);
1987
1987
BUG_ON(from > to);
1988
1988
1989
-
head = create_page_buffers(page, inode, 0);
1989
+
head = create_page_buffers(&folio->page, inode, 0);
1990
1990
blocksize = head->b_size;
1991
1991
bbits = block_size_bits(blocksize);
1992
1992
1993
-
block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1993
+
block = (sector_t)folio->index << (PAGE_SHIFT - bbits);
1994
1994
1995
1995
for(bh = head, block_start = 0; bh != head || !block_start;
1996
1996
block++, block_start=block_end, bh = bh->b_this_page) {
1997
1997
block_end = block_start + blocksize;
1998
1998
if (block_end <= from || block_start >= to) {
1999
-
if (PageUptodate(page)) {
1999
+
if (folio_test_uptodate(folio)) {
2000
2000
if (!buffer_uptodate(bh))
2001
2001
set_buffer_uptodate(bh);
2002
2002
}
···
2016
2016
2017
2017
if (buffer_new(bh)) {
2018
2018
clean_bdev_bh_alias(bh);
2019
-
if (PageUptodate(page)) {
2019
+
if (folio_test_uptodate(folio)) {
2020
2020
clear_buffer_new(bh);
2021
2021
set_buffer_uptodate(bh);
2022
2022
mark_buffer_dirty(bh);
2023
2023
continue;
2024
2024
}
2025
2025
if (block_end > to || block_start < from)
2026
-
zero_user_segments(page,
2026
+
folio_zero_segments(folio,
2027
2027
to, block_end,
2028
2028
block_start, from);
2029
2029
continue;
2030
2030
}
2031
2031
}
2032
-
if (PageUptodate(page)) {
2032
+
if (folio_test_uptodate(folio)) {
2033
2033
if (!buffer_uptodate(bh))
2034
2034
set_buffer_uptodate(bh);
2035
2035
continue;
···
2050
2050
err = -EIO;
2051
2051
}
2052
2052
if (unlikely(err))
2053
-
page_zero_new_buffers(page, from, to);
2053
+
page_zero_new_buffers(&folio->page, from, to);
2054
2054
return err;
2055
2055
}
2056
2056
2057
2057
int __block_write_begin(struct page *page, loff_t pos, unsigned len,
2058
2058
get_block_t *get_block)
2059
2059
{
2060
-
return __block_write_begin_int(page, pos, len, get_block, NULL);
2060
+
return __block_write_begin_int(page_folio(page), pos, len, get_block,
2061
+
NULL);
2061
2062
}
2062
2063
EXPORT_SYMBOL(__block_write_begin);
2063
2064
+1
-1
fs/internal.h
+1
-1
fs/internal.h
+283
-268
fs/iomap/buffered-io.c
+283
-268
fs/iomap/buffered-io.c
···
22
22
#include "../internal.h"
23
23
24
24
/*
25
-
* Structure allocated for each page or THP when block size < page size
26
-
* to track sub-page uptodate status and I/O completions.
25
+
* Structure allocated for each folio when block size < folio size
26
+
* to track sub-folio uptodate status and I/O completions.
27
27
*/
28
28
struct iomap_page {
29
29
atomic_t read_bytes_pending;
···
32
32
unsigned long uptodate[];
33
33
};
34
34
35
-
static inline struct iomap_page *to_iomap_page(struct page *page)
35
+
static inline struct iomap_page *to_iomap_page(struct folio *folio)
36
36
{
37
-
/*
38
-
* per-block data is stored in the head page. Callers should
39
-
* not be dealing with tail pages, and if they are, they can
40
-
* call thp_head() first.
41
-
*/
42
-
VM_BUG_ON_PGFLAGS(PageTail(page), page);
43
-
44
-
if (page_has_private(page))
45
-
return (struct iomap_page *)page_private(page);
37
+
if (folio_test_private(folio))
38
+
return folio_get_private(folio);
46
39
return NULL;
47
40
}
48
41
49
42
static struct bio_set iomap_ioend_bioset;
50
43
51
44
static struct iomap_page *
52
-
iomap_page_create(struct inode *inode, struct page *page)
45
+
iomap_page_create(struct inode *inode, struct folio *folio)
53
46
{
54
-
struct iomap_page *iop = to_iomap_page(page);
55
-
unsigned int nr_blocks = i_blocks_per_page(inode, page);
47
+
struct iomap_page *iop = to_iomap_page(folio);
48
+
unsigned int nr_blocks = i_blocks_per_folio(inode, folio);
56
49
57
50
if (iop || nr_blocks <= 1)
58
51
return iop;
···
53
60
iop = kzalloc(struct_size(iop, uptodate, BITS_TO_LONGS(nr_blocks)),
54
61
GFP_NOFS | __GFP_NOFAIL);
55
62
spin_lock_init(&iop->uptodate_lock);
56
-
if (PageUptodate(page))
63
+
if (folio_test_uptodate(folio))
57
64
bitmap_fill(iop->uptodate, nr_blocks);
58
-
attach_page_private(page, iop);
65
+
folio_attach_private(folio, iop);
59
66
return iop;
60
67
}
61
68
62
-
static void
63
-
iomap_page_release(struct page *page)
69
+
static void iomap_page_release(struct folio *folio)
64
70
{
65
-
struct iomap_page *iop = detach_page_private(page);
66
-
unsigned int nr_blocks = i_blocks_per_page(page->mapping->host, page);
71
+
struct iomap_page *iop = folio_detach_private(folio);
72
+
struct inode *inode = folio->mapping->host;
73
+
unsigned int nr_blocks = i_blocks_per_folio(inode, folio);
67
74
68
75
if (!iop)
69
76
return;
70
77
WARN_ON_ONCE(atomic_read(&iop->read_bytes_pending));
71
78
WARN_ON_ONCE(atomic_read(&iop->write_bytes_pending));
72
79
WARN_ON_ONCE(bitmap_full(iop->uptodate, nr_blocks) !=
73
-
PageUptodate(page));
80
+
folio_test_uptodate(folio));
74
81
kfree(iop);
75
82
}
76
83
77
84
/*
78
-
* Calculate the range inside the page that we actually need to read.
85
+
* Calculate the range inside the folio that we actually need to read.
79
86
*/
80
-
static void
81
-
iomap_adjust_read_range(struct inode *inode, struct iomap_page *iop,
82
-
loff_t *pos, loff_t length, unsigned *offp, unsigned *lenp)
87
+
static void iomap_adjust_read_range(struct inode *inode, struct folio *folio,
88
+
loff_t *pos, loff_t length, size_t *offp, size_t *lenp)
83
89
{
90
+
struct iomap_page *iop = to_iomap_page(folio);
84
91
loff_t orig_pos = *pos;
85
92
loff_t isize = i_size_read(inode);
86
93
unsigned block_bits = inode->i_blkbits;
87
94
unsigned block_size = (1 << block_bits);
88
-
unsigned poff = offset_in_page(*pos);
89
-
unsigned plen = min_t(loff_t, PAGE_SIZE - poff, length);
95
+
size_t poff = offset_in_folio(folio, *pos);
96
+
size_t plen = min_t(loff_t, folio_size(folio) - poff, length);
90
97
unsigned first = poff >> block_bits;
91
98
unsigned last = (poff + plen - 1) >> block_bits;
92
99
···
124
131
* page cache for blocks that are entirely outside of i_size.
125
132
*/
126
133
if (orig_pos <= isize && orig_pos + length > isize) {
127
-
unsigned end = offset_in_page(isize - 1) >> block_bits;
134
+
unsigned end = offset_in_folio(folio, isize - 1) >> block_bits;
128
135
129
136
if (first <= end && last > end)
130
137
plen -= (last - end) * block_size;
···
134
141
*lenp = plen;
135
142
}
136
143
137
-
static void
138
-
iomap_iop_set_range_uptodate(struct page *page, unsigned off, unsigned len)
144
+
static void iomap_iop_set_range_uptodate(struct folio *folio,
145
+
struct iomap_page *iop, size_t off, size_t len)
139
146
{
140
-
struct iomap_page *iop = to_iomap_page(page);
141
-
struct inode *inode = page->mapping->host;
147
+
struct inode *inode = folio->mapping->host;
142
148
unsigned first = off >> inode->i_blkbits;
143
149
unsigned last = (off + len - 1) >> inode->i_blkbits;
144
150
unsigned long flags;
145
151
146
152
spin_lock_irqsave(&iop->uptodate_lock, flags);
147
153
bitmap_set(iop->uptodate, first, last - first + 1);
148
-
if (bitmap_full(iop->uptodate, i_blocks_per_page(inode, page)))
149
-
SetPageUptodate(page);
154
+
if (bitmap_full(iop->uptodate, i_blocks_per_folio(inode, folio)))
155
+
folio_mark_uptodate(folio);
150
156
spin_unlock_irqrestore(&iop->uptodate_lock, flags);
151
157
}
152
158
153
-
static void
154
-
iomap_set_range_uptodate(struct page *page, unsigned off, unsigned len)
159
+
static void iomap_set_range_uptodate(struct folio *folio,
160
+
struct iomap_page *iop, size_t off, size_t len)
155
161
{
156
-
if (PageError(page))
162
+
if (folio_test_error(folio))
157
163
return;
158
164
159
-
if (page_has_private(page))
160
-
iomap_iop_set_range_uptodate(page, off, len);
165
+
if (iop)
166
+
iomap_iop_set_range_uptodate(folio, iop, off, len);
161
167
else
162
-
SetPageUptodate(page);
168
+
folio_mark_uptodate(folio);
163
169
}
164
170
165
-
static void
166
-
iomap_read_page_end_io(struct bio_vec *bvec, int error)
171
+
static void iomap_finish_folio_read(struct folio *folio, size_t offset,
172
+
size_t len, int error)
167
173
{
168
-
struct page *page = bvec->bv_page;
169
-
struct iomap_page *iop = to_iomap_page(page);
174
+
struct iomap_page *iop = to_iomap_page(folio);
170
175
171
176
if (unlikely(error)) {
172
-
ClearPageUptodate(page);
173
-
SetPageError(page);
177
+
folio_clear_uptodate(folio);
178
+
folio_set_error(folio);
174
179
} else {
175
-
iomap_set_range_uptodate(page, bvec->bv_offset, bvec->bv_len);
180
+
iomap_set_range_uptodate(folio, iop, offset, len);
176
181
}
177
182
178
-
if (!iop || atomic_sub_and_test(bvec->bv_len, &iop->read_bytes_pending))
179
-
unlock_page(page);
183
+
if (!iop || atomic_sub_and_test(len, &iop->read_bytes_pending))
184
+
folio_unlock(folio);
180
185
}
181
186
182
-
static void
183
-
iomap_read_end_io(struct bio *bio)
187
+
static void iomap_read_end_io(struct bio *bio)
184
188
{
185
189
int error = blk_status_to_errno(bio->bi_status);
186
-
struct bio_vec *bvec;
187
-
struct bvec_iter_all iter_all;
190
+
struct folio_iter fi;
188
191
189
-
bio_for_each_segment_all(bvec, bio, iter_all)
190
-
iomap_read_page_end_io(bvec, error);
192
+
bio_for_each_folio_all(fi, bio)
193
+
iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error);
191
194
bio_put(bio);
192
195
}
193
196
194
197
struct iomap_readpage_ctx {
195
-
struct page *cur_page;
196
-
bool cur_page_in_bio;
198
+
struct folio *cur_folio;
199
+
bool cur_folio_in_bio;
197
200
struct bio *bio;
198
201
struct readahead_control *rac;
199
202
};
···
197
208
/**
198
209
* iomap_read_inline_data - copy inline data into the page cache
199
210
* @iter: iteration structure
200
-
* @page: page to copy to
211
+
* @folio: folio to copy to
201
212
*
202
-
* Copy the inline data in @iter into @page and zero out the rest of the page.
213
+
* Copy the inline data in @iter into @folio and zero out the rest of the folio.
203
214
* Only a single IOMAP_INLINE extent is allowed at the end of each file.
204
215
* Returns zero for success to complete the read, or the usual negative errno.
205
216
*/
206
217
static int iomap_read_inline_data(const struct iomap_iter *iter,
207
-
struct page *page)
218
+
struct folio *folio)
208
219
{
220
+
struct iomap_page *iop;
209
221
const struct iomap *iomap = iomap_iter_srcmap(iter);
210
222
size_t size = i_size_read(iter->inode) - iomap->offset;
211
223
size_t poff = offset_in_page(iomap->offset);
224
+
size_t offset = offset_in_folio(folio, iomap->offset);
212
225
void *addr;
213
226
214
-
if (PageUptodate(page))
227
+
if (folio_test_uptodate(folio))
215
228
return 0;
216
229
217
230
if (WARN_ON_ONCE(size > PAGE_SIZE - poff))
···
223
232
return -EIO;
224
233
if (WARN_ON_ONCE(size > iomap->length))
225
234
return -EIO;
226
-
if (poff > 0)
227
-
iomap_page_create(iter->inode, page);
235
+
if (offset > 0)
236
+
iop = iomap_page_create(iter->inode, folio);
237
+
else
238
+
iop = to_iomap_page(folio);
228
239
229
-
addr = kmap_local_page(page) + poff;
240
+
addr = kmap_local_folio(folio, offset);
230
241
memcpy(addr, iomap->inline_data, size);
231
242
memset(addr + size, 0, PAGE_SIZE - poff - size);
232
243
kunmap_local(addr);
233
-
iomap_set_range_uptodate(page, poff, PAGE_SIZE - poff);
244
+
iomap_set_range_uptodate(folio, iop, offset, PAGE_SIZE - poff);
234
245
return 0;
235
246
}
236
247
···
252
259
const struct iomap *iomap = &iter->iomap;
253
260
loff_t pos = iter->pos + offset;
254
261
loff_t length = iomap_length(iter) - offset;
255
-
struct page *page = ctx->cur_page;
262
+
struct folio *folio = ctx->cur_folio;
256
263
struct iomap_page *iop;
257
264
loff_t orig_pos = pos;
258
-
unsigned poff, plen;
265
+
size_t poff, plen;
259
266
sector_t sector;
260
267
261
268
if (iomap->type == IOMAP_INLINE)
262
-
return iomap_read_inline_data(iter, page);
269
+
return iomap_read_inline_data(iter, folio);
263
270
264
271
/* zero post-eof blocks as the page may be mapped */
265
-
iop = iomap_page_create(iter->inode, page);
266
-
iomap_adjust_read_range(iter->inode, iop, &pos, length, &poff, &plen);
272
+
iop = iomap_page_create(iter->inode, folio);
273
+
iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff, &plen);
267
274
if (plen == 0)
268
275
goto done;
269
276
270
277
if (iomap_block_needs_zeroing(iter, pos)) {
271
-
zero_user(page, poff, plen);
272
-
iomap_set_range_uptodate(page, poff, plen);
278
+
folio_zero_range(folio, poff, plen);
279
+
iomap_set_range_uptodate(folio, iop, poff, plen);
273
280
goto done;
274
281
}
275
282
276
-
ctx->cur_page_in_bio = true;
283
+
ctx->cur_folio_in_bio = true;
277
284
if (iop)
278
285
atomic_add(plen, &iop->read_bytes_pending);
279
286
280
287
sector = iomap_sector(iomap, pos);
281
288
if (!ctx->bio ||
282
289
bio_end_sector(ctx->bio) != sector ||
283
-
bio_add_page(ctx->bio, page, plen, poff) != plen) {
284
-
gfp_t gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL);
290
+
!bio_add_folio(ctx->bio, folio, plen, poff)) {
291
+
gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
285
292
gfp_t orig_gfp = gfp;
286
293
unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE);
287
294
···
304
311
ctx->bio->bi_iter.bi_sector = sector;
305
312
bio_set_dev(ctx->bio, iomap->bdev);
306
313
ctx->bio->bi_end_io = iomap_read_end_io;
307
-
__bio_add_page(ctx->bio, page, plen, poff);
314
+
bio_add_folio(ctx->bio, folio, plen, poff);
308
315
}
316
+
309
317
done:
310
318
/*
311
319
* Move the caller beyond our range so that it keeps making progress.
···
320
326
int
321
327
iomap_readpage(struct page *page, const struct iomap_ops *ops)
322
328
{
329
+
struct folio *folio = page_folio(page);
323
330
struct iomap_iter iter = {
324
-
.inode = page->mapping->host,
325
-
.pos = page_offset(page),
326
-
.len = PAGE_SIZE,
331
+
.inode = folio->mapping->host,
332
+
.pos = folio_pos(folio),
333
+
.len = folio_size(folio),
327
334
};
328
335
struct iomap_readpage_ctx ctx = {
329
-
.cur_page = page,
336
+
.cur_folio = folio,
330
337
};
331
338
int ret;
332
339
333
-
trace_iomap_readpage(page->mapping->host, 1);
340
+
trace_iomap_readpage(iter.inode, 1);
334
341
335
342
while ((ret = iomap_iter(&iter, ops)) > 0)
336
343
iter.processed = iomap_readpage_iter(&iter, &ctx, 0);
337
344
338
345
if (ret < 0)
339
-
SetPageError(page);
346
+
folio_set_error(folio);
340
347
341
348
if (ctx.bio) {
342
349
submit_bio(ctx.bio);
343
-
WARN_ON_ONCE(!ctx.cur_page_in_bio);
350
+
WARN_ON_ONCE(!ctx.cur_folio_in_bio);
344
351
} else {
345
-
WARN_ON_ONCE(ctx.cur_page_in_bio);
346
-
unlock_page(page);
352
+
WARN_ON_ONCE(ctx.cur_folio_in_bio);
353
+
folio_unlock(folio);
347
354
}
348
355
349
356
/*
···
363
368
loff_t done, ret;
364
369
365
370
for (done = 0; done < length; done += ret) {
366
-
if (ctx->cur_page && offset_in_page(iter->pos + done) == 0) {
367
-
if (!ctx->cur_page_in_bio)
368
-
unlock_page(ctx->cur_page);
369
-
put_page(ctx->cur_page);
370
-
ctx->cur_page = NULL;
371
+
if (ctx->cur_folio &&
372
+
offset_in_folio(ctx->cur_folio, iter->pos + done) == 0) {
373
+
if (!ctx->cur_folio_in_bio)
374
+
folio_unlock(ctx->cur_folio);
375
+
ctx->cur_folio = NULL;
371
376
}
372
-
if (!ctx->cur_page) {
373
-
ctx->cur_page = readahead_page(ctx->rac);
374
-
ctx->cur_page_in_bio = false;
377
+
if (!ctx->cur_folio) {
378
+
ctx->cur_folio = readahead_folio(ctx->rac);
379
+
ctx->cur_folio_in_bio = false;
375
380
}
376
381
ret = iomap_readpage_iter(iter, ctx, done);
377
382
if (ret <= 0)
···
414
419
415
420
if (ctx.bio)
416
421
submit_bio(ctx.bio);
417
-
if (ctx.cur_page) {
418
-
if (!ctx.cur_page_in_bio)
419
-
unlock_page(ctx.cur_page);
420
-
put_page(ctx.cur_page);
422
+
if (ctx.cur_folio) {
423
+
if (!ctx.cur_folio_in_bio)
424
+
folio_unlock(ctx.cur_folio);
421
425
}
422
426
}
423
427
EXPORT_SYMBOL_GPL(iomap_readahead);
···
432
438
iomap_is_partially_uptodate(struct page *page, unsigned long from,
433
439
unsigned long count)
434
440
{
435
-
struct iomap_page *iop = to_iomap_page(page);
441
+
struct folio *folio = page_folio(page);
442
+
struct iomap_page *iop = to_iomap_page(folio);
436
443
struct inode *inode = page->mapping->host;
437
444
unsigned len, first, last;
438
445
unsigned i;
···
459
464
int
460
465
iomap_releasepage(struct page *page, gfp_t gfp_mask)
461
466
{
462
-
trace_iomap_releasepage(page->mapping->host, page_offset(page),
463
-
PAGE_SIZE);
467
+
struct folio *folio = page_folio(page);
468
+
469
+
trace_iomap_releasepage(folio->mapping->host, folio_pos(folio),
470
+
folio_size(folio));
464
471
465
472
/*
466
473
* mm accommodates an old ext3 case where clean pages might not have had
467
474
* the dirty bit cleared. Thus, it can send actual dirty pages to
468
475
* ->releasepage() via shrink_active_list(); skip those here.
469
476
*/
470
-
if (PageDirty(page) || PageWriteback(page))
477
+
if (folio_test_dirty(folio) || folio_test_writeback(folio))
471
478
return 0;
472
-
iomap_page_release(page);
479
+
iomap_page_release(folio);
473
480
return 1;
474
481
}
475
482
EXPORT_SYMBOL_GPL(iomap_releasepage);
476
483
477
-
void
478
-
iomap_invalidatepage(struct page *page, unsigned int offset, unsigned int len)
484
+
void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len)
479
485
{
480
-
trace_iomap_invalidatepage(page->mapping->host, offset, len);
486
+
trace_iomap_invalidatepage(folio->mapping->host, offset, len);
481
487
482
488
/*
483
-
* If we're invalidating the entire page, clear the dirty state from it
484
-
* and release it to avoid unnecessary buildup of the LRU.
489
+
* If we're invalidating the entire folio, clear the dirty state
490
+
* from it and release it to avoid unnecessary buildup of the LRU.
485
491
*/
486
-
if (offset == 0 && len == PAGE_SIZE) {
487
-
WARN_ON_ONCE(PageWriteback(page));
488
-
cancel_dirty_page(page);
489
-
iomap_page_release(page);
492
+
if (offset == 0 && len == folio_size(folio)) {
493
+
WARN_ON_ONCE(folio_test_writeback(folio));
494
+
folio_cancel_dirty(folio);
495
+
iomap_page_release(folio);
496
+
} else if (folio_test_large(folio)) {
497
+
/* Must release the iop so the page can be split */
498
+
WARN_ON_ONCE(!folio_test_uptodate(folio) &&
499
+
folio_test_dirty(folio));
500
+
iomap_page_release(folio);
490
501
}
502
+
}
503
+
EXPORT_SYMBOL_GPL(iomap_invalidate_folio);
504
+
505
+
void iomap_invalidatepage(struct page *page, unsigned int offset,
506
+
unsigned int len)
507
+
{
508
+
iomap_invalidate_folio(page_folio(page), offset, len);
491
509
}
492
510
EXPORT_SYMBOL_GPL(iomap_invalidatepage);
493
511
···
509
501
iomap_migrate_page(struct address_space *mapping, struct page *newpage,
510
502
struct page *page, enum migrate_mode mode)
511
503
{
504
+
struct folio *folio = page_folio(page);
505
+
struct folio *newfolio = page_folio(newpage);
512
506
int ret;
513
507
514
-
ret = migrate_page_move_mapping(mapping, newpage, page, 0);
508
+
ret = folio_migrate_mapping(mapping, newfolio, folio, 0);
515
509
if (ret != MIGRATEPAGE_SUCCESS)
516
510
return ret;
517
511
518
-
if (page_has_private(page))
519
-
attach_page_private(newpage, detach_page_private(page));
512
+
if (folio_test_private(folio))
513
+
folio_attach_private(newfolio, folio_detach_private(folio));
520
514
521
515
if (mode != MIGRATE_SYNC_NO_COPY)
522
-
migrate_page_copy(newpage, page);
516
+
folio_migrate_copy(newfolio, folio);
523
517
else
524
-
migrate_page_states(newpage, page);
518
+
folio_migrate_flags(newfolio, folio);
525
519
return MIGRATEPAGE_SUCCESS;
526
520
}
527
521
EXPORT_SYMBOL_GPL(iomap_migrate_page);
···
542
532
truncate_pagecache_range(inode, max(pos, i_size), pos + len);
543
533
}
544
534
545
-
static int
546
-
iomap_read_page_sync(loff_t block_start, struct page *page, unsigned poff,
547
-
unsigned plen, const struct iomap *iomap)
535
+
static int iomap_read_folio_sync(loff_t block_start, struct folio *folio,
536
+
size_t poff, size_t plen, const struct iomap *iomap)
548
537
{
549
538
struct bio_vec bvec;
550
539
struct bio bio;
···
552
543
bio.bi_opf = REQ_OP_READ;
553
544
bio.bi_iter.bi_sector = iomap_sector(iomap, block_start);
554
545
bio_set_dev(&bio, iomap->bdev);
555
-
__bio_add_page(&bio, page, plen, poff);
546
+
bio_add_folio(&bio, folio, plen, poff);
556
547
return submit_bio_wait(&bio);
557
548
}
558
549
559
550
static int __iomap_write_begin(const struct iomap_iter *iter, loff_t pos,
560
-
unsigned len, struct page *page)
551
+
size_t len, struct folio *folio)
561
552
{
562
553
const struct iomap *srcmap = iomap_iter_srcmap(iter);
563
-
struct iomap_page *iop = iomap_page_create(iter->inode, page);
554
+
struct iomap_page *iop = iomap_page_create(iter->inode, folio);
564
555
loff_t block_size = i_blocksize(iter->inode);
565
556
loff_t block_start = round_down(pos, block_size);
566
557
loff_t block_end = round_up(pos + len, block_size);
567
-
unsigned from = offset_in_page(pos), to = from + len, poff, plen;
558
+
size_t from = offset_in_folio(folio, pos), to = from + len;
559
+
size_t poff, plen;
568
560
569
-
if (PageUptodate(page))
561
+
if (folio_test_uptodate(folio))
570
562
return 0;
571
-
ClearPageError(page);
563
+
folio_clear_error(folio);
572
564
573
565
do {
574
-
iomap_adjust_read_range(iter->inode, iop, &block_start,
566
+
iomap_adjust_read_range(iter->inode, folio, &block_start,
575
567
block_end - block_start, &poff, &plen);
576
568
if (plen == 0)
577
569
break;
···
585
575
if (iomap_block_needs_zeroing(iter, block_start)) {
586
576
if (WARN_ON_ONCE(iter->flags & IOMAP_UNSHARE))
587
577
return -EIO;
588
-
zero_user_segments(page, poff, from, to, poff + plen);
578
+
folio_zero_segments(folio, poff, from, to, poff + plen);
589
579
} else {
590
-
int status = iomap_read_page_sync(block_start, page,
580
+
int status = iomap_read_folio_sync(block_start, folio,
591
581
poff, plen, srcmap);
592
582
if (status)
593
583
return status;
594
584
}
595
-
iomap_set_range_uptodate(page, poff, plen);
585
+
iomap_set_range_uptodate(folio, iop, poff, plen);
596
586
} while ((block_start += plen) < block_end);
597
587
598
588
return 0;
599
589
}
600
590
601
591
static int iomap_write_begin_inline(const struct iomap_iter *iter,
602
-
struct page *page)
592
+
struct folio *folio)
603
593
{
604
594
/* needs more work for the tailpacking case; disable for now */
605
595
if (WARN_ON_ONCE(iomap_iter_srcmap(iter)->offset != 0))
606
596
return -EIO;
607
-
return iomap_read_inline_data(iter, page);
597
+
return iomap_read_inline_data(iter, folio);
608
598
}
609
599
610
600
static int iomap_write_begin(const struct iomap_iter *iter, loff_t pos,
611
-
unsigned len, struct page **pagep)
601
+
size_t len, struct folio **foliop)
612
602
{
613
603
const struct iomap_page_ops *page_ops = iter->iomap.page_ops;
614
604
const struct iomap *srcmap = iomap_iter_srcmap(iter);
615
-
struct page *page;
605
+
struct folio *folio;
606
+
unsigned fgp = FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_STABLE | FGP_NOFS;
616
607
int status = 0;
617
608
618
609
BUG_ON(pos + len > iter->iomap.offset + iter->iomap.length);
···
623
612
if (fatal_signal_pending(current))
624
613
return -EINTR;
625
614
615
+
if (!mapping_large_folio_support(iter->inode->i_mapping))
616
+
len = min_t(size_t, len, PAGE_SIZE - offset_in_page(pos));
617
+
626
618
if (page_ops && page_ops->page_prepare) {
627
619
status = page_ops->page_prepare(iter->inode, pos, len);
628
620
if (status)
629
621
return status;
630
622
}
631
623
632
-
page = grab_cache_page_write_begin(iter->inode->i_mapping,
633
-
pos >> PAGE_SHIFT, AOP_FLAG_NOFS);
634
-
if (!page) {
624
+
folio = __filemap_get_folio(iter->inode->i_mapping, pos >> PAGE_SHIFT,
625
+
fgp, mapping_gfp_mask(iter->inode->i_mapping));
626
+
if (!folio) {
635
627
status = -ENOMEM;
636
628
goto out_no_page;
637
629
}
630
+
if (pos + len > folio_pos(folio) + folio_size(folio))
631
+
len = folio_pos(folio) + folio_size(folio) - pos;
638
632
639
633
if (srcmap->type == IOMAP_INLINE)
640
-
status = iomap_write_begin_inline(iter, page);
634
+
status = iomap_write_begin_inline(iter, folio);
641
635
else if (srcmap->flags & IOMAP_F_BUFFER_HEAD)
642
-
status = __block_write_begin_int(page, pos, len, NULL, srcmap);
636
+
status = __block_write_begin_int(folio, pos, len, NULL, srcmap);
643
637
else
644
-
status = __iomap_write_begin(iter, pos, len, page);
638
+
status = __iomap_write_begin(iter, pos, len, folio);
645
639
646
640
if (unlikely(status))
647
641
goto out_unlock;
648
642
649
-
*pagep = page;
643
+
*foliop = folio;
650
644
return 0;
651
645
652
646
out_unlock:
653
-
unlock_page(page);
654
-
put_page(page);
647
+
folio_unlock(folio);
648
+
folio_put(folio);
655
649
iomap_write_failed(iter->inode, pos, len);
656
650
657
651
out_no_page:
···
666
650
}
667
651
668
652
static size_t __iomap_write_end(struct inode *inode, loff_t pos, size_t len,
669
-
size_t copied, struct page *page)
653
+
size_t copied, struct folio *folio)
670
654
{
671
-
flush_dcache_page(page);
655
+
struct iomap_page *iop = to_iomap_page(folio);
656
+
flush_dcache_folio(folio);
672
657
673
658
/*
674
659
* The blocks that were entirely written will now be uptodate, so we
···
682
665
* non-uptodate page as a zero-length write, and force the caller to
683
666
* redo the whole thing.
684
667
*/
685
-
if (unlikely(copied < len && !PageUptodate(page)))
668
+
if (unlikely(copied < len && !folio_test_uptodate(folio)))
686
669
return 0;
687
-
iomap_set_range_uptodate(page, offset_in_page(pos), len);
688
-
__set_page_dirty_nobuffers(page);
670
+
iomap_set_range_uptodate(folio, iop, offset_in_folio(folio, pos), len);
671
+
filemap_dirty_folio(inode->i_mapping, folio);
689
672
return copied;
690
673
}
691
674
692
675
static size_t iomap_write_end_inline(const struct iomap_iter *iter,
693
-
struct page *page, loff_t pos, size_t copied)
676
+
struct folio *folio, loff_t pos, size_t copied)
694
677
{
695
678
const struct iomap *iomap = &iter->iomap;
696
679
void *addr;
697
680
698
-
WARN_ON_ONCE(!PageUptodate(page));
681
+
WARN_ON_ONCE(!folio_test_uptodate(folio));
699
682
BUG_ON(!iomap_inline_data_valid(iomap));
700
683
701
-
flush_dcache_page(page);
702
-
addr = kmap_local_page(page) + pos;
684
+
flush_dcache_folio(folio);
685
+
addr = kmap_local_folio(folio, pos);
703
686
memcpy(iomap_inline_data(iomap, pos), addr, copied);
704
687
kunmap_local(addr);
705
688
···
709
692
710
693
/* Returns the number of bytes copied. May be 0. Cannot be an errno. */
711
694
static size_t iomap_write_end(struct iomap_iter *iter, loff_t pos, size_t len,
712
-
size_t copied, struct page *page)
695
+
size_t copied, struct folio *folio)
713
696
{
714
697
const struct iomap_page_ops *page_ops = iter->iomap.page_ops;
715
698
const struct iomap *srcmap = iomap_iter_srcmap(iter);
···
717
700
size_t ret;
718
701
719
702
if (srcmap->type == IOMAP_INLINE) {
720
-
ret = iomap_write_end_inline(iter, page, pos, copied);
703
+
ret = iomap_write_end_inline(iter, folio, pos, copied);
721
704
} else if (srcmap->flags & IOMAP_F_BUFFER_HEAD) {
722
705
ret = block_write_end(NULL, iter->inode->i_mapping, pos, len,
723
-
copied, page, NULL);
706
+
copied, &folio->page, NULL);
724
707
} else {
725
-
ret = __iomap_write_end(iter->inode, pos, len, copied, page);
708
+
ret = __iomap_write_end(iter->inode, pos, len, copied, folio);
726
709
}
727
710
728
711
/*
···
734
717
i_size_write(iter->inode, pos + ret);
735
718
iter->iomap.flags |= IOMAP_F_SIZE_CHANGED;
736
719
}
737
-
unlock_page(page);
720
+
folio_unlock(folio);
738
721
739
722
if (old_size < pos)
740
723
pagecache_isize_extended(iter->inode, old_size, pos);
741
724
if (page_ops && page_ops->page_done)
742
-
page_ops->page_done(iter->inode, pos, ret, page);
743
-
put_page(page);
725
+
page_ops->page_done(iter->inode, pos, ret, &folio->page);
726
+
folio_put(folio);
744
727
745
728
if (ret < len)
746
729
iomap_write_failed(iter->inode, pos, len);
···
755
738
long status = 0;
756
739
757
740
do {
741
+
struct folio *folio;
758
742
struct page *page;
759
743
unsigned long offset; /* Offset into pagecache page */
760
744
unsigned long bytes; /* Bytes to write to page */
···
779
761
break;
780
762
}
781
763
782
-
status = iomap_write_begin(iter, pos, bytes, &page);
764
+
status = iomap_write_begin(iter, pos, bytes, &folio);
783
765
if (unlikely(status))
784
766
break;
785
767
768
+
page = folio_file_page(folio, pos >> PAGE_SHIFT);
786
769
if (mapping_writably_mapped(iter->inode->i_mapping))
787
770
flush_dcache_page(page);
788
771
789
772
copied = copy_page_from_iter_atomic(page, offset, bytes, i);
790
773
791
-
status = iomap_write_end(iter, pos, bytes, copied, page);
774
+
status = iomap_write_end(iter, pos, bytes, copied, folio);
792
775
793
776
if (unlikely(copied != status))
794
777
iov_iter_revert(i, copied - status);
···
855
836
do {
856
837
unsigned long offset = offset_in_page(pos);
857
838
unsigned long bytes = min_t(loff_t, PAGE_SIZE - offset, length);
858
-
struct page *page;
839
+
struct folio *folio;
859
840
860
-
status = iomap_write_begin(iter, pos, bytes, &page);
841
+
status = iomap_write_begin(iter, pos, bytes, &folio);
861
842
if (unlikely(status))
862
843
return status;
863
844
864
-
status = iomap_write_end(iter, pos, bytes, bytes, page);
845
+
status = iomap_write_end(iter, pos, bytes, bytes, folio);
865
846
if (WARN_ON_ONCE(status == 0))
866
847
return -EIO;
867
848
···
895
876
}
896
877
EXPORT_SYMBOL_GPL(iomap_file_unshare);
897
878
898
-
static s64 __iomap_zero_iter(struct iomap_iter *iter, loff_t pos, u64 length)
899
-
{
900
-
struct page *page;
901
-
int status;
902
-
unsigned offset = offset_in_page(pos);
903
-
unsigned bytes = min_t(u64, PAGE_SIZE - offset, length);
904
-
905
-
status = iomap_write_begin(iter, pos, bytes, &page);
906
-
if (status)
907
-
return status;
908
-
909
-
zero_user(page, offset, bytes);
910
-
mark_page_accessed(page);
911
-
912
-
return iomap_write_end(iter, pos, bytes, bytes, page);
913
-
}
914
-
915
879
static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero)
916
880
{
917
881
struct iomap *iomap = &iter->iomap;
···
908
906
return length;
909
907
910
908
do {
911
-
s64 bytes;
909
+
struct folio *folio;
910
+
int status;
911
+
size_t offset;
912
+
size_t bytes = min_t(u64, SIZE_MAX, length);
912
913
913
-
if (IS_DAX(iter->inode))
914
-
bytes = dax_iomap_zero(pos, length, iomap);
915
-
else
916
-
bytes = __iomap_zero_iter(iter, pos, length);
917
-
if (bytes < 0)
918
-
return bytes;
914
+
if (IS_DAX(iter->inode)) {
915
+
s64 tmp = dax_iomap_zero(pos, bytes, iomap);
916
+
if (tmp < 0)
917
+
return tmp;
918
+
bytes = tmp;
919
+
goto good;
920
+
}
921
+
922
+
status = iomap_write_begin(iter, pos, bytes, &folio);
923
+
if (status)
924
+
return status;
925
+
926
+
offset = offset_in_folio(folio, pos);
927
+
if (bytes > folio_size(folio) - offset)
928
+
bytes = folio_size(folio) - offset;
929
+
930
+
folio_zero_range(folio, offset, bytes);
931
+
folio_mark_accessed(folio);
932
+
933
+
bytes = iomap_write_end(iter, pos, bytes, bytes, folio);
934
+
good:
935
+
if (WARN_ON_ONCE(bytes == 0))
936
+
return -EIO;
919
937
920
938
pos += bytes;
921
939
length -= bytes;
···
979
957
}
980
958
EXPORT_SYMBOL_GPL(iomap_truncate_page);
981
959
982
-
static loff_t iomap_page_mkwrite_iter(struct iomap_iter *iter,
983
-
struct page *page)
960
+
static loff_t iomap_folio_mkwrite_iter(struct iomap_iter *iter,
961
+
struct folio *folio)
984
962
{
985
963
loff_t length = iomap_length(iter);
986
964
int ret;
987
965
988
966
if (iter->iomap.flags & IOMAP_F_BUFFER_HEAD) {
989
-
ret = __block_write_begin_int(page, iter->pos, length, NULL,
967
+
ret = __block_write_begin_int(folio, iter->pos, length, NULL,
990
968
&iter->iomap);
991
969
if (ret)
992
970
return ret;
993
-
block_commit_write(page, 0, length);
971
+
block_commit_write(&folio->page, 0, length);
994
972
} else {
995
-
WARN_ON_ONCE(!PageUptodate(page));
996
-
set_page_dirty(page);
973
+
WARN_ON_ONCE(!folio_test_uptodate(folio));
974
+
folio_mark_dirty(folio);
997
975
}
998
976
999
977
return length;
···
1005
983
.inode = file_inode(vmf->vma->vm_file),
1006
984
.flags = IOMAP_WRITE | IOMAP_FAULT,
1007
985
};
1008
-
struct page *page = vmf->page;
986
+
struct folio *folio = page_folio(vmf->page);
1009
987
ssize_t ret;
1010
988
1011
-
lock_page(page);
1012
-
ret = page_mkwrite_check_truncate(page, iter.inode);
989
+
folio_lock(folio);
990
+
ret = folio_mkwrite_check_truncate(folio, iter.inode);
1013
991
if (ret < 0)
1014
992
goto out_unlock;
1015
-
iter.pos = page_offset(page);
993
+
iter.pos = folio_pos(folio);
1016
994
iter.len = ret;
1017
995
while ((ret = iomap_iter(&iter, ops)) > 0)
1018
-
iter.processed = iomap_page_mkwrite_iter(&iter, page);
996
+
iter.processed = iomap_folio_mkwrite_iter(&iter, folio);
1019
997
1020
998
if (ret < 0)
1021
999
goto out_unlock;
1022
-
wait_for_stable_page(page);
1000
+
folio_wait_stable(folio);
1023
1001
return VM_FAULT_LOCKED;
1024
1002
out_unlock:
1025
-
unlock_page(page);
1003
+
folio_unlock(folio);
1026
1004
return block_page_mkwrite_return(ret);
1027
1005
}
1028
1006
EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
1029
1007
1030
-
static void
1031
-
iomap_finish_page_writeback(struct inode *inode, struct page *page,
1032
-
int error, unsigned int len)
1008
+
static void iomap_finish_folio_write(struct inode *inode, struct folio *folio,
1009
+
size_t len, int error)
1033
1010
{
1034
-
struct iomap_page *iop = to_iomap_page(page);
1011
+
struct iomap_page *iop = to_iomap_page(folio);
1035
1012
1036
1013
if (error) {
1037
-
SetPageError(page);
1014
+
folio_set_error(folio);
1038
1015
mapping_set_error(inode->i_mapping, error);
1039
1016
}
1040
1017
1041
-
WARN_ON_ONCE(i_blocks_per_page(inode, page) > 1 && !iop);
1018
+
WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !iop);
1042
1019
WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) <= 0);
1043
1020
1044
1021
if (!iop || atomic_sub_and_test(len, &iop->write_bytes_pending))
1045
-
end_page_writeback(page);
1022
+
folio_end_writeback(folio);
1046
1023
}
1047
1024
1048
1025
/*
···
1060
1039
bool quiet = bio_flagged(bio, BIO_QUIET);
1061
1040
1062
1041
for (bio = &ioend->io_inline_bio; bio; bio = next) {
1063
-
struct bio_vec *bv;
1064
-
struct bvec_iter_all iter_all;
1042
+
struct folio_iter fi;
1065
1043
1066
1044
/*
1067
1045
* For the last bio, bi_private points to the ioend, so we
···
1071
1051
else
1072
1052
next = bio->bi_private;
1073
1053
1074
-
/* walk each page on bio, ending page IO on them */
1075
-
bio_for_each_segment_all(bv, bio, iter_all)
1076
-
iomap_finish_page_writeback(inode, bv->bv_page, error,
1077
-
bv->bv_len);
1054
+
/* walk all folios in bio, ending page IO on them */
1055
+
bio_for_each_folio_all(fi, bio)
1056
+
iomap_finish_folio_write(inode, fi.folio, fi.length,
1057
+
error);
1078
1058
bio_put(bio);
1079
1059
}
1080
1060
/* The ioend has been freed by bio_put() */
···
1269
1249
* first; otherwise finish off the current ioend and start another.
1270
1250
*/
1271
1251
static void
1272
-
iomap_add_to_ioend(struct inode *inode, loff_t offset, struct page *page,
1252
+
iomap_add_to_ioend(struct inode *inode, loff_t pos, struct folio *folio,
1273
1253
struct iomap_page *iop, struct iomap_writepage_ctx *wpc,
1274
1254
struct writeback_control *wbc, struct list_head *iolist)
1275
1255
{
1276
-
sector_t sector = iomap_sector(&wpc->iomap, offset);
1256
+
sector_t sector = iomap_sector(&wpc->iomap, pos);
1277
1257
unsigned len = i_blocksize(inode);
1278
-
unsigned poff = offset & (PAGE_SIZE - 1);
1258
+
size_t poff = offset_in_folio(folio, pos);
1279
1259
1280
-
if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, offset, sector)) {
1260
+
if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, pos, sector)) {
1281
1261
if (wpc->ioend)
1282
1262
list_add(&wpc->ioend->io_list, iolist);
1283
-
wpc->ioend = iomap_alloc_ioend(inode, wpc, offset, sector, wbc);
1263
+
wpc->ioend = iomap_alloc_ioend(inode, wpc, pos, sector, wbc);
1284
1264
}
1285
1265
1286
-
if (bio_add_page(wpc->ioend->io_bio, page, len, poff) != len) {
1266
+
if (!bio_add_folio(wpc->ioend->io_bio, folio, len, poff)) {
1287
1267
wpc->ioend->io_bio = iomap_chain_bio(wpc->ioend->io_bio);
1288
-
__bio_add_page(wpc->ioend->io_bio, page, len, poff);
1268
+
bio_add_folio(wpc->ioend->io_bio, folio, len, poff);
1289
1269
}
1290
1270
1291
1271
if (iop)
1292
1272
atomic_add(len, &iop->write_bytes_pending);
1293
1273
wpc->ioend->io_size += len;
1294
-
wbc_account_cgroup_owner(wbc, page, len);
1274
+
wbc_account_cgroup_owner(wbc, &folio->page, len);
1295
1275
}
1296
1276
1297
1277
/*
···
1313
1293
static int
1314
1294
iomap_writepage_map(struct iomap_writepage_ctx *wpc,
1315
1295
struct writeback_control *wbc, struct inode *inode,
1316
-
struct page *page, u64 end_offset)
1296
+
struct folio *folio, u64 end_pos)
1317
1297
{
1318
-
struct iomap_page *iop = iomap_page_create(inode, page);
1298
+
struct iomap_page *iop = iomap_page_create(inode, folio);
1319
1299
struct iomap_ioend *ioend, *next;
1320
1300
unsigned len = i_blocksize(inode);
1321
-
u64 file_offset; /* file offset of page */
1301
+
unsigned nblocks = i_blocks_per_folio(inode, folio);
1302
+
u64 pos = folio_pos(folio);
1322
1303
int error = 0, count = 0, i;
1323
1304
LIST_HEAD(submit_list);
1324
1305
1325
1306
WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) != 0);
1326
1307
1327
1308
/*
1328
-
* Walk through the page to find areas to write back. If we run off the
1329
-
* end of the current map or find the current map invalid, grab a new
1330
-
* one.
1309
+
* Walk through the folio to find areas to write back. If we
1310
+
* run off the end of the current map or find the current map
1311
+
* invalid, grab a new one.
1331
1312
*/
1332
-
for (i = 0, file_offset = page_offset(page);
1333
-
i < (PAGE_SIZE >> inode->i_blkbits) && file_offset < end_offset;
1334
-
i++, file_offset += len) {
1313
+
for (i = 0; i < nblocks && pos < end_pos; i++, pos += len) {
1335
1314
if (iop && !test_bit(i, iop->uptodate))
1336
1315
continue;
1337
1316
1338
-
error = wpc->ops->map_blocks(wpc, inode, file_offset);
1317
+
error = wpc->ops->map_blocks(wpc, inode, pos);
1339
1318
if (error)
1340
1319
break;
1341
1320
if (WARN_ON_ONCE(wpc->iomap.type == IOMAP_INLINE))
1342
1321
continue;
1343
1322
if (wpc->iomap.type == IOMAP_HOLE)
1344
1323
continue;
1345
-
iomap_add_to_ioend(inode, file_offset, page, iop, wpc, wbc,
1324
+
iomap_add_to_ioend(inode, pos, folio, iop, wpc, wbc,
1346
1325
&submit_list);
1347
1326
count++;
1348
1327
}
1349
1328
1350
1329
WARN_ON_ONCE(!wpc->ioend && !list_empty(&submit_list));
1351
-
WARN_ON_ONCE(!PageLocked(page));
1352
-
WARN_ON_ONCE(PageWriteback(page));
1353
-
WARN_ON_ONCE(PageDirty(page));
1330
+
WARN_ON_ONCE(!folio_test_locked(folio));
1331
+
WARN_ON_ONCE(folio_test_writeback(folio));
1332
+
WARN_ON_ONCE(folio_test_dirty(folio));
1354
1333
1355
1334
/*
1356
1335
* We cannot cancel the ioend directly here on error. We may have
···
1364
1345
* won't be affected by I/O completion and we must unlock it
1365
1346
* now.
1366
1347
*/
1367
-
if (wpc->ops->discard_page)
1368
-
wpc->ops->discard_page(page, file_offset);
1348
+
if (wpc->ops->discard_folio)
1349
+
wpc->ops->discard_folio(folio, pos);
1369
1350
if (!count) {
1370
-
ClearPageUptodate(page);
1371
-
unlock_page(page);
1351
+
folio_clear_uptodate(folio);
1352
+
folio_unlock(folio);
1372
1353
goto done;
1373
1354
}
1374
1355
}
1375
1356
1376
-
set_page_writeback(page);
1377
-
unlock_page(page);
1357
+
folio_start_writeback(folio);
1358
+
folio_unlock(folio);
1378
1359
1379
1360
/*
1380
1361
* Preserve the original error if there was one; catch
···
1395
1376
* with a partial page truncate on a sub-page block sized filesystem.
1396
1377
*/
1397
1378
if (!count)
1398
-
end_page_writeback(page);
1379
+
folio_end_writeback(folio);
1399
1380
done:
1400
-
mapping_set_error(page->mapping, error);
1381
+
mapping_set_error(folio->mapping, error);
1401
1382
return error;
1402
1383
}
1403
1384
···
1411
1392
static int
1412
1393
iomap_do_writepage(struct page *page, struct writeback_control *wbc, void *data)
1413
1394
{
1395
+
struct folio *folio = page_folio(page);
1414
1396
struct iomap_writepage_ctx *wpc = data;
1415
-
struct inode *inode = page->mapping->host;
1416
-
pgoff_t end_index;
1417
-
u64 end_offset;
1418
-
loff_t offset;
1397
+
struct inode *inode = folio->mapping->host;
1398
+
u64 end_pos, isize;
1419
1399
1420
-
trace_iomap_writepage(inode, page_offset(page), PAGE_SIZE);
1400
+
trace_iomap_writepage(inode, folio_pos(folio), folio_size(folio));
1421
1401
1422
1402
/*
1423
-
* Refuse to write the page out if we're called from reclaim context.
1403
+
* Refuse to write the folio out if we're called from reclaim context.
1424
1404
*
1425
1405
* This avoids stack overflows when called from deeply used stacks in
1426
1406
* random callers for direct reclaim or memcg reclaim. We explicitly
···
1433
1415
goto redirty;
1434
1416
1435
1417
/*
1436
-
* Is this page beyond the end of the file?
1418
+
* Is this folio beyond the end of the file?
1437
1419
*
1438
-
* The page index is less than the end_index, adjust the end_offset
1439
-
* to the highest offset that this page should represent.
1420
+
* The folio index is less than the end_index, adjust the end_pos
1421
+
* to the highest offset that this folio should represent.
1440
1422
* -----------------------------------------------------
1441
1423
* | file mapping | <EOF> |
1442
1424
* -----------------------------------------------------
···
1445
1427
* | desired writeback range | see else |
1446
1428
* ---------------------------------^------------------|
1447
1429
*/
1448
-
offset = i_size_read(inode);
1449
-
end_index = offset >> PAGE_SHIFT;
1450
-
if (page->index < end_index)
1451
-
end_offset = (loff_t)(page->index + 1) << PAGE_SHIFT;
1452
-
else {
1430
+
isize = i_size_read(inode);
1431
+
end_pos = folio_pos(folio) + folio_size(folio);
1432
+
if (end_pos > isize) {
1453
1433
/*
1454
1434
* Check whether the page to write out is beyond or straddles
1455
1435
* i_size or not.
···
1459
1443
* | | Straddles |
1460
1444
* ---------------------------------^-----------|--------|
1461
1445
*/
1462
-
unsigned offset_into_page = offset & (PAGE_SIZE - 1);
1446
+
size_t poff = offset_in_folio(folio, isize);
1447
+
pgoff_t end_index = isize >> PAGE_SHIFT;
1463
1448
1464
1449
/*
1465
1450
* Skip the page if it's fully outside i_size, e.g. due to a
···
1479
1462
* checking if the page is totally beyond i_size or if its
1480
1463
* offset is just equal to the EOF.
1481
1464
*/
1482
-
if (page->index > end_index ||
1483
-
(page->index == end_index && offset_into_page == 0))
1465
+
if (folio->index > end_index ||
1466
+
(folio->index == end_index && poff == 0))
1484
1467
goto redirty;
1485
1468
1486
1469
/*
···
1491
1474
* memory is zeroed when mapped, and writes to that region are
1492
1475
* not written out to the file."
1493
1476
*/
1494
-
zero_user_segment(page, offset_into_page, PAGE_SIZE);
1495
-
1496
-
/* Adjust the end_offset to the end of file */
1497
-
end_offset = offset;
1477
+
folio_zero_segment(folio, poff, folio_size(folio));
1478
+
end_pos = isize;
1498
1479
}
1499
1480
1500
-
return iomap_writepage_map(wpc, wbc, inode, page, end_offset);
1481
+
return iomap_writepage_map(wpc, wbc, inode, folio, end_pos);
1501
1482
1502
1483
redirty:
1503
-
redirty_page_for_writepage(wbc, page);
1504
-
unlock_page(page);
1484
+
folio_redirty_for_writepage(wbc, folio);
1485
+
folio_unlock(folio);
1505
1486
return 0;
1506
1487
}
1507
1488
+12
-12
fs/xfs/xfs_aops.c
+12
-12
fs/xfs/xfs_aops.c
···
437
437
* see a ENOSPC in writeback).
438
438
*/
439
439
static void
440
-
xfs_discard_page(
441
-
struct page *page,
442
-
loff_t fileoff)
440
+
xfs_discard_folio(
441
+
struct folio *folio,
442
+
loff_t pos)
443
443
{
444
-
struct inode *inode = page->mapping->host;
444
+
struct inode *inode = folio->mapping->host;
445
445
struct xfs_inode *ip = XFS_I(inode);
446
446
struct xfs_mount *mp = ip->i_mount;
447
-
unsigned int pageoff = offset_in_page(fileoff);
448
-
xfs_fileoff_t start_fsb = XFS_B_TO_FSBT(mp, fileoff);
449
-
xfs_fileoff_t pageoff_fsb = XFS_B_TO_FSBT(mp, pageoff);
447
+
size_t offset = offset_in_folio(folio, pos);
448
+
xfs_fileoff_t start_fsb = XFS_B_TO_FSBT(mp, pos);
449
+
xfs_fileoff_t pageoff_fsb = XFS_B_TO_FSBT(mp, offset);
450
450
int error;
451
451
452
452
if (xfs_is_shutdown(mp))
453
453
goto out_invalidate;
454
454
455
455
xfs_alert_ratelimited(mp,
456
-
"page discard on page "PTR_FMT", inode 0x%llx, offset %llu.",
457
-
page, ip->i_ino, fileoff);
456
+
"page discard on page "PTR_FMT", inode 0x%llx, pos %llu.",
457
+
folio, ip->i_ino, pos);
458
458
459
459
error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
460
-
i_blocks_per_page(inode, page) - pageoff_fsb);
460
+
i_blocks_per_folio(inode, folio) - pageoff_fsb);
461
461
if (error && !xfs_is_shutdown(mp))
462
462
xfs_alert(mp, "page discard unable to remove delalloc mapping.");
463
463
out_invalidate:
464
-
iomap_invalidatepage(page, pageoff, PAGE_SIZE - pageoff);
464
+
iomap_invalidate_folio(folio, offset, folio_size(folio) - offset);
465
465
}
466
466
467
467
static const struct iomap_writeback_ops xfs_writeback_ops = {
468
468
.map_blocks = xfs_map_blocks,
469
469
.prepare_ioend = xfs_prepare_ioend,
470
-
.discard_page = xfs_discard_page,
470
+
.discard_folio = xfs_discard_folio,
471
471
};
472
472
473
473
STATIC int
+2
fs/xfs/xfs_icache.c
+2
fs/xfs/xfs_icache.c
···
87
87
/* VFS doesn't initialise i_mode or i_state! */
88
88
VFS_I(ip)->i_mode = 0;
89
89
VFS_I(ip)->i_state = 0;
90
+
mapping_set_large_folios(VFS_I(ip)->i_mapping);
90
91
91
92
XFS_STATS_INC(mp, vn_active);
92
93
ASSERT(atomic_read(&ip->i_pincount) == 0);
···
321
320
inode->i_rdev = dev;
322
321
inode->i_uid = uid;
323
322
inode->i_gid = gid;
323
+
mapping_set_large_folios(inode->i_mapping);
324
324
return error;
325
325
}
326
326
+54
-2
include/linux/bio.h
+54
-2
include/linux/bio.h
···
166
166
*/
167
167
#define bio_for_each_bvec_all(bvl, bio, i) \
168
168
for (i = 0, bvl = bio_first_bvec_all(bio); \
169
-
i < (bio)->bi_vcnt; i++, bvl++) \
169
+
i < (bio)->bi_vcnt; i++, bvl++)
170
170
171
171
#define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
172
172
···
259
259
WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
260
260
return &bio->bi_io_vec[bio->bi_vcnt - 1];
261
261
}
262
+
263
+
/**
264
+
* struct folio_iter - State for iterating all folios in a bio.
265
+
* @folio: The current folio we're iterating. NULL after the last folio.
266
+
* @offset: The byte offset within the current folio.
267
+
* @length: The number of bytes in this iteration (will not cross folio
268
+
* boundary).
269
+
*/
270
+
struct folio_iter {
271
+
struct folio *folio;
272
+
size_t offset;
273
+
size_t length;
274
+
/* private: for use by the iterator */
275
+
size_t _seg_count;
276
+
int _i;
277
+
};
278
+
279
+
static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio,
280
+
int i)
281
+
{
282
+
struct bio_vec *bvec = bio_first_bvec_all(bio) + i;
283
+
284
+
fi->folio = page_folio(bvec->bv_page);
285
+
fi->offset = bvec->bv_offset +
286
+
PAGE_SIZE * (bvec->bv_page - &fi->folio->page);
287
+
fi->_seg_count = bvec->bv_len;
288
+
fi->length = min(folio_size(fi->folio) - fi->offset, fi->_seg_count);
289
+
fi->_i = i;
290
+
}
291
+
292
+
static inline void bio_next_folio(struct folio_iter *fi, struct bio *bio)
293
+
{
294
+
fi->_seg_count -= fi->length;
295
+
if (fi->_seg_count) {
296
+
fi->folio = folio_next(fi->folio);
297
+
fi->offset = 0;
298
+
fi->length = min(folio_size(fi->folio), fi->_seg_count);
299
+
} else if (fi->_i + 1 < bio->bi_vcnt) {
300
+
bio_first_folio(fi, bio, fi->_i + 1);
301
+
} else {
302
+
fi->folio = NULL;
303
+
}
304
+
}
305
+
306
+
/**
307
+
* bio_for_each_folio_all - Iterate over each folio in a bio.
308
+
* @fi: struct folio_iter which is updated for each folio.
309
+
* @bio: struct bio to iterate over.
310
+
*/
311
+
#define bio_for_each_folio_all(fi, bio) \
312
+
for (bio_first_folio(&fi, bio, 0); fi.folio; bio_next_folio(&fi, bio))
262
313
263
314
enum bip_flags {
264
315
BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */
···
460
409
extern void bio_reset(struct bio *);
461
410
void bio_chain(struct bio *, struct bio *);
462
411
463
-
extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
412
+
int bio_add_page(struct bio *, struct page *, unsigned len, unsigned off);
413
+
bool bio_add_folio(struct bio *, struct folio *, size_t len, size_t off);
464
414
extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
465
415
unsigned int, unsigned int);
466
416
int bio_add_zone_append_page(struct bio *bio, struct page *page,
+2
-1
include/linux/iomap.h
+2
-1
include/linux/iomap.h
···
225
225
int iomap_is_partially_uptodate(struct page *page, unsigned long from,
226
226
unsigned long count);
227
227
int iomap_releasepage(struct page *page, gfp_t gfp_mask);
228
+
void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len);
228
229
void iomap_invalidatepage(struct page *page, unsigned int offset,
229
230
unsigned int len);
230
231
#ifdef CONFIG_MIGRATION
···
285
284
* Optional, allows the file system to discard state on a page where
286
285
* we failed to submit any I/O.
287
286
*/
288
-
void (*discard_page)(struct page *page, loff_t fileoff);
287
+
void (*discard_folio)(struct folio *folio, loff_t pos);
289
288
};
290
289
291
290
struct iomap_writepage_ctx {