Merge tag 'f2fs-for-6.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs

+70 -10

Documentation/ABI/testing/sysfs-fs-f2fs

··· 49 49 Description: Controls the in-place-update policy. 50 50 updates in f2fs. User can set: 51 51 52 - ==== ================= 53 - 0x01 F2FS_IPU_FORCE 54 - 0x02 F2FS_IPU_SSR 55 - 0x04 F2FS_IPU_UTIL 56 - 0x08 F2FS_IPU_SSR_UTIL 57 - 0x10 F2FS_IPU_FSYNC 58 - 0x20 F2FS_IPU_ASYNC 59 - 0x40 F2FS_IPU_NOCACHE 60 - 0x80 F2FS_IPU_HONOR_OPU_WRITE 61 - ==== ================= 52 + ===== =============== =================================================== 53 + value policy description 54 + 0x00 DISABLE disable IPU(=default option in LFS mode) 55 + 0x01 FORCE all the time 56 + 0x02 SSR if SSR mode is activated 57 + 0x04 UTIL if FS utilization is over threashold 58 + 0x08 SSR_UTIL if SSR mode is activated and FS utilization is over 59 + threashold 60 + 0x10 FSYNC activated in fsync path only for high performance 61 + flash storages. IPU will be triggered only if the 62 + # of dirty pages over min_fsync_blocks. 63 + (=default option) 64 + 0x20 ASYNC do IPU given by asynchronous write requests 65 + 0x40 NOCACHE disable IPU bio cache 66 + 0x80 HONOR_OPU_WRITE use OPU write prior to IPU write if inode has 67 + FI_OPU_WRITE flag 68 + ===== =============== =================================================== 62 69 63 70 Refer segment.h for details. 64 71 ··· 676 669 Description: When DATA SEPARATION is on, it controls the age threshold to indicate 677 670 the data blocks as warm. By default it was initialized as 2621440 blocks 678 671 (equals to 10GB). 672 + 673 + What: /sys/fs/f2fs/<disk>/fault_rate 674 + Date: May 2016 675 + Contact: "Sheng Yong" <shengyong@oppo.com> 676 + Contact: "Chao Yu" <chao@kernel.org> 677 + Description: Enable fault injection in all supported types with 678 + specified injection rate. 679 + 680 + What: /sys/fs/f2fs/<disk>/fault_type 681 + Date: May 2016 682 + Contact: "Sheng Yong" <shengyong@oppo.com> 683 + Contact: "Chao Yu" <chao@kernel.org> 684 + Description: Support configuring fault injection type, should be 685 + enabled with fault_injection option, fault type value 686 + is shown below, it supports single or combined type. 687 + 688 + =================== =========== 689 + Type_Name Type_Value 690 + =================== =========== 691 + FAULT_KMALLOC 0x000000001 692 + FAULT_KVMALLOC 0x000000002 693 + FAULT_PAGE_ALLOC 0x000000004 694 + FAULT_PAGE_GET 0x000000008 695 + FAULT_ALLOC_BIO 0x000000010 (obsolete) 696 + FAULT_ALLOC_NID 0x000000020 697 + FAULT_ORPHAN 0x000000040 698 + FAULT_BLOCK 0x000000080 699 + FAULT_DIR_DEPTH 0x000000100 700 + FAULT_EVICT_INODE 0x000000200 701 + FAULT_TRUNCATE 0x000000400 702 + FAULT_READ_IO 0x000000800 703 + FAULT_CHECKPOINT 0x000001000 704 + FAULT_DISCARD 0x000002000 705 + FAULT_WRITE_IO 0x000004000 706 + FAULT_SLAB_ALLOC 0x000008000 707 + FAULT_DQUOT_INIT 0x000010000 708 + FAULT_LOCK_OP 0x000020000 709 + FAULT_BLKADDR 0x000040000 710 + =================== =========== 711 + 712 + What: /sys/fs/f2fs/<disk>/discard_io_aware_gran 713 + Date: January 2023 714 + Contact: "Yangtao Li" <frank.li@vivo.com> 715 + Description: Controls background discard granularity of inner discard thread 716 + when is not in idle. Inner thread will not issue discards with size that 717 + is smaller than granularity. The unit size is one block(4KB), now only 718 + support configuring in range of [0, 512]. 719 + Default: 512 720 + 721 + What: /sys/fs/f2fs/<disk>/last_age_weight 722 + Date: January 2023 723 + Contact: "Ping Xiong" <xiongping1@xiaomi.com> 724 + Description: When DATA SEPARATION is on, it controls the weight of last data block age.

+1 -1

Documentation/filesystems/f2fs.rst

··· 158 158 If this option is set, no cache_flush commands are issued 159 159 but f2fs still guarantees the write ordering of all the 160 160 data writes. 161 - barrier If this option is set, cache_flush commands are allowed to be 161 + barrier If this option is set, cache_flush commands are allowed to be 162 162 issued. 163 163 fastboot This option is used when a system wants to reduce mount 164 164 time as much as possible, even though normal performance

+1

MAINTAINERS

··· 7795 7795 L: linux-f2fs-devel@lists.sourceforge.net 7796 7796 S: Maintained 7797 7797 W: https://f2fs.wiki.kernel.org/ 7798 + Q: https://patchwork.kernel.org/project/f2fs/list/ 7798 7799 B: https://bugzilla.kernel.org/enter_bug.cgi?product=File%20System&component=f2fs 7799 7800 T: git git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git 7800 7801 F: Documentation/ABI/testing/sysfs-fs-f2fs

+16 -21

fs/f2fs/checkpoint.c

··· 70 70 .old_blkaddr = index, 71 71 .new_blkaddr = index, 72 72 .encrypted_page = NULL, 73 - .is_por = !is_meta, 73 + .is_por = !is_meta ? 1 : 0, 74 74 }; 75 75 int err; 76 76 ··· 171 171 bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi, 172 172 block_t blkaddr, int type) 173 173 { 174 - if (time_to_inject(sbi, FAULT_BLKADDR)) { 175 - f2fs_show_injection_info(sbi, FAULT_BLKADDR); 174 + if (time_to_inject(sbi, FAULT_BLKADDR)) 176 175 return false; 177 - } 178 176 179 177 switch (type) { 180 178 case META_NAT: ··· 237 239 .op = REQ_OP_READ, 238 240 .op_flags = sync ? (REQ_META | REQ_PRIO) : REQ_RAHEAD, 239 241 .encrypted_page = NULL, 240 - .in_list = false, 241 - .is_por = (type == META_POR), 242 + .in_list = 0, 243 + .is_por = (type == META_POR) ? 1 : 0, 242 244 }; 243 245 struct blk_plug plug; 244 246 int err; ··· 623 625 624 626 if (time_to_inject(sbi, FAULT_ORPHAN)) { 625 627 spin_unlock(&im->ino_lock); 626 - f2fs_show_injection_info(sbi, FAULT_ORPHAN); 627 628 return -ENOSPC; 628 629 } 629 630 ··· 795 798 */ 796 799 head = &im->ino_list; 797 800 798 - /* loop for each orphan inode entry and write them in Jornal block */ 801 + /* loop for each orphan inode entry and write them in journal block */ 799 802 list_for_each_entry(orphan, head, list) { 800 803 if (!page) { 801 804 page = f2fs_grab_meta_page(sbi, start_blk++); ··· 1125 1128 } else { 1126 1129 /* 1127 1130 * We should submit bio, since it exists several 1128 - * wribacking dentry pages in the freeing inode. 1131 + * writebacking dentry pages in the freeing inode. 1129 1132 */ 1130 1133 f2fs_submit_merged_write(sbi, DATA); 1131 1134 cond_resched(); ··· 1473 1476 ckpt->elapsed_time = cpu_to_le64(get_mtime(sbi, true)); 1474 1477 ckpt->free_segment_count = cpu_to_le32(free_segments(sbi)); 1475 1478 for (i = 0; i < NR_CURSEG_NODE_TYPE; i++) { 1476 - ckpt->cur_node_segno[i] = 1477 - cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_NODE)); 1478 - ckpt->cur_node_blkoff[i] = 1479 - cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_NODE)); 1480 - ckpt->alloc_type[i + CURSEG_HOT_NODE] = 1481 - curseg_alloc_type(sbi, i + CURSEG_HOT_NODE); 1479 + struct curseg_info *curseg = CURSEG_I(sbi, i + CURSEG_HOT_NODE); 1480 + 1481 + ckpt->cur_node_segno[i] = cpu_to_le32(curseg->segno); 1482 + ckpt->cur_node_blkoff[i] = cpu_to_le16(curseg->next_blkoff); 1483 + ckpt->alloc_type[i + CURSEG_HOT_NODE] = curseg->alloc_type; 1482 1484 } 1483 1485 for (i = 0; i < NR_CURSEG_DATA_TYPE; i++) { 1484 - ckpt->cur_data_segno[i] = 1485 - cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_DATA)); 1486 - ckpt->cur_data_blkoff[i] = 1487 - cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_DATA)); 1488 - ckpt->alloc_type[i + CURSEG_HOT_DATA] = 1489 - curseg_alloc_type(sbi, i + CURSEG_HOT_DATA); 1486 + struct curseg_info *curseg = CURSEG_I(sbi, i + CURSEG_HOT_DATA); 1487 + 1488 + ckpt->cur_data_segno[i] = cpu_to_le32(curseg->segno); 1489 + ckpt->cur_data_blkoff[i] = cpu_to_le16(curseg->next_blkoff); 1490 + ckpt->alloc_type[i + CURSEG_HOT_DATA] = curseg->alloc_type; 1490 1491 } 1491 1492 1492 1493 /* 2 cp + n data seg summary + orphan inode blocks */

+11 -13

fs/f2fs/compress.c

··· 241 241 unsigned int size = LZ4_MEM_COMPRESS; 242 242 243 243 #ifdef CONFIG_F2FS_FS_LZ4HC 244 - if (F2FS_I(cc->inode)->i_compress_flag >> COMPRESS_LEVEL_OFFSET) 244 + if (F2FS_I(cc->inode)->i_compress_level) 245 245 size = LZ4HC_MEM_COMPRESS; 246 246 #endif 247 247 ··· 267 267 #ifdef CONFIG_F2FS_FS_LZ4HC 268 268 static int lz4hc_compress_pages(struct compress_ctx *cc) 269 269 { 270 - unsigned char level = F2FS_I(cc->inode)->i_compress_flag >> 271 - COMPRESS_LEVEL_OFFSET; 270 + unsigned char level = F2FS_I(cc->inode)->i_compress_level; 272 271 int len; 273 272 274 273 if (level) ··· 339 340 zstd_cstream *stream; 340 341 void *workspace; 341 342 unsigned int workspace_size; 342 - unsigned char level = F2FS_I(cc->inode)->i_compress_flag >> 343 - COMPRESS_LEVEL_OFFSET; 343 + unsigned char level = F2FS_I(cc->inode)->i_compress_level; 344 344 345 345 if (!level) 346 346 level = F2FS_ZSTD_DEFAULT_CLEVEL; ··· 562 564 MODULE_PARM_DESC(num_compress_pages, 563 565 "Number of intermediate compress pages to preallocate"); 564 566 565 - int f2fs_init_compress_mempool(void) 567 + int __init f2fs_init_compress_mempool(void) 566 568 { 567 569 compress_page_pool = mempool_create_page_pool(num_compress_pages, 0); 568 570 return compress_page_pool ? 0 : -ENOMEM; ··· 688 690 vm_unmap_ram(cc->cbuf, cc->nr_cpages); 689 691 vm_unmap_ram(cc->rbuf, cc->cluster_size); 690 692 691 - for (i = 0; i < cc->nr_cpages; i++) { 692 - if (i < new_nr_cpages) 693 - continue; 693 + for (i = new_nr_cpages; i < cc->nr_cpages; i++) { 694 694 f2fs_compress_free_page(cc->cpages[i]); 695 695 cc->cpages[i] = NULL; 696 696 } ··· 1066 1070 if (ret) 1067 1071 goto out; 1068 1072 if (bio) 1069 - f2fs_submit_bio(sbi, bio, DATA); 1073 + f2fs_submit_read_bio(sbi, bio, DATA); 1070 1074 1071 1075 ret = f2fs_init_compress_ctx(cc); 1072 1076 if (ret) ··· 1211 1215 .page = NULL, 1212 1216 .encrypted_page = NULL, 1213 1217 .compressed_page = NULL, 1214 - .submitted = false, 1218 + .submitted = 0, 1215 1219 .io_type = io_type, 1216 1220 .io_wbc = wbc, 1217 - .encrypted = fscrypt_inode_uses_fs_layer_crypto(cc->inode), 1221 + .encrypted = fscrypt_inode_uses_fs_layer_crypto(cc->inode) ? 1222 + 1 : 0, 1218 1223 }; 1219 1224 struct dnode_of_data dn; 1220 1225 struct node_info ni; ··· 1225 1228 loff_t psize; 1226 1229 int i, err; 1227 1230 1228 - /* we should bypass data pages to proceed the kworkder jobs */ 1231 + /* we should bypass data pages to proceed the kworker jobs */ 1229 1232 if (unlikely(f2fs_cp_error(sbi))) { 1230 1233 mapping_set_error(cc->rpages[0]->mapping, -EIO); 1231 1234 goto out_free; ··· 1810 1813 const struct address_space_operations f2fs_compress_aops = { 1811 1814 .release_folio = f2fs_release_folio, 1812 1815 .invalidate_folio = f2fs_invalidate_folio, 1816 + .migrate_folio = filemap_migrate_folio, 1813 1817 }; 1814 1818 1815 1819 struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi)

+321 -319

fs/f2fs/data.c

··· 292 292 struct bio_post_read_ctx *ctx; 293 293 bool intask = in_task(); 294 294 295 - iostat_update_and_unbind_ctx(bio, 0); 295 + iostat_update_and_unbind_ctx(bio); 296 296 ctx = bio->bi_private; 297 297 298 - if (time_to_inject(sbi, FAULT_READ_IO)) { 299 - f2fs_show_injection_info(sbi, FAULT_READ_IO); 298 + if (time_to_inject(sbi, FAULT_READ_IO)) 300 299 bio->bi_status = BLK_STS_IOERR; 301 - } 302 300 303 301 if (bio->bi_status) { 304 302 f2fs_finish_read_bio(bio, intask); ··· 330 332 struct bio_vec *bvec; 331 333 struct bvec_iter_all iter_all; 332 334 333 - iostat_update_and_unbind_ctx(bio, 1); 335 + iostat_update_and_unbind_ctx(bio); 334 336 sbi = bio->bi_private; 335 337 336 - if (time_to_inject(sbi, FAULT_WRITE_IO)) { 337 - f2fs_show_injection_info(sbi, FAULT_WRITE_IO); 338 + if (time_to_inject(sbi, FAULT_WRITE_IO)) 338 339 bio->bi_status = BLK_STS_IOERR; 339 - } 340 340 341 341 bio_for_each_segment_all(bvec, bio, iter_all) { 342 342 struct page *page = bvec->bv_page; ··· 503 507 return fscrypt_mergeable_bio(bio, inode, next_idx); 504 508 } 505 509 506 - static inline void __submit_bio(struct f2fs_sb_info *sbi, 507 - struct bio *bio, enum page_type type) 510 + void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio, 511 + enum page_type type) 508 512 { 509 - if (!is_read_io(bio_op(bio))) { 510 - unsigned int start; 511 - 512 - if (type != DATA && type != NODE) 513 - goto submit_io; 514 - 515 - if (f2fs_lfs_mode(sbi) && current->plug) 516 - blk_finish_plug(current->plug); 517 - 518 - if (!F2FS_IO_ALIGNED(sbi)) 519 - goto submit_io; 520 - 521 - start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS; 522 - start %= F2FS_IO_SIZE(sbi); 523 - 524 - if (start == 0) 525 - goto submit_io; 526 - 527 - /* fill dummy pages */ 528 - for (; start < F2FS_IO_SIZE(sbi); start++) { 529 - struct page *page = 530 - mempool_alloc(sbi->write_io_dummy, 531 - GFP_NOIO | __GFP_NOFAIL); 532 - f2fs_bug_on(sbi, !page); 533 - 534 - lock_page(page); 535 - 536 - zero_user_segment(page, 0, PAGE_SIZE); 537 - set_page_private_dummy(page); 538 - 539 - if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) 540 - f2fs_bug_on(sbi, 1); 541 - } 542 - /* 543 - * In the NODE case, we lose next block address chain. So, we 544 - * need to do checkpoint in f2fs_sync_file. 545 - */ 546 - if (type == NODE) 547 - set_sbi_flag(sbi, SBI_NEED_CP); 548 - } 549 - submit_io: 550 - if (is_read_io(bio_op(bio))) 551 - trace_f2fs_submit_read_bio(sbi->sb, type, bio); 552 - else 553 - trace_f2fs_submit_write_bio(sbi->sb, type, bio); 513 + WARN_ON_ONCE(!is_read_io(bio_op(bio))); 514 + trace_f2fs_submit_read_bio(sbi->sb, type, bio); 554 515 555 516 iostat_update_submit_ctx(bio, type); 556 517 submit_bio(bio); 557 518 } 558 519 559 - void f2fs_submit_bio(struct f2fs_sb_info *sbi, 560 - struct bio *bio, enum page_type type) 520 + static void f2fs_align_write_bio(struct f2fs_sb_info *sbi, struct bio *bio) 561 521 { 562 - __submit_bio(sbi, bio, type); 522 + unsigned int start = 523 + (bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS) % F2FS_IO_SIZE(sbi); 524 + 525 + if (start == 0) 526 + return; 527 + 528 + /* fill dummy pages */ 529 + for (; start < F2FS_IO_SIZE(sbi); start++) { 530 + struct page *page = 531 + mempool_alloc(sbi->write_io_dummy, 532 + GFP_NOIO | __GFP_NOFAIL); 533 + f2fs_bug_on(sbi, !page); 534 + 535 + lock_page(page); 536 + 537 + zero_user_segment(page, 0, PAGE_SIZE); 538 + set_page_private_dummy(page); 539 + 540 + if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) 541 + f2fs_bug_on(sbi, 1); 542 + } 543 + } 544 + 545 + static void f2fs_submit_write_bio(struct f2fs_sb_info *sbi, struct bio *bio, 546 + enum page_type type) 547 + { 548 + WARN_ON_ONCE(is_read_io(bio_op(bio))); 549 + 550 + if (type == DATA || type == NODE) { 551 + if (f2fs_lfs_mode(sbi) && current->plug) 552 + blk_finish_plug(current->plug); 553 + 554 + if (F2FS_IO_ALIGNED(sbi)) { 555 + f2fs_align_write_bio(sbi, bio); 556 + /* 557 + * In the NODE case, we lose next block address chain. 558 + * So, we need to do checkpoint in f2fs_sync_file. 559 + */ 560 + if (type == NODE) 561 + set_sbi_flag(sbi, SBI_NEED_CP); 562 + } 563 + } 564 + 565 + trace_f2fs_submit_write_bio(sbi->sb, type, bio); 566 + iostat_update_submit_ctx(bio, type); 567 + submit_bio(bio); 563 568 } 564 569 565 570 static void __submit_merged_bio(struct f2fs_bio_info *io) ··· 570 573 if (!io->bio) 571 574 return; 572 575 573 - if (is_read_io(fio->op)) 576 + if (is_read_io(fio->op)) { 574 577 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); 575 - else 578 + f2fs_submit_read_bio(io->sbi, io->bio, fio->type); 579 + } else { 576 580 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); 577 - 578 - __submit_bio(io->sbi, io->bio, fio->type); 581 + f2fs_submit_write_bio(io->sbi, io->bio, fio->type); 582 + } 579 583 io->bio = NULL; 580 584 } 581 585 ··· 653 655 654 656 f2fs_down_write(&io->io_rwsem); 655 657 658 + if (!io->bio) 659 + goto unlock_out; 660 + 656 661 /* change META to META_FLUSH in the checkpoint procedure */ 657 662 if (type >= META_FLUSH) { 658 663 io->fio.type = META_FLUSH; ··· 664 663 io->bio->bi_opf |= REQ_PREFLUSH | REQ_FUA; 665 664 } 666 665 __submit_merged_bio(io); 666 + unlock_out: 667 667 f2fs_up_write(&io->io_rwsem); 668 668 } 669 669 ··· 743 741 } 744 742 745 743 if (fio->io_wbc && !is_read_io(fio->op)) 746 - wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE); 744 + wbc_account_cgroup_owner(fio->io_wbc, fio->page, PAGE_SIZE); 747 745 748 746 inc_page_count(fio->sbi, is_read_io(fio->op) ? 749 747 __read_io_type(page) : WB_DATA_TYPE(fio->page)); 750 748 751 - __submit_bio(fio->sbi, bio, fio->type); 749 + if (is_read_io(bio_op(bio))) 750 + f2fs_submit_read_bio(fio->sbi, bio, fio->type); 751 + else 752 + f2fs_submit_write_bio(fio->sbi, bio, fio->type); 752 753 return 0; 753 754 } 754 755 ··· 853 848 854 849 /* page can't be merged into bio; submit the bio */ 855 850 del_bio_entry(be); 856 - __submit_bio(sbi, *bio, DATA); 851 + f2fs_submit_write_bio(sbi, *bio, DATA); 857 852 break; 858 853 } 859 854 f2fs_up_write(&io->bio_list_lock); ··· 916 911 } 917 912 918 913 if (found) 919 - __submit_bio(sbi, target, DATA); 914 + f2fs_submit_write_bio(sbi, target, DATA); 920 915 if (bio && *bio) { 921 916 bio_put(*bio); 922 917 *bio = NULL; ··· 953 948 } 954 949 955 950 if (fio->io_wbc) 956 - wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE); 951 + wbc_account_cgroup_owner(fio->io_wbc, fio->page, PAGE_SIZE); 957 952 958 953 inc_page_count(fio->sbi, WB_DATA_TYPE(page)); 959 954 ··· 996 991 bio_page = fio->page; 997 992 998 993 /* set submitted = true as a return value */ 999 - fio->submitted = true; 994 + fio->submitted = 1; 1000 995 1001 996 inc_page_count(sbi, WB_DATA_TYPE(bio_page)); 1002 997 ··· 1012 1007 (fio->type == DATA || fio->type == NODE) && 1013 1008 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) { 1014 1009 dec_page_count(sbi, WB_DATA_TYPE(bio_page)); 1015 - fio->retry = true; 1010 + fio->retry = 1; 1016 1011 goto skip; 1017 1012 } 1018 1013 io->bio = __bio_alloc(fio, BIO_MAX_VECS); ··· 1027 1022 } 1028 1023 1029 1024 if (fio->io_wbc) 1030 - wbc_account_cgroup_owner(fio->io_wbc, bio_page, PAGE_SIZE); 1025 + wbc_account_cgroup_owner(fio->io_wbc, fio->page, PAGE_SIZE); 1031 1026 1032 1027 io->last_block_in_bio = fio->new_blkaddr; 1033 1028 ··· 1112 1107 } 1113 1108 inc_page_count(sbi, F2FS_RD_DATA); 1114 1109 f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE); 1115 - __submit_bio(sbi, bio, DATA); 1110 + f2fs_submit_read_bio(sbi, bio, DATA); 1116 1111 return 0; 1117 1112 } 1118 1113 ··· 1212 1207 return err; 1213 1208 } 1214 1209 1215 - int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index) 1216 - { 1217 - struct extent_info ei = {0, }; 1218 - struct inode *inode = dn->inode; 1219 - 1220 - if (f2fs_lookup_read_extent_cache(inode, index, &ei)) { 1221 - dn->data_blkaddr = ei.blk + index - ei.fofs; 1222 - return 0; 1223 - } 1224 - 1225 - return f2fs_reserve_block(dn, index); 1226 - } 1227 - 1228 1210 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index, 1229 1211 blk_opf_t op_flags, bool for_write, 1230 1212 pgoff_t *next_pgofs) ··· 1219 1227 struct address_space *mapping = inode->i_mapping; 1220 1228 struct dnode_of_data dn; 1221 1229 struct page *page; 1222 - struct extent_info ei = {0, }; 1223 1230 int err; 1224 1231 1225 1232 page = f2fs_grab_cache_page(mapping, index, for_write); 1226 1233 if (!page) 1227 1234 return ERR_PTR(-ENOMEM); 1228 1235 1229 - if (f2fs_lookup_read_extent_cache(inode, index, &ei)) { 1230 - dn.data_blkaddr = ei.blk + index - ei.fofs; 1236 + if (f2fs_lookup_read_extent_cache_block(inode, index, 1237 + &dn.data_blkaddr)) { 1231 1238 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr, 1232 1239 DATA_GENERIC_ENHANCE_READ)) { 1233 1240 err = -EFSCORRUPTED; ··· 1423 1432 return err; 1424 1433 1425 1434 dn->data_blkaddr = f2fs_data_blkaddr(dn); 1426 - if (dn->data_blkaddr != NULL_ADDR) 1427 - goto alloc; 1435 + if (dn->data_blkaddr == NULL_ADDR) { 1436 + err = inc_valid_block_count(sbi, dn->inode, &count); 1437 + if (unlikely(err)) 1438 + return err; 1439 + } 1428 1440 1429 - if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count)))) 1430 - return err; 1431 - 1432 - alloc: 1433 1441 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 1434 1442 old_blkaddr = dn->data_blkaddr; 1435 1443 f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr, ··· 1442 1452 return 0; 1443 1453 } 1444 1454 1445 - void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock) 1455 + static void f2fs_map_lock(struct f2fs_sb_info *sbi, int flag) 1446 1456 { 1447 - if (flag == F2FS_GET_BLOCK_PRE_AIO) { 1448 - if (lock) 1449 - f2fs_down_read(&sbi->node_change); 1450 - else 1451 - f2fs_up_read(&sbi->node_change); 1457 + if (flag == F2FS_GET_BLOCK_PRE_AIO) 1458 + f2fs_down_read(&sbi->node_change); 1459 + else 1460 + f2fs_lock_op(sbi); 1461 + } 1462 + 1463 + static void f2fs_map_unlock(struct f2fs_sb_info *sbi, int flag) 1464 + { 1465 + if (flag == F2FS_GET_BLOCK_PRE_AIO) 1466 + f2fs_up_read(&sbi->node_change); 1467 + else 1468 + f2fs_unlock_op(sbi); 1469 + } 1470 + 1471 + int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index) 1472 + { 1473 + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1474 + int err = 0; 1475 + 1476 + f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 1477 + if (!f2fs_lookup_read_extent_cache_block(dn->inode, index, 1478 + &dn->data_blkaddr)) 1479 + err = f2fs_reserve_block(dn, index); 1480 + f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO); 1481 + 1482 + return err; 1483 + } 1484 + 1485 + static int f2fs_map_no_dnode(struct inode *inode, 1486 + struct f2fs_map_blocks *map, struct dnode_of_data *dn, 1487 + pgoff_t pgoff) 1488 + { 1489 + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1490 + 1491 + /* 1492 + * There is one exceptional case that read_node_page() may return 1493 + * -ENOENT due to filesystem has been shutdown or cp_error, return 1494 + * -EIO in that case. 1495 + */ 1496 + if (map->m_may_create && 1497 + (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || f2fs_cp_error(sbi))) 1498 + return -EIO; 1499 + 1500 + if (map->m_next_pgofs) 1501 + *map->m_next_pgofs = f2fs_get_next_page_offset(dn, pgoff); 1502 + if (map->m_next_extent) 1503 + *map->m_next_extent = f2fs_get_next_page_offset(dn, pgoff); 1504 + return 0; 1505 + } 1506 + 1507 + static bool f2fs_map_blocks_cached(struct inode *inode, 1508 + struct f2fs_map_blocks *map, int flag) 1509 + { 1510 + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1511 + unsigned int maxblocks = map->m_len; 1512 + pgoff_t pgoff = (pgoff_t)map->m_lblk; 1513 + struct extent_info ei = {}; 1514 + 1515 + if (!f2fs_lookup_read_extent_cache(inode, pgoff, &ei)) 1516 + return false; 1517 + 1518 + map->m_pblk = ei.blk + pgoff - ei.fofs; 1519 + map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgoff); 1520 + map->m_flags = F2FS_MAP_MAPPED; 1521 + if (map->m_next_extent) 1522 + *map->m_next_extent = pgoff + map->m_len; 1523 + 1524 + /* for hardware encryption, but to avoid potential issue in future */ 1525 + if (flag == F2FS_GET_BLOCK_DIO) 1526 + f2fs_wait_on_block_writeback_range(inode, 1527 + map->m_pblk, map->m_len); 1528 + 1529 + if (f2fs_allow_multi_device_dio(sbi, flag)) { 1530 + int bidx = f2fs_target_device_index(sbi, map->m_pblk); 1531 + struct f2fs_dev_info *dev = &sbi->devs[bidx]; 1532 + 1533 + map->m_bdev = dev->bdev; 1534 + map->m_pblk -= dev->start_blk; 1535 + map->m_len = min(map->m_len, dev->end_blk + 1 - map->m_pblk); 1452 1536 } else { 1453 - if (lock) 1454 - f2fs_lock_op(sbi); 1455 - else 1456 - f2fs_unlock_op(sbi); 1537 + map->m_bdev = inode->i_sb->s_bdev; 1457 1538 } 1539 + return true; 1458 1540 } 1459 1541 1460 1542 /* ··· 1534 1472 * maps continuous logical blocks to physical blocks, and return such 1535 1473 * info via f2fs_map_blocks structure. 1536 1474 */ 1537 - int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, 1538 - int create, int flag) 1475 + int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) 1539 1476 { 1540 1477 unsigned int maxblocks = map->m_len; 1541 1478 struct dnode_of_data dn; ··· 1544 1483 int err = 0, ofs = 1; 1545 1484 unsigned int ofs_in_node, last_ofs_in_node; 1546 1485 blkcnt_t prealloc; 1547 - struct extent_info ei = {0, }; 1548 1486 block_t blkaddr; 1549 1487 unsigned int start_pgofs; 1550 1488 int bidx = 0; 1489 + bool is_hole; 1551 1490 1552 1491 if (!maxblocks) 1553 1492 return 0; 1493 + 1494 + if (!map->m_may_create && f2fs_map_blocks_cached(inode, map, flag)) 1495 + goto out; 1554 1496 1555 1497 map->m_bdev = inode->i_sb->s_bdev; 1556 1498 map->m_multidev_dio = ··· 1566 1502 pgofs = (pgoff_t)map->m_lblk; 1567 1503 end = pgofs + maxblocks; 1568 1504 1569 - if (!create && f2fs_lookup_read_extent_cache(inode, pgofs, &ei)) { 1570 - if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO && 1571 - map->m_may_create) 1572 - goto next_dnode; 1573 - 1574 - map->m_pblk = ei.blk + pgofs - ei.fofs; 1575 - map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs); 1576 - map->m_flags = F2FS_MAP_MAPPED; 1577 - if (map->m_next_extent) 1578 - *map->m_next_extent = pgofs + map->m_len; 1579 - 1580 - /* for hardware encryption, but to avoid potential issue in future */ 1581 - if (flag == F2FS_GET_BLOCK_DIO) 1582 - f2fs_wait_on_block_writeback_range(inode, 1583 - map->m_pblk, map->m_len); 1584 - 1585 - if (map->m_multidev_dio) { 1586 - block_t blk_addr = map->m_pblk; 1587 - 1588 - bidx = f2fs_target_device_index(sbi, map->m_pblk); 1589 - 1590 - map->m_bdev = FDEV(bidx).bdev; 1591 - map->m_pblk -= FDEV(bidx).start_blk; 1592 - map->m_len = min(map->m_len, 1593 - FDEV(bidx).end_blk + 1 - map->m_pblk); 1594 - 1595 - if (map->m_may_create) 1596 - f2fs_update_device_state(sbi, inode->i_ino, 1597 - blk_addr, map->m_len); 1598 - } 1599 - goto out; 1600 - } 1601 - 1602 1505 next_dnode: 1603 1506 if (map->m_may_create) 1604 - f2fs_do_map_lock(sbi, flag, true); 1507 + f2fs_map_lock(sbi, flag); 1605 1508 1606 1509 /* When reading holes, we need its node page */ 1607 1510 set_new_dnode(&dn, inode, NULL, NULL, 0); ··· 1576 1545 if (err) { 1577 1546 if (flag == F2FS_GET_BLOCK_BMAP) 1578 1547 map->m_pblk = 0; 1579 - 1580 - if (err == -ENOENT) { 1581 - /* 1582 - * There is one exceptional case that read_node_page() 1583 - * may return -ENOENT due to filesystem has been 1584 - * shutdown or cp_error, so force to convert error 1585 - * number to EIO for such case. 1586 - */ 1587 - if (map->m_may_create && 1588 - (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || 1589 - f2fs_cp_error(sbi))) { 1590 - err = -EIO; 1591 - goto unlock_out; 1592 - } 1593 - 1594 - err = 0; 1595 - if (map->m_next_pgofs) 1596 - *map->m_next_pgofs = 1597 - f2fs_get_next_page_offset(&dn, pgofs); 1598 - if (map->m_next_extent) 1599 - *map->m_next_extent = 1600 - f2fs_get_next_page_offset(&dn, pgofs); 1601 - } 1548 + if (err == -ENOENT) 1549 + err = f2fs_map_no_dnode(inode, map, &dn, pgofs); 1602 1550 goto unlock_out; 1603 1551 } 1604 1552 ··· 1588 1578 1589 1579 next_block: 1590 1580 blkaddr = f2fs_data_blkaddr(&dn); 1591 - 1592 - if (__is_valid_data_blkaddr(blkaddr) && 1593 - !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) { 1581 + is_hole = !__is_valid_data_blkaddr(blkaddr); 1582 + if (!is_hole && 1583 + !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) { 1594 1584 err = -EFSCORRUPTED; 1595 1585 f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); 1596 1586 goto sync_out; 1597 1587 } 1598 1588 1599 - if (__is_valid_data_blkaddr(blkaddr)) { 1600 - /* use out-place-update for driect IO under LFS mode */ 1601 - if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO && 1602 - map->m_may_create) { 1589 + /* use out-place-update for direct IO under LFS mode */ 1590 + if (map->m_may_create && 1591 + (is_hole || (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO))) { 1592 + if (unlikely(f2fs_cp_error(sbi))) { 1593 + err = -EIO; 1594 + goto sync_out; 1595 + } 1596 + 1597 + switch (flag) { 1598 + case F2FS_GET_BLOCK_PRE_AIO: 1599 + if (blkaddr == NULL_ADDR) { 1600 + prealloc++; 1601 + last_ofs_in_node = dn.ofs_in_node; 1602 + } 1603 + break; 1604 + case F2FS_GET_BLOCK_PRE_DIO: 1605 + case F2FS_GET_BLOCK_DIO: 1603 1606 err = __allocate_data_block(&dn, map->m_seg_type); 1604 1607 if (err) 1605 1608 goto sync_out; 1606 - blkaddr = dn.data_blkaddr; 1609 + if (flag == F2FS_GET_BLOCK_PRE_DIO) 1610 + file_need_truncate(inode); 1607 1611 set_inode_flag(inode, FI_APPEND_WRITE); 1612 + break; 1613 + default: 1614 + WARN_ON_ONCE(1); 1615 + err = -EIO; 1616 + goto sync_out; 1608 1617 } 1609 - } else { 1610 - if (create) { 1611 - if (unlikely(f2fs_cp_error(sbi))) { 1612 - err = -EIO; 1613 - goto sync_out; 1614 - } 1615 - if (flag == F2FS_GET_BLOCK_PRE_AIO) { 1616 - if (blkaddr == NULL_ADDR) { 1617 - prealloc++; 1618 - last_ofs_in_node = dn.ofs_in_node; 1619 - } 1620 - } else { 1621 - WARN_ON(flag != F2FS_GET_BLOCK_PRE_DIO && 1622 - flag != F2FS_GET_BLOCK_DIO); 1623 - err = __allocate_data_block(&dn, 1624 - map->m_seg_type); 1625 - if (!err) { 1626 - if (flag == F2FS_GET_BLOCK_PRE_DIO) 1627 - file_need_truncate(inode); 1628 - set_inode_flag(inode, FI_APPEND_WRITE); 1629 - } 1630 - } 1631 - if (err) 1632 - goto sync_out; 1618 + 1619 + blkaddr = dn.data_blkaddr; 1620 + if (is_hole) 1633 1621 map->m_flags |= F2FS_MAP_NEW; 1634 - blkaddr = dn.data_blkaddr; 1635 - } else { 1636 - if (f2fs_compressed_file(inode) && 1637 - f2fs_sanity_check_cluster(&dn) && 1638 - (flag != F2FS_GET_BLOCK_FIEMAP || 1639 - IS_ENABLED(CONFIG_F2FS_CHECK_FS))) { 1640 - err = -EFSCORRUPTED; 1641 - f2fs_handle_error(sbi, 1642 - ERROR_CORRUPTED_CLUSTER); 1643 - goto sync_out; 1644 - } 1645 - if (flag == F2FS_GET_BLOCK_BMAP) { 1646 - map->m_pblk = 0; 1647 - goto sync_out; 1648 - } 1649 - if (flag == F2FS_GET_BLOCK_PRECACHE) 1650 - goto sync_out; 1651 - if (flag == F2FS_GET_BLOCK_FIEMAP && 1652 - blkaddr == NULL_ADDR) { 1622 + } else if (is_hole) { 1623 + if (f2fs_compressed_file(inode) && 1624 + f2fs_sanity_check_cluster(&dn) && 1625 + (flag != F2FS_GET_BLOCK_FIEMAP || 1626 + IS_ENABLED(CONFIG_F2FS_CHECK_FS))) { 1627 + err = -EFSCORRUPTED; 1628 + f2fs_handle_error(sbi, 1629 + ERROR_CORRUPTED_CLUSTER); 1630 + goto sync_out; 1631 + } 1632 + 1633 + switch (flag) { 1634 + case F2FS_GET_BLOCK_PRECACHE: 1635 + goto sync_out; 1636 + case F2FS_GET_BLOCK_BMAP: 1637 + map->m_pblk = 0; 1638 + goto sync_out; 1639 + case F2FS_GET_BLOCK_FIEMAP: 1640 + if (blkaddr == NULL_ADDR) { 1653 1641 if (map->m_next_pgofs) 1654 1642 *map->m_next_pgofs = pgofs + 1; 1655 1643 goto sync_out; 1656 1644 } 1657 - if (flag != F2FS_GET_BLOCK_FIEMAP) { 1658 - /* for defragment case */ 1659 - if (map->m_next_pgofs) 1660 - *map->m_next_pgofs = pgofs + 1; 1661 - goto sync_out; 1662 - } 1645 + break; 1646 + default: 1647 + /* for defragment case */ 1648 + if (map->m_next_pgofs) 1649 + *map->m_next_pgofs = pgofs + 1; 1650 + goto sync_out; 1663 1651 } 1664 1652 } 1665 1653 ··· 1668 1660 bidx = f2fs_target_device_index(sbi, blkaddr); 1669 1661 1670 1662 if (map->m_len == 0) { 1671 - /* preallocated unwritten block should be mapped for fiemap. */ 1663 + /* reserved delalloc block should be mapped for fiemap. */ 1672 1664 if (blkaddr == NEW_ADDR) 1673 - map->m_flags |= F2FS_MAP_UNWRITTEN; 1665 + map->m_flags |= F2FS_MAP_DELALLOC; 1674 1666 map->m_flags |= F2FS_MAP_MAPPED; 1675 1667 1676 1668 map->m_pblk = blkaddr; ··· 1729 1721 f2fs_put_dnode(&dn); 1730 1722 1731 1723 if (map->m_may_create) { 1732 - f2fs_do_map_lock(sbi, flag, false); 1724 + f2fs_map_unlock(sbi, flag); 1733 1725 f2fs_balance_fs(sbi, dn.node_changed); 1734 1726 } 1735 1727 goto next_dnode; ··· 1775 1767 f2fs_put_dnode(&dn); 1776 1768 unlock_out: 1777 1769 if (map->m_may_create) { 1778 - f2fs_do_map_lock(sbi, flag, false); 1770 + f2fs_map_unlock(sbi, flag); 1779 1771 f2fs_balance_fs(sbi, dn.node_changed); 1780 1772 } 1781 1773 out: 1782 - trace_f2fs_map_blocks(inode, map, create, flag, err); 1774 + trace_f2fs_map_blocks(inode, map, flag, err); 1783 1775 return err; 1784 1776 } 1785 1777 ··· 1801 1793 1802 1794 while (map.m_lblk < last_lblk) { 1803 1795 map.m_len = last_lblk - map.m_lblk; 1804 - err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT); 1796 + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT); 1805 1797 if (err || map.m_len == 0) 1806 1798 return false; 1807 1799 map.m_lblk += map.m_len; ··· 1975 1967 map.m_len = cluster_size - count_in_cluster; 1976 1968 } 1977 1969 1978 - ret = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_FIEMAP); 1970 + ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP); 1979 1971 if (ret) 1980 1972 goto out; 1981 1973 ··· 1992 1984 1993 1985 compr_appended = false; 1994 1986 /* In a case of compressed cluster, append this to the last extent */ 1995 - if (compr_cluster && ((map.m_flags & F2FS_MAP_UNWRITTEN) || 1987 + if (compr_cluster && ((map.m_flags & F2FS_MAP_DELALLOC) || 1996 1988 !(map.m_flags & F2FS_MAP_FLAGS))) { 1997 1989 compr_appended = true; 1998 1990 goto skip_fill; ··· 2038 2030 compr_cluster = false; 2039 2031 size += blks_to_bytes(inode, 1); 2040 2032 } 2041 - } else if (map.m_flags & F2FS_MAP_UNWRITTEN) { 2033 + } else if (map.m_flags & F2FS_MAP_DELALLOC) { 2042 2034 flags = FIEMAP_EXTENT_UNWRITTEN; 2043 2035 } 2044 2036 ··· 2107 2099 map->m_lblk = block_in_file; 2108 2100 map->m_len = last_block - block_in_file; 2109 2101 2110 - ret = f2fs_map_blocks(inode, map, 0, F2FS_GET_BLOCK_DEFAULT); 2102 + ret = f2fs_map_blocks(inode, map, F2FS_GET_BLOCK_DEFAULT); 2111 2103 if (ret) 2112 2104 goto out; 2113 2105 got_it: ··· 2144 2136 *last_block_in_bio, block_nr) || 2145 2137 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) { 2146 2138 submit_and_realloc: 2147 - __submit_bio(F2FS_I_SB(inode), bio, DATA); 2139 + f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); 2148 2140 bio = NULL; 2149 2141 } 2150 2142 if (bio == NULL) { ··· 2291 2283 *last_block_in_bio, blkaddr) || 2292 2284 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) { 2293 2285 submit_and_realloc: 2294 - __submit_bio(sbi, bio, DATA); 2286 + f2fs_submit_read_bio(sbi, bio, DATA); 2295 2287 bio = NULL; 2296 2288 } 2297 2289 ··· 2385 2377 2386 2378 #ifdef CONFIG_F2FS_FS_COMPRESSION 2387 2379 if (f2fs_compressed_file(inode)) { 2388 - /* there are remained comressed pages, submit them */ 2380 + /* there are remained compressed pages, submit them */ 2389 2381 if (!f2fs_cluster_can_merge_page(&cc, page->index)) { 2390 2382 ret = f2fs_read_multi_pages(&cc, &bio, 2391 2383 max_nr_pages, ··· 2452 2444 #endif 2453 2445 } 2454 2446 if (bio) 2455 - __submit_bio(F2FS_I_SB(inode), bio, DATA); 2447 + f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); 2456 2448 return ret; 2457 2449 } 2458 2450 ··· 2538 2530 struct f2fs_io_info *fio) 2539 2531 { 2540 2532 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2541 - unsigned int policy = SM_I(sbi)->ipu_policy; 2542 2533 2543 - if (policy & (0x1 << F2FS_IPU_HONOR_OPU_WRITE) && 2544 - is_inode_flag_set(inode, FI_OPU_WRITE)) 2534 + if (IS_F2FS_IPU_HONOR_OPU_WRITE(sbi) && 2535 + is_inode_flag_set(inode, FI_OPU_WRITE)) 2545 2536 return false; 2546 - if (policy & (0x1 << F2FS_IPU_FORCE)) 2537 + if (IS_F2FS_IPU_FORCE(sbi)) 2547 2538 return true; 2548 - if (policy & (0x1 << F2FS_IPU_SSR) && f2fs_need_SSR(sbi)) 2539 + if (IS_F2FS_IPU_SSR(sbi) && f2fs_need_SSR(sbi)) 2549 2540 return true; 2550 - if (policy & (0x1 << F2FS_IPU_UTIL) && 2551 - utilization(sbi) > SM_I(sbi)->min_ipu_util) 2541 + if (IS_F2FS_IPU_UTIL(sbi) && utilization(sbi) > SM_I(sbi)->min_ipu_util) 2552 2542 return true; 2553 - if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && f2fs_need_SSR(sbi) && 2554 - utilization(sbi) > SM_I(sbi)->min_ipu_util) 2543 + if (IS_F2FS_IPU_SSR_UTIL(sbi) && f2fs_need_SSR(sbi) && 2544 + utilization(sbi) > SM_I(sbi)->min_ipu_util) 2555 2545 return true; 2556 2546 2557 2547 /* 2558 2548 * IPU for rewrite async pages 2559 2549 */ 2560 - if (policy & (0x1 << F2FS_IPU_ASYNC) && 2561 - fio && fio->op == REQ_OP_WRITE && 2562 - !(fio->op_flags & REQ_SYNC) && 2563 - !IS_ENCRYPTED(inode)) 2550 + if (IS_F2FS_IPU_ASYNC(sbi) && fio && fio->op == REQ_OP_WRITE && 2551 + !(fio->op_flags & REQ_SYNC) && !IS_ENCRYPTED(inode)) 2564 2552 return true; 2565 2553 2566 2554 /* this is only set during fdatasync */ 2567 - if (policy & (0x1 << F2FS_IPU_FSYNC) && 2568 - is_inode_flag_set(inode, FI_NEED_IPU)) 2555 + if (IS_F2FS_IPU_FSYNC(sbi) && is_inode_flag_set(inode, FI_NEED_IPU)) 2569 2556 return true; 2570 2557 2571 2558 if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) && ··· 2638 2635 struct page *page = fio->page; 2639 2636 struct inode *inode = page->mapping->host; 2640 2637 struct dnode_of_data dn; 2641 - struct extent_info ei = {0, }; 2642 2638 struct node_info ni; 2643 2639 bool ipu_force = false; 2644 2640 int err = 0; ··· 2649 2647 set_new_dnode(&dn, inode, NULL, NULL, 0); 2650 2648 2651 2649 if (need_inplace_update(fio) && 2652 - f2fs_lookup_read_extent_cache(inode, page->index, &ei)) { 2653 - fio->old_blkaddr = ei.blk + page->index - ei.fofs; 2654 - 2650 + f2fs_lookup_read_extent_cache_block(inode, page->index, 2651 + &fio->old_blkaddr)) { 2655 2652 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, 2656 2653 DATA_GENERIC_ENHANCE)) { 2657 2654 f2fs_handle_error(fio->sbi, ··· 2700 2699 goto out_writepage; 2701 2700 2702 2701 set_page_writeback(page); 2703 - ClearPageError(page); 2704 2702 f2fs_put_dnode(&dn); 2705 2703 if (fio->need_lock == LOCK_REQ) 2706 2704 f2fs_unlock_op(fio->sbi); ··· 2735 2735 goto out_writepage; 2736 2736 2737 2737 set_page_writeback(page); 2738 - ClearPageError(page); 2739 2738 2740 2739 if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR) 2741 2740 f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false); ··· 2779 2780 .old_blkaddr = NULL_ADDR, 2780 2781 .page = page, 2781 2782 .encrypted_page = NULL, 2782 - .submitted = false, 2783 + .submitted = 0, 2783 2784 .compr_blocks = compr_blocks, 2784 2785 .need_lock = LOCK_RETRY, 2785 - .post_read = f2fs_post_read_required(inode), 2786 + .post_read = f2fs_post_read_required(inode) ? 1 : 0, 2786 2787 .io_type = io_type, 2787 2788 .io_wbc = wbc, 2788 2789 .bio = bio, ··· 2791 2792 2792 2793 trace_f2fs_writepage(page, DATA); 2793 2794 2794 - /* we should bypass data pages to proceed the kworkder jobs */ 2795 + /* we should bypass data pages to proceed the kworker jobs */ 2795 2796 if (unlikely(f2fs_cp_error(sbi))) { 2796 2797 mapping_set_error(page->mapping, -EIO); 2797 2798 /* ··· 2903 2904 } 2904 2905 2905 2906 if (submitted) 2906 - *submitted = fio.submitted ? 1 : 0; 2907 + *submitted = fio.submitted; 2907 2908 2908 2909 return 0; 2909 2910 2910 2911 redirty_out: 2911 2912 redirty_page_for_writepage(wbc, page); 2912 2913 /* 2913 - * pageout() in MM traslates EAGAIN, so calls handle_write_error() 2914 + * pageout() in MM translates EAGAIN, so calls handle_write_error() 2914 2915 * -> mapping_set_error() -> set_bit(AS_EIO, ...). 2915 2916 * file_write_and_wait_range() will see EIO error, which is critical 2916 2917 * to return value of fsync() followed by atomic_write failure to user. ··· 2944 2945 } 2945 2946 2946 2947 /* 2947 - * This function was copied from write_cche_pages from mm/page-writeback.c. 2948 + * This function was copied from write_cache_pages from mm/page-writeback.c. 2948 2949 * The major change is making write step of cold data page separately from 2949 2950 * warm/hot data page. 2950 2951 */ ··· 3353 3354 struct dnode_of_data dn; 3354 3355 struct page *ipage; 3355 3356 bool locked = false; 3356 - struct extent_info ei = {0, }; 3357 + int flag = F2FS_GET_BLOCK_PRE_AIO; 3357 3358 int err = 0; 3358 - int flag; 3359 3359 3360 3360 /* 3361 3361 * If a whole page is being written and we already preallocated all the ··· 3364 3366 return 0; 3365 3367 3366 3368 /* f2fs_lock_op avoids race between write CP and convert_inline_page */ 3367 - if (f2fs_has_inline_data(inode) && pos + len > MAX_INLINE_DATA(inode)) 3368 - flag = F2FS_GET_BLOCK_DEFAULT; 3369 - else 3370 - flag = F2FS_GET_BLOCK_PRE_AIO; 3371 - 3372 - if (f2fs_has_inline_data(inode) || 3373 - (pos & PAGE_MASK) >= i_size_read(inode)) { 3374 - f2fs_do_map_lock(sbi, flag, true); 3369 + if (f2fs_has_inline_data(inode)) { 3370 + if (pos + len > MAX_INLINE_DATA(inode)) 3371 + flag = F2FS_GET_BLOCK_DEFAULT; 3372 + f2fs_map_lock(sbi, flag); 3373 + locked = true; 3374 + } else if ((pos & PAGE_MASK) >= i_size_read(inode)) { 3375 + f2fs_map_lock(sbi, flag); 3375 3376 locked = true; 3376 3377 } 3377 3378 ··· 3390 3393 set_inode_flag(inode, FI_DATA_EXIST); 3391 3394 if (inode->i_nlink) 3392 3395 set_page_private_inline(ipage); 3393 - } else { 3394 - err = f2fs_convert_inline_page(&dn, page); 3395 - if (err) 3396 - goto out; 3397 - if (dn.data_blkaddr == NULL_ADDR) 3398 - err = f2fs_get_block(&dn, index); 3396 + goto out; 3399 3397 } 3400 - } else if (locked) { 3401 - err = f2fs_get_block(&dn, index); 3402 - } else { 3403 - if (f2fs_lookup_read_extent_cache(inode, index, &ei)) { 3404 - dn.data_blkaddr = ei.blk + index - ei.fofs; 3405 - } else { 3406 - /* hole case */ 3407 - err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 3408 - if (err || dn.data_blkaddr == NULL_ADDR) { 3409 - f2fs_put_dnode(&dn); 3410 - f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, 3411 - true); 3412 - WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO); 3413 - locked = true; 3414 - goto restart; 3415 - } 3416 - } 3398 + err = f2fs_convert_inline_page(&dn, page); 3399 + if (err || dn.data_blkaddr != NULL_ADDR) 3400 + goto out; 3417 3401 } 3418 3402 3419 - /* convert_inline_page can make node_changed */ 3420 - *blk_addr = dn.data_blkaddr; 3421 - *node_changed = dn.node_changed; 3403 + if (!f2fs_lookup_read_extent_cache_block(inode, index, 3404 + &dn.data_blkaddr)) { 3405 + if (locked) { 3406 + err = f2fs_reserve_block(&dn, index); 3407 + goto out; 3408 + } 3409 + 3410 + /* hole case */ 3411 + err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 3412 + if (!err && dn.data_blkaddr != NULL_ADDR) 3413 + goto out; 3414 + f2fs_put_dnode(&dn); 3415 + f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3416 + WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO); 3417 + locked = true; 3418 + goto restart; 3419 + } 3422 3420 out: 3421 + if (!err) { 3422 + /* convert_inline_page can make node_changed */ 3423 + *blk_addr = dn.data_blkaddr; 3424 + *node_changed = dn.node_changed; 3425 + } 3423 3426 f2fs_put_dnode(&dn); 3424 3427 unlock_out: 3425 3428 if (locked) 3426 - f2fs_do_map_lock(sbi, flag, false); 3429 + f2fs_map_unlock(sbi, flag); 3427 3430 return err; 3428 3431 } 3429 3432 ··· 3432 3435 { 3433 3436 struct dnode_of_data dn; 3434 3437 struct page *ipage; 3435 - struct extent_info ei = {0, }; 3436 3438 int err = 0; 3437 3439 3438 3440 ipage = f2fs_get_node_page(F2FS_I_SB(inode), inode->i_ino); ··· 3440 3444 3441 3445 set_new_dnode(&dn, inode, ipage, ipage, 0); 3442 3446 3443 - if (f2fs_lookup_read_extent_cache(inode, index, &ei)) { 3444 - dn.data_blkaddr = ei.blk + index - ei.fofs; 3445 - } else { 3447 + if (!f2fs_lookup_read_extent_cache_block(inode, index, 3448 + &dn.data_blkaddr)) { 3446 3449 /* hole case */ 3447 3450 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 3448 3451 if (err) { ··· 3462 3467 struct page *ipage; 3463 3468 int err = 0; 3464 3469 3465 - f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true); 3470 + f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3466 3471 3467 3472 ipage = f2fs_get_node_page(sbi, inode->i_ino); 3468 3473 if (IS_ERR(ipage)) { ··· 3471 3476 } 3472 3477 set_new_dnode(&dn, inode, ipage, ipage, 0); 3473 3478 3474 - err = f2fs_get_block(&dn, index); 3479 + if (!f2fs_lookup_read_extent_cache_block(dn.inode, index, 3480 + &dn.data_blkaddr)) 3481 + err = f2fs_reserve_block(&dn, index); 3475 3482 3476 3483 *blk_addr = dn.data_blkaddr; 3477 3484 *node_changed = dn.node_changed; 3478 3485 f2fs_put_dnode(&dn); 3479 3486 3480 3487 unlock_out: 3481 - f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false); 3488 + f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3482 3489 return err; 3483 3490 } 3484 3491 ··· 3726 3729 } 3727 3730 } 3728 3731 3732 + clear_page_private_reference(&folio->page); 3729 3733 clear_page_private_gcing(&folio->page); 3730 3734 3731 3735 if (test_opt(sbi, COMPRESS_CACHE) && ··· 3752 3754 clear_page_private_data(&folio->page); 3753 3755 } 3754 3756 3757 + clear_page_private_reference(&folio->page); 3755 3758 clear_page_private_gcing(&folio->page); 3756 3759 3757 3760 folio_detach_private(folio); ··· 3834 3835 map.m_next_pgofs = NULL; 3835 3836 map.m_seg_type = NO_CHECK_TYPE; 3836 3837 3837 - if (!f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_BMAP)) 3838 + if (!f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_BMAP)) 3838 3839 blknr = map.m_pblk; 3839 3840 } 3840 3841 out: ··· 3942 3943 map.m_seg_type = NO_CHECK_TYPE; 3943 3944 map.m_may_create = false; 3944 3945 3945 - ret = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_FIEMAP); 3946 + ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP); 3946 3947 if (ret) 3947 3948 goto out; 3948 3949 ··· 4167 4168 if (flags & IOMAP_WRITE) 4168 4169 map.m_may_create = true; 4169 4170 4170 - err = f2fs_map_blocks(inode, &map, flags & IOMAP_WRITE, 4171 - F2FS_GET_BLOCK_DIO); 4171 + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DIO); 4172 4172 if (err) 4173 4173 return err; 4174 4174 ··· 4180 4182 */ 4181 4183 map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len); 4182 4184 4183 - if (map.m_flags & (F2FS_MAP_MAPPED | F2FS_MAP_UNWRITTEN)) { 4184 - iomap->length = blks_to_bytes(inode, map.m_len); 4185 - if (map.m_flags & F2FS_MAP_MAPPED) { 4186 - iomap->type = IOMAP_MAPPED; 4187 - iomap->flags |= IOMAP_F_MERGED; 4188 - } else { 4189 - iomap->type = IOMAP_UNWRITTEN; 4190 - } 4191 - if (WARN_ON_ONCE(!__is_valid_data_blkaddr(map.m_pblk))) 4192 - return -EINVAL; 4185 + /* 4186 + * We should never see delalloc or compressed extents here based on 4187 + * prior flushing and checks. 4188 + */ 4189 + if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR)) 4190 + return -EINVAL; 4191 + if (WARN_ON_ONCE(map.m_pblk == COMPRESS_ADDR)) 4192 + return -EINVAL; 4193 4193 4194 + if (map.m_pblk != NULL_ADDR) { 4195 + iomap->length = blks_to_bytes(inode, map.m_len); 4196 + iomap->type = IOMAP_MAPPED; 4197 + iomap->flags |= IOMAP_F_MERGED; 4194 4198 iomap->bdev = map.m_bdev; 4195 4199 iomap->addr = blks_to_bytes(inode, map.m_pblk); 4196 4200 } else { 4201 + if (flags & IOMAP_WRITE) 4202 + return -ENOTBLK; 4197 4203 iomap->length = blks_to_bytes(inode, next_pgofs) - 4198 4204 iomap->offset; 4199 4205 iomap->type = IOMAP_HOLE;

+45 -19

fs/f2fs/debug.c

··· 354 354 [SBI_IS_FREEZING] = " freezefs", 355 355 }; 356 356 357 + static const char *ipu_mode_names[F2FS_IPU_MAX] = { 358 + [F2FS_IPU_FORCE] = "FORCE", 359 + [F2FS_IPU_SSR] = "SSR", 360 + [F2FS_IPU_UTIL] = "UTIL", 361 + [F2FS_IPU_SSR_UTIL] = "SSR_UTIL", 362 + [F2FS_IPU_FSYNC] = "FSYNC", 363 + [F2FS_IPU_ASYNC] = "ASYNC", 364 + [F2FS_IPU_NOCACHE] = "NOCACHE", 365 + [F2FS_IPU_HONOR_OPU_WRITE] = "HONOR_OPU_WRITE", 366 + }; 367 + 357 368 static int stat_show(struct seq_file *s, void *v) 358 369 { 359 370 struct f2fs_stat_info *si; ··· 373 362 374 363 raw_spin_lock_irqsave(&f2fs_stat_lock, flags); 375 364 list_for_each_entry(si, &f2fs_stat_list, stat_list) { 376 - update_general_status(si->sbi); 365 + struct f2fs_sb_info *sbi = si->sbi; 366 + 367 + update_general_status(sbi); 377 368 378 369 seq_printf(s, "\n=====[ partition info(%pg). #%d, %s, CP: %s]=====\n", 379 - si->sbi->sb->s_bdev, i++, 380 - f2fs_readonly(si->sbi->sb) ? "RO" : "RW", 381 - is_set_ckpt_flags(si->sbi, CP_DISABLED_FLAG) ? 382 - "Disabled" : (f2fs_cp_error(si->sbi) ? "Error" : "Good")); 383 - if (si->sbi->s_flag) { 370 + sbi->sb->s_bdev, i++, 371 + f2fs_readonly(sbi->sb) ? "RO" : "RW", 372 + is_set_ckpt_flags(sbi, CP_DISABLED_FLAG) ? 373 + "Disabled" : (f2fs_cp_error(sbi) ? "Error" : "Good")); 374 + if (sbi->s_flag) { 384 375 seq_puts(s, "[SBI:"); 385 - for_each_set_bit(j, &si->sbi->s_flag, 32) 376 + for_each_set_bit(j, &sbi->s_flag, 32) 386 377 seq_puts(s, s_flag[j]); 387 378 seq_puts(s, "]\n"); 388 379 } ··· 396 383 si->overp_segs, si->rsvd_segs); 397 384 seq_printf(s, "Current Time Sec: %llu / Mounted Time Sec: %llu\n\n", 398 385 ktime_get_boottime_seconds(), 399 - SIT_I(si->sbi)->mounted_time); 400 - if (test_opt(si->sbi, DISCARD)) 386 + SIT_I(sbi)->mounted_time); 387 + 388 + seq_puts(s, "Policy:\n"); 389 + seq_puts(s, " - IPU: ["); 390 + if (IS_F2FS_IPU_DISABLE(sbi)) { 391 + seq_puts(s, " DISABLE"); 392 + } else { 393 + unsigned long policy = SM_I(sbi)->ipu_policy; 394 + 395 + for_each_set_bit(j, &policy, F2FS_IPU_MAX) 396 + seq_printf(s, " %s", ipu_mode_names[j]); 397 + } 398 + seq_puts(s, " ]\n\n"); 399 + 400 + if (test_opt(sbi, DISCARD)) 401 401 seq_printf(s, "Utilization: %u%% (%u valid blocks, %u discard blocks)\n", 402 402 si->utilization, si->valid_count, si->discard_blks); 403 403 else ··· 517 491 seq_printf(s, " - node segments : %d (%d)\n", 518 492 si->node_segs, si->bg_node_segs); 519 493 seq_puts(s, " - Reclaimed segs :\n"); 520 - seq_printf(s, " - Normal : %d\n", si->sbi->gc_reclaimed_segs[GC_NORMAL]); 521 - seq_printf(s, " - Idle CB : %d\n", si->sbi->gc_reclaimed_segs[GC_IDLE_CB]); 494 + seq_printf(s, " - Normal : %d\n", sbi->gc_reclaimed_segs[GC_NORMAL]); 495 + seq_printf(s, " - Idle CB : %d\n", sbi->gc_reclaimed_segs[GC_IDLE_CB]); 522 496 seq_printf(s, " - Idle Greedy : %d\n", 523 - si->sbi->gc_reclaimed_segs[GC_IDLE_GREEDY]); 524 - seq_printf(s, " - Idle AT : %d\n", si->sbi->gc_reclaimed_segs[GC_IDLE_AT]); 497 + sbi->gc_reclaimed_segs[GC_IDLE_GREEDY]); 498 + seq_printf(s, " - Idle AT : %d\n", sbi->gc_reclaimed_segs[GC_IDLE_AT]); 525 499 seq_printf(s, " - Urgent High : %d\n", 526 - si->sbi->gc_reclaimed_segs[GC_URGENT_HIGH]); 527 - seq_printf(s, " - Urgent Mid : %d\n", si->sbi->gc_reclaimed_segs[GC_URGENT_MID]); 528 - seq_printf(s, " - Urgent Low : %d\n", si->sbi->gc_reclaimed_segs[GC_URGENT_LOW]); 500 + sbi->gc_reclaimed_segs[GC_URGENT_HIGH]); 501 + seq_printf(s, " - Urgent Mid : %d\n", sbi->gc_reclaimed_segs[GC_URGENT_MID]); 502 + seq_printf(s, " - Urgent Low : %d\n", sbi->gc_reclaimed_segs[GC_URGENT_LOW]); 529 503 seq_printf(s, "Try to move %d blocks (BG: %d)\n", si->tot_blks, 530 504 si->bg_data_blks + si->bg_node_blks); 531 505 seq_printf(s, " - data blocks : %d (%d)\n", si->data_blks, ··· 591 565 si->ndirty_imeta); 592 566 seq_printf(s, " - fsync mark: %4lld\n", 593 567 percpu_counter_sum_positive( 594 - &si->sbi->rf_node_block_count)); 568 + &sbi->rf_node_block_count)); 595 569 seq_printf(s, " - NATs: %9d/%9d\n - SITs: %9d/%9d\n", 596 570 si->dirty_nats, si->nats, si->dirty_sits, si->sits); 597 571 seq_printf(s, " - free_nids: %9d/%9d\n - alloc_nids: %9d\n", ··· 618 592 si->block_count[LFS], si->segment_count[LFS]); 619 593 620 594 /* segment usage info */ 621 - f2fs_update_sit_info(si->sbi); 595 + f2fs_update_sit_info(sbi); 622 596 seq_printf(s, "\nBDF: %u, avg. vblocks: %u\n", 623 597 si->bimodal, si->avg_vblocks); 624 598 625 599 /* memory footprint */ 626 - update_mem_info(si->sbi); 600 + update_mem_info(sbi); 627 601 seq_printf(s, "\nMemory: %llu KB\n", 628 602 (si->base_mem + si->cache_mem + si->page_mem) >> 10); 629 603 seq_printf(s, " - static: %llu KB\n",

+1 -3

fs/f2fs/dir.c

··· 732 732 } 733 733 734 734 start: 735 - if (time_to_inject(F2FS_I_SB(dir), FAULT_DIR_DEPTH)) { 736 - f2fs_show_injection_info(F2FS_I_SB(dir), FAULT_DIR_DEPTH); 735 + if (time_to_inject(F2FS_I_SB(dir), FAULT_DIR_DEPTH)) 737 736 return -ENOSPC; 738 - } 739 737 740 738 if (unlikely(current_depth == MAX_DIR_HASH_DEPTH)) 741 739 return -ENOSPC;

+53 -7

fs/f2fs/extent_cache.c

··· 19 19 #include "node.h" 20 20 #include <trace/events/f2fs.h> 21 21 22 + bool sanity_check_extent_cache(struct inode *inode) 23 + { 24 + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 25 + struct f2fs_inode_info *fi = F2FS_I(inode); 26 + struct extent_info *ei; 27 + 28 + if (!fi->extent_tree[EX_READ]) 29 + return true; 30 + 31 + ei = &fi->extent_tree[EX_READ]->largest; 32 + 33 + if (ei->len && 34 + (!f2fs_is_valid_blkaddr(sbi, ei->blk, 35 + DATA_GENERIC_ENHANCE) || 36 + !f2fs_is_valid_blkaddr(sbi, ei->blk + ei->len - 1, 37 + DATA_GENERIC_ENHANCE))) { 38 + set_sbi_flag(sbi, SBI_NEED_FSCK); 39 + f2fs_warn(sbi, "%s: inode (ino=%lx) extent info [%u, %u, %u] is incorrect, run fsck to fix", 40 + __func__, inode->i_ino, 41 + ei->blk, ei->fofs, ei->len); 42 + return false; 43 + } 44 + return true; 45 + } 46 + 22 47 static void __set_extent_info(struct extent_info *ei, 23 48 unsigned int fofs, unsigned int len, 24 49 block_t blk, bool keep_clen, ··· 258 233 * @prev_ex: extent before ofs 259 234 * @next_ex: extent after ofs 260 235 * @insert_p: insert point for new extent at ofs 261 - * in order to simpfy the insertion after. 236 + * in order to simplify the insertion after. 262 237 * tree must stay unchanged between lookup and insertion. 263 238 */ 264 239 struct rb_entry *f2fs_lookup_rb_tree_ret(struct rb_root_cached *root, ··· 743 718 if (!en) 744 719 en = next_en; 745 720 746 - /* 2. invlidate all extent nodes in range [fofs, fofs + len - 1] */ 721 + /* 2. invalidate all extent nodes in range [fofs, fofs + len - 1] */ 747 722 while (en && en->ei.fofs < end) { 748 723 unsigned int org_end; 749 724 int parts = 0; /* # of parts current extent split into */ ··· 896 871 } 897 872 #endif 898 873 899 - static unsigned long long __calculate_block_age(unsigned long long new, 874 + static unsigned long long __calculate_block_age(struct f2fs_sb_info *sbi, 875 + unsigned long long new, 900 876 unsigned long long old) 901 877 { 902 - unsigned long long diff; 878 + unsigned int rem_old, rem_new; 879 + unsigned long long res; 880 + unsigned int weight = sbi->last_age_weight; 903 881 904 - diff = (new >= old) ? new - (new - old) : new + (old - new); 882 + res = div_u64_rem(new, 100, &rem_new) * (100 - weight) 883 + + div_u64_rem(old, 100, &rem_old) * weight; 905 884 906 - return div_u64(diff * LAST_AGE_WEIGHT, 100); 885 + if (rem_new) 886 + res += rem_new * (100 - weight) / 100; 887 + if (rem_old) 888 + res += rem_old * weight / 100; 889 + 890 + return res; 907 891 } 908 892 909 893 /* This returns a new age and allocated blocks in ei */ ··· 944 910 cur_age = ULLONG_MAX - tei.last_blocks + cur_blocks; 945 911 946 912 if (tei.age) 947 - ei->age = __calculate_block_age(cur_age, tei.age); 913 + ei->age = __calculate_block_age(sbi, cur_age, tei.age); 948 914 else 949 915 ei->age = cur_age; 950 916 ei->last_blocks = cur_blocks; ··· 1079 1045 return false; 1080 1046 1081 1047 return __lookup_extent_tree(inode, pgofs, ei, EX_READ); 1048 + } 1049 + 1050 + bool f2fs_lookup_read_extent_cache_block(struct inode *inode, pgoff_t index, 1051 + block_t *blkaddr) 1052 + { 1053 + struct extent_info ei = {}; 1054 + 1055 + if (!f2fs_lookup_read_extent_cache(inode, index, &ei)) 1056 + return false; 1057 + *blkaddr = ei.blk + index - ei.fofs; 1058 + return true; 1082 1059 } 1083 1060 1084 1061 void f2fs_update_read_extent_cache(struct dnode_of_data *dn) ··· 1271 1226 atomic64_set(&sbi->allocated_data_blocks, 0); 1272 1227 sbi->hot_data_age_threshold = DEF_HOT_DATA_AGE_THRESHOLD; 1273 1228 sbi->warm_data_age_threshold = DEF_WARM_DATA_AGE_THRESHOLD; 1229 + sbi->last_age_weight = LAST_AGE_WEIGHT; 1274 1230 } 1275 1231 1276 1232 int __init f2fs_create_extent_cache(void)

+53 -75

fs/f2fs/f2fs.h

··· 402 402 struct list_head wait_list; /* store on-flushing entries */ 403 403 struct list_head fstrim_list; /* in-flight discard from fstrim */ 404 404 wait_queue_head_t discard_wait_queue; /* waiting queue for wake-up */ 405 - unsigned int discard_wake; /* to wake up discard thread */ 406 405 struct mutex cmd_lock; 407 406 unsigned int nr_discards; /* # of discards in the list */ 408 407 unsigned int max_discards; /* max. discards to be issued */ ··· 409 410 unsigned int min_discard_issue_time; /* min. interval between discard issue */ 410 411 unsigned int mid_discard_issue_time; /* mid. interval between discard issue */ 411 412 unsigned int max_discard_issue_time; /* max. interval between discard issue */ 413 + unsigned int discard_io_aware_gran; /* minimum discard granularity not be aware of I/O */ 412 414 unsigned int discard_urgent_util; /* utilization which issue discard proactively */ 413 415 unsigned int discard_granularity; /* discard granularity */ 414 416 unsigned int max_ordered_discard; /* maximum discard granularity issued by lba order */ ··· 420 420 atomic_t discard_cmd_cnt; /* # of cached cmd count */ 421 421 struct rb_root_cached root; /* root of discard rb-tree */ 422 422 bool rbtree_check; /* config for consistence check */ 423 + bool discard_wake; /* to wake up discard thread */ 423 424 }; 424 425 425 426 /* for the list of fsync inodes, used only during recovery */ ··· 693 692 }; 694 693 695 694 /* 696 - * This structure is taken from ext4_map_blocks. 697 - * 698 - * Note that, however, f2fs uses NEW and MAPPED flags for f2fs_map_blocks(). 695 + * State of block returned by f2fs_map_blocks. 699 696 */ 700 - #define F2FS_MAP_NEW (1 << BH_New) 701 - #define F2FS_MAP_MAPPED (1 << BH_Mapped) 702 - #define F2FS_MAP_UNWRITTEN (1 << BH_Unwritten) 697 + #define F2FS_MAP_NEW (1U << 0) 698 + #define F2FS_MAP_MAPPED (1U << 1) 699 + #define F2FS_MAP_DELALLOC (1U << 2) 703 700 #define F2FS_MAP_FLAGS (F2FS_MAP_NEW | F2FS_MAP_MAPPED |\ 704 - F2FS_MAP_UNWRITTEN) 701 + F2FS_MAP_DELALLOC) 705 702 706 703 struct f2fs_map_blocks { 707 704 struct block_device *m_bdev; /* for multi-device dio */ ··· 869 870 unsigned char i_compress_algorithm; /* algorithm type */ 870 871 unsigned char i_log_cluster_size; /* log of cluster size */ 871 872 unsigned char i_compress_level; /* compress level (lz4hc,zstd) */ 872 - unsigned short i_compress_flag; /* compress flag */ 873 + unsigned char i_compress_flag; /* compress flag */ 873 874 unsigned int i_cluster_size; /* cluster size */ 874 875 875 876 unsigned int atomic_write_cnt; ··· 1192 1193 FS_META_READ_IO, /* meta read IOs */ 1193 1194 1194 1195 /* other */ 1195 - FS_DISCARD, /* discard */ 1196 + FS_DISCARD_IO, /* discard */ 1197 + FS_FLUSH_IO, /* flush */ 1196 1198 NR_IO_TYPE, 1197 1199 }; 1198 1200 ··· 1210 1210 struct page *encrypted_page; /* encrypted page */ 1211 1211 struct page *compressed_page; /* compressed page */ 1212 1212 struct list_head list; /* serialize IOs */ 1213 - bool submitted; /* indicate IO submission */ 1214 - int need_lock; /* indicate we need to lock cp_rwsem */ 1215 - bool in_list; /* indicate fio is in io_list */ 1216 - bool is_por; /* indicate IO is from recovery or not */ 1217 - bool retry; /* need to reallocate block address */ 1218 - int compr_blocks; /* # of compressed block addresses */ 1219 - bool encrypted; /* indicate file is encrypted */ 1220 - bool post_read; /* require post read */ 1213 + unsigned int compr_blocks; /* # of compressed block addresses */ 1214 + unsigned int need_lock:8; /* indicate we need to lock cp_rwsem */ 1215 + unsigned int version:8; /* version of the node */ 1216 + unsigned int submitted:1; /* indicate IO submission */ 1217 + unsigned int in_list:1; /* indicate fio is in io_list */ 1218 + unsigned int is_por:1; /* indicate IO is from recovery or not */ 1219 + unsigned int retry:1; /* need to reallocate block address */ 1220 + unsigned int encrypted:1; /* indicate file is encrypted */ 1221 + unsigned int post_read:1; /* require post read */ 1221 1222 enum iostat_type io_type; /* io type */ 1222 1223 struct writeback_control *io_wbc; /* writeback control */ 1223 1224 struct bio **bio; /* bio for ipu */ 1224 1225 sector_t *last_block; /* last block number in bio */ 1225 - unsigned char version; /* version of the node */ 1226 1226 }; 1227 1227 1228 1228 struct bio_entry { ··· 1384 1384 MEMORY_MODE_LOW, /* memory mode for low memry devices */ 1385 1385 }; 1386 1386 1387 - 1388 - 1389 1387 static inline int f2fs_test_bit(unsigned int nr, char *addr); 1390 1388 static inline void f2fs_set_bit(unsigned int nr, char *addr); 1391 1389 static inline void f2fs_clear_bit(unsigned int nr, char *addr); ··· 1394 1396 * Layout A: lowest bit should be 1 1395 1397 * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... | 1396 1398 * bit 0 PAGE_PRIVATE_NOT_POINTER 1397 - * bit 1 PAGE_PRIVATE_ATOMIC_WRITE 1398 - * bit 2 PAGE_PRIVATE_DUMMY_WRITE 1399 - * bit 3 PAGE_PRIVATE_ONGOING_MIGRATION 1400 - * bit 4 PAGE_PRIVATE_INLINE_INODE 1401 - * bit 5 PAGE_PRIVATE_REF_RESOURCE 1402 - * bit 6- f2fs private data 1399 + * bit 1 PAGE_PRIVATE_DUMMY_WRITE 1400 + * bit 2 PAGE_PRIVATE_ONGOING_MIGRATION 1401 + * bit 3 PAGE_PRIVATE_INLINE_INODE 1402 + * bit 4 PAGE_PRIVATE_REF_RESOURCE 1403 + * bit 5- f2fs private data 1403 1404 * 1404 1405 * Layout B: lowest bit should be 0 1405 1406 * page.private is a wrapped pointer. 1406 1407 */ 1407 1408 enum { 1408 1409 PAGE_PRIVATE_NOT_POINTER, /* private contains non-pointer data */ 1409 - PAGE_PRIVATE_ATOMIC_WRITE, /* data page from atomic write path */ 1410 1410 PAGE_PRIVATE_DUMMY_WRITE, /* data page for padding aligned IO */ 1411 1411 PAGE_PRIVATE_ONGOING_MIGRATION, /* data page which is on-going migrating */ 1412 1412 PAGE_PRIVATE_INLINE_INODE, /* inode page contains inline data */ ··· 1446 1450 } 1447 1451 1448 1452 PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER); 1449 - PAGE_PRIVATE_GET_FUNC(reference, REF_RESOURCE); 1450 1453 PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE); 1451 1454 PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION); 1452 - PAGE_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE); 1453 1455 PAGE_PRIVATE_GET_FUNC(dummy, DUMMY_WRITE); 1454 1456 1455 1457 PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE); 1456 1458 PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE); 1457 1459 PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION); 1458 - PAGE_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE); 1459 1460 PAGE_PRIVATE_SET_FUNC(dummy, DUMMY_WRITE); 1460 1461 1461 1462 PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE); 1462 1463 PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE); 1463 1464 PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION); 1464 - PAGE_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE); 1465 1465 PAGE_PRIVATE_CLEAR_FUNC(dummy, DUMMY_WRITE); 1466 1466 1467 1467 static inline unsigned long get_page_private_data(struct page *page) ··· 1671 1679 /* The threshold used for hot and warm data seperation*/ 1672 1680 unsigned int hot_data_age_threshold; 1673 1681 unsigned int warm_data_age_threshold; 1682 + unsigned int last_age_weight; 1674 1683 1675 1684 /* basic filesystem units */ 1676 1685 unsigned int log_sectors_per_block; /* log2 sectors per block */ ··· 1857 1864 #ifdef CONFIG_F2FS_IOSTAT 1858 1865 /* For app/fs IO statistics */ 1859 1866 spinlock_t iostat_lock; 1860 - unsigned long long rw_iostat[NR_IO_TYPE]; 1861 - unsigned long long prev_rw_iostat[NR_IO_TYPE]; 1867 + unsigned long long iostat_count[NR_IO_TYPE]; 1868 + unsigned long long iostat_bytes[NR_IO_TYPE]; 1869 + unsigned long long prev_iostat_bytes[NR_IO_TYPE]; 1862 1870 bool iostat_enable; 1863 1871 unsigned long iostat_next_period; 1864 1872 unsigned int iostat_period_ms; ··· 1871 1877 }; 1872 1878 1873 1879 #ifdef CONFIG_F2FS_FAULT_INJECTION 1874 - #define f2fs_show_injection_info(sbi, type) \ 1875 - printk_ratelimited("%sF2FS-fs (%s) : inject %s in %s of %pS\n", \ 1876 - KERN_INFO, sbi->sb->s_id, \ 1877 - f2fs_fault_name[type], \ 1878 - __func__, __builtin_return_address(0)) 1879 - static inline bool time_to_inject(struct f2fs_sb_info *sbi, int type) 1880 + #define time_to_inject(sbi, type) __time_to_inject(sbi, type, __func__, \ 1881 + __builtin_return_address(0)) 1882 + static inline bool __time_to_inject(struct f2fs_sb_info *sbi, int type, 1883 + const char *func, const char *parent_func) 1880 1884 { 1881 1885 struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info; 1882 1886 ··· 1887 1895 atomic_inc(&ffi->inject_ops); 1888 1896 if (atomic_read(&ffi->inject_ops) >= ffi->inject_rate) { 1889 1897 atomic_set(&ffi->inject_ops, 0); 1898 + printk_ratelimited("%sF2FS-fs (%s) : inject %s in %s of %pS\n", 1899 + KERN_INFO, sbi->sb->s_id, f2fs_fault_name[type], 1900 + func, parent_func); 1890 1901 return true; 1891 1902 } 1892 1903 return false; 1893 1904 } 1894 1905 #else 1895 - #define f2fs_show_injection_info(sbi, type) do { } while (0) 1896 1906 static inline bool time_to_inject(struct f2fs_sb_info *sbi, int type) 1897 1907 { 1898 1908 return false; ··· 2227 2233 2228 2234 static inline int f2fs_trylock_op(struct f2fs_sb_info *sbi) 2229 2235 { 2230 - if (time_to_inject(sbi, FAULT_LOCK_OP)) { 2231 - f2fs_show_injection_info(sbi, FAULT_LOCK_OP); 2236 + if (time_to_inject(sbi, FAULT_LOCK_OP)) 2232 2237 return 0; 2233 - } 2234 2238 return f2fs_down_read_trylock(&sbi->cp_rwsem); 2235 2239 } 2236 2240 ··· 2316 2324 return ret; 2317 2325 2318 2326 if (time_to_inject(sbi, FAULT_BLOCK)) { 2319 - f2fs_show_injection_info(sbi, FAULT_BLOCK); 2320 2327 release = *count; 2321 2328 goto release_quota; 2322 2329 } ··· 2595 2604 return err; 2596 2605 } 2597 2606 2598 - if (time_to_inject(sbi, FAULT_BLOCK)) { 2599 - f2fs_show_injection_info(sbi, FAULT_BLOCK); 2607 + if (time_to_inject(sbi, FAULT_BLOCK)) 2600 2608 goto enospc; 2601 - } 2602 2609 2603 2610 spin_lock(&sbi->stat_lock); 2604 2611 ··· 2720 2731 if (page) 2721 2732 return page; 2722 2733 2723 - if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_ALLOC)) { 2724 - f2fs_show_injection_info(F2FS_M_SB(mapping), 2725 - FAULT_PAGE_ALLOC); 2734 + if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_ALLOC)) 2726 2735 return NULL; 2727 - } 2728 2736 } 2729 2737 2730 2738 if (!for_write) ··· 2738 2752 struct address_space *mapping, pgoff_t index, 2739 2753 int fgp_flags, gfp_t gfp_mask) 2740 2754 { 2741 - if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_GET)) { 2742 - f2fs_show_injection_info(F2FS_M_SB(mapping), FAULT_PAGE_GET); 2755 + if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_GET)) 2743 2756 return NULL; 2744 - } 2745 2757 2746 2758 return pagecache_get_page(mapping, index, fgp_flags, gfp_mask); 2747 2759 } ··· 2789 2805 if (nofail) 2790 2806 return f2fs_kmem_cache_alloc_nofail(cachep, flags); 2791 2807 2792 - if (time_to_inject(sbi, FAULT_SLAB_ALLOC)) { 2793 - f2fs_show_injection_info(sbi, FAULT_SLAB_ALLOC); 2808 + if (time_to_inject(sbi, FAULT_SLAB_ALLOC)) 2794 2809 return NULL; 2795 - } 2796 2810 2797 2811 return kmem_cache_alloc(cachep, flags); 2798 2812 } ··· 3364 3382 static inline void *f2fs_kmalloc(struct f2fs_sb_info *sbi, 3365 3383 size_t size, gfp_t flags) 3366 3384 { 3367 - if (time_to_inject(sbi, FAULT_KMALLOC)) { 3368 - f2fs_show_injection_info(sbi, FAULT_KMALLOC); 3385 + if (time_to_inject(sbi, FAULT_KMALLOC)) 3369 3386 return NULL; 3370 - } 3371 3387 3372 3388 return kmalloc(size, flags); 3373 3389 } ··· 3379 3399 static inline void *f2fs_kvmalloc(struct f2fs_sb_info *sbi, 3380 3400 size_t size, gfp_t flags) 3381 3401 { 3382 - if (time_to_inject(sbi, FAULT_KVMALLOC)) { 3383 - f2fs_show_injection_info(sbi, FAULT_KVMALLOC); 3402 + if (time_to_inject(sbi, FAULT_KVMALLOC)) 3384 3403 return NULL; 3385 - } 3386 3404 3387 3405 return kvmalloc(size, flags); 3388 3406 } ··· 3766 3788 void f2fs_destroy_bioset(void); 3767 3789 int f2fs_init_bio_entry_cache(void); 3768 3790 void f2fs_destroy_bio_entry_cache(void); 3769 - void f2fs_submit_bio(struct f2fs_sb_info *sbi, 3770 - struct bio *bio, enum page_type type); 3791 + void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio, 3792 + enum page_type type); 3771 3793 int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi); 3772 3794 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type); 3773 3795 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, ··· 3786 3808 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr); 3787 3809 int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count); 3788 3810 int f2fs_reserve_new_block(struct dnode_of_data *dn); 3789 - int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index); 3811 + int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index); 3790 3812 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index); 3791 3813 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index, 3792 3814 blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs); ··· 3797 3819 struct page *f2fs_get_new_data_page(struct inode *inode, 3798 3820 struct page *ipage, pgoff_t index, bool new_i_size); 3799 3821 int f2fs_do_write_data_page(struct f2fs_io_info *fio); 3800 - void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock); 3801 - int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, 3802 - int create, int flag); 3822 + int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag); 3803 3823 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 3804 3824 u64 start, u64 len); 3805 3825 int f2fs_encrypt_one_page(struct f2fs_io_info *fio); ··· 4137 4161 /* 4138 4162 * extent_cache.c 4139 4163 */ 4164 + bool sanity_check_extent_cache(struct inode *inode); 4140 4165 struct rb_entry *f2fs_lookup_rb_tree(struct rb_root_cached *root, 4141 4166 struct rb_entry *cached_re, unsigned int ofs); 4142 4167 struct rb_node **f2fs_lookup_rb_tree_ext(struct f2fs_sb_info *sbi, ··· 4167 4190 void f2fs_init_read_extent_tree(struct inode *inode, struct page *ipage); 4168 4191 bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs, 4169 4192 struct extent_info *ei); 4193 + bool f2fs_lookup_read_extent_cache_block(struct inode *inode, pgoff_t index, 4194 + block_t *blkaddr); 4170 4195 void f2fs_update_read_extent_cache(struct dnode_of_data *dn); 4171 4196 void f2fs_update_read_extent_cache_range(struct dnode_of_data *dn, 4172 4197 pgoff_t fofs, block_t blkaddr, unsigned int len); ··· 4238 4259 int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock); 4239 4260 void f2fs_compress_write_end_io(struct bio *bio, struct page *page); 4240 4261 bool f2fs_is_compress_backend_ready(struct inode *inode); 4241 - int f2fs_init_compress_mempool(void); 4262 + int __init f2fs_init_compress_mempool(void); 4242 4263 void f2fs_destroy_compress_mempool(void); 4243 4264 void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task); 4244 4265 void f2fs_end_read_compressed_page(struct page *page, bool failed, ··· 4307 4328 WARN_ON_ONCE(1); 4308 4329 return ERR_PTR(-EINVAL); 4309 4330 } 4310 - static inline int f2fs_init_compress_mempool(void) { return 0; } 4331 + static inline int __init f2fs_init_compress_mempool(void) { return 0; } 4311 4332 static inline void f2fs_destroy_compress_mempool(void) { } 4312 4333 static inline void f2fs_decompress_cluster(struct decompress_io_ctx *dic, 4313 4334 bool in_task) { } ··· 4360 4381 if ((F2FS_I(inode)->i_compress_algorithm == COMPRESS_LZ4 || 4361 4382 F2FS_I(inode)->i_compress_algorithm == COMPRESS_ZSTD) && 4362 4383 F2FS_OPTION(sbi).compress_level) 4363 - F2FS_I(inode)->i_compress_flag |= 4364 - F2FS_OPTION(sbi).compress_level << 4365 - COMPRESS_LEVEL_OFFSET; 4384 + F2FS_I(inode)->i_compress_level = 4385 + F2FS_OPTION(sbi).compress_level; 4366 4386 F2FS_I(inode)->i_flags |= F2FS_COMPR_FL; 4367 4387 set_inode_flag(inode, FI_COMPRESSED_FILE); 4368 4388 stat_inc_compr_inode(inode);

+89 -82

fs/f2fs/file.c

··· 113 113 114 114 if (need_alloc) { 115 115 /* block allocation */ 116 - f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true); 117 116 set_new_dnode(&dn, inode, NULL, NULL, 0); 118 - err = f2fs_get_block(&dn, page->index); 119 - f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false); 117 + err = f2fs_get_block_locked(&dn, page->index); 120 118 } 121 119 122 120 #ifdef CONFIG_F2FS_FS_COMPRESSION ··· 303 305 * for OPU case, during fsync(), node can be persisted before 304 306 * data when lower device doesn't support write barrier, result 305 307 * in data corruption after SPO. 306 - * So for strict fsync mode, force to use atomic write sematics 308 + * So for strict fsync mode, force to use atomic write semantics 307 309 * to keep write order in between data/node and last node to 308 310 * avoid potential data corruption. 309 311 */ ··· 617 619 fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_page), 618 620 dn->inode) + ofs; 619 621 f2fs_update_read_extent_cache_range(dn, fofs, 0, len); 620 - f2fs_update_age_extent_cache_range(dn, fofs, nr_free); 622 + f2fs_update_age_extent_cache_range(dn, fofs, len); 621 623 dec_valid_block_count(sbi, dn->inode, nr_free); 622 624 } 623 625 dn->ofs_in_node = ofs; ··· 782 784 783 785 trace_f2fs_truncate(inode); 784 786 785 - if (time_to_inject(F2FS_I_SB(inode), FAULT_TRUNCATE)) { 786 - f2fs_show_injection_info(F2FS_I_SB(inode), FAULT_TRUNCATE); 787 + if (time_to_inject(F2FS_I_SB(inode), FAULT_TRUNCATE)) 787 788 return -EIO; 788 - } 789 789 790 790 err = f2fs_dquot_initialize(inode); 791 791 if (err) ··· 1108 1112 return 0; 1109 1113 } 1110 1114 1111 - static int punch_hole(struct inode *inode, loff_t offset, loff_t len) 1115 + static int f2fs_punch_hole(struct inode *inode, loff_t offset, loff_t len) 1112 1116 { 1113 1117 pgoff_t pg_start, pg_end; 1114 1118 loff_t off_start, off_end; ··· 1494 1498 } 1495 1499 1496 1500 f2fs_update_read_extent_cache_range(dn, start, 0, index - start); 1501 + f2fs_update_age_extent_cache_range(dn, start, index - start); 1497 1502 1498 1503 return ret; 1499 1504 } ··· 1681 1684 return ret; 1682 1685 } 1683 1686 1684 - static int expand_inode_data(struct inode *inode, loff_t offset, 1687 + static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, 1685 1688 loff_t len, int mode) 1686 1689 { 1687 1690 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); ··· 1694 1697 .err_gc_skipped = true, 1695 1698 .nr_free_secs = 0 }; 1696 1699 pgoff_t pg_start, pg_end; 1697 - loff_t new_size = i_size_read(inode); 1700 + loff_t new_size; 1698 1701 loff_t off_end; 1699 1702 block_t expanded = 0; 1700 1703 int err; ··· 1742 1745 f2fs_unlock_op(sbi); 1743 1746 1744 1747 map.m_seg_type = CURSEG_COLD_DATA_PINNED; 1745 - err = f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_DIO); 1748 + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_PRE_DIO); 1746 1749 file_dont_truncate(inode); 1747 1750 1748 1751 f2fs_up_write(&sbi->pin_sem); ··· 1755 1758 1756 1759 map.m_len = expanded; 1757 1760 } else { 1758 - err = f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO); 1761 + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_PRE_AIO); 1759 1762 expanded = map.m_len; 1760 1763 } 1761 1764 out_err: ··· 1806 1809 return -EOPNOTSUPP; 1807 1810 1808 1811 /* 1809 - * Pinned file should not support partial trucation since the block 1812 + * Pinned file should not support partial truncation since the block 1810 1813 * can be used by applications. 1811 1814 */ 1812 1815 if ((f2fs_compressed_file(inode) || f2fs_is_pinned_file(inode)) && ··· 1829 1832 if (offset >= inode->i_size) 1830 1833 goto out; 1831 1834 1832 - ret = punch_hole(inode, offset, len); 1835 + ret = f2fs_punch_hole(inode, offset, len); 1833 1836 } else if (mode & FALLOC_FL_COLLAPSE_RANGE) { 1834 1837 ret = f2fs_collapse_range(inode, offset, len); 1835 1838 } else if (mode & FALLOC_FL_ZERO_RANGE) { ··· 1837 1840 } else if (mode & FALLOC_FL_INSERT_RANGE) { 1838 1841 ret = f2fs_insert_range(inode, offset, len); 1839 1842 } else { 1840 - ret = expand_inode_data(inode, offset, len, mode); 1843 + ret = f2fs_expand_inode_data(inode, offset, len, mode); 1841 1844 } 1842 1845 1843 1846 if (!ret) { ··· 1856 1859 static int f2fs_release_file(struct inode *inode, struct file *filp) 1857 1860 { 1858 1861 /* 1859 - * f2fs_relase_file is called at every close calls. So we should 1862 + * f2fs_release_file is called at every close calls. So we should 1860 1863 * not drop any inmemory pages by close called by other process. 1861 1864 */ 1862 1865 if (!(filp->f_mode & FMODE_WRITE) || 1863 1866 atomic_read(&inode->i_writecount) != 1) 1864 1867 return 0; 1865 1868 1869 + inode_lock(inode); 1866 1870 f2fs_abort_atomic_write(inode, true); 1871 + inode_unlock(inode); 1872 + 1867 1873 return 0; 1868 1874 } 1869 1875 ··· 1880 1880 * until all the writers close its file. Since this should be done 1881 1881 * before dropping file lock, it needs to do in ->flush. 1882 1882 */ 1883 - if (F2FS_I(inode)->atomic_write_task == current) 1883 + if (F2FS_I(inode)->atomic_write_task == current && 1884 + (current->flags & PF_EXITING)) { 1885 + inode_lock(inode); 1884 1886 f2fs_abort_atomic_write(inode, true); 1887 + inode_unlock(inode); 1888 + } 1889 + 1885 1890 return 0; 1886 1891 } 1887 1892 ··· 2092 2087 goto out; 2093 2088 } 2094 2089 2095 - /* Create a COW inode for atomic write */ 2096 - pinode = f2fs_iget(inode->i_sb, fi->i_pino); 2097 - if (IS_ERR(pinode)) { 2098 - f2fs_up_write(&fi->i_gc_rwsem[WRITE]); 2099 - ret = PTR_ERR(pinode); 2100 - goto out; 2101 - } 2090 + /* Check if the inode already has a COW inode */ 2091 + if (fi->cow_inode == NULL) { 2092 + /* Create a COW inode for atomic write */ 2093 + pinode = f2fs_iget(inode->i_sb, fi->i_pino); 2094 + if (IS_ERR(pinode)) { 2095 + f2fs_up_write(&fi->i_gc_rwsem[WRITE]); 2096 + ret = PTR_ERR(pinode); 2097 + goto out; 2098 + } 2102 2099 2103 - ret = f2fs_get_tmpfile(idmap, pinode, &fi->cow_inode); 2104 - iput(pinode); 2105 - if (ret) { 2106 - f2fs_up_write(&fi->i_gc_rwsem[WRITE]); 2107 - goto out; 2100 + ret = f2fs_get_tmpfile(idmap, pinode, &fi->cow_inode); 2101 + iput(pinode); 2102 + if (ret) { 2103 + f2fs_up_write(&fi->i_gc_rwsem[WRITE]); 2104 + goto out; 2105 + } 2106 + 2107 + set_inode_flag(fi->cow_inode, FI_COW_FILE); 2108 + clear_inode_flag(fi->cow_inode, FI_INLINE_DATA); 2109 + } else { 2110 + /* Reuse the already created COW inode */ 2111 + f2fs_do_truncate_blocks(fi->cow_inode, 0, true); 2108 2112 } 2109 2113 2110 2114 f2fs_write_inode(inode, NULL); ··· 2121 2107 stat_inc_atomic_inode(inode); 2122 2108 2123 2109 set_inode_flag(inode, FI_ATOMIC_FILE); 2124 - set_inode_flag(fi->cow_inode, FI_COW_FILE); 2125 - clear_inode_flag(fi->cow_inode, FI_INLINE_DATA); 2126 2110 2127 2111 isize = i_size_read(inode); 2128 2112 fi->original_i_size = isize; ··· 2350 2338 { 2351 2339 struct inode *inode = file_inode(filp); 2352 2340 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2341 + u8 encrypt_pw_salt[16]; 2353 2342 int err; 2354 2343 2355 2344 if (!f2fs_sb_has_encrypt(sbi)) ··· 2375 2362 goto out_err; 2376 2363 } 2377 2364 got_it: 2378 - if (copy_to_user((__u8 __user *)arg, sbi->raw_super->encrypt_pw_salt, 2379 - 16)) 2380 - err = -EFAULT; 2365 + memcpy(encrypt_pw_salt, sbi->raw_super->encrypt_pw_salt, 16); 2381 2366 out_err: 2382 2367 f2fs_up_write(&sbi->sb_lock); 2383 2368 mnt_drop_write_file(filp); 2369 + 2370 + if (!err && copy_to_user((__u8 __user *)arg, encrypt_pw_salt, 16)) 2371 + err = -EFAULT; 2372 + 2384 2373 return err; 2385 2374 } 2386 2375 ··· 2539 2524 return __f2fs_ioc_gc_range(filp, &range); 2540 2525 } 2541 2526 2542 - static int f2fs_ioc_write_checkpoint(struct file *filp, unsigned long arg) 2527 + static int f2fs_ioc_write_checkpoint(struct file *filp) 2543 2528 { 2544 2529 struct inode *inode = file_inode(filp); 2545 2530 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); ··· 2621 2606 */ 2622 2607 while (map.m_lblk < pg_end) { 2623 2608 map.m_len = pg_end - map.m_lblk; 2624 - err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT); 2609 + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT); 2625 2610 if (err) 2626 2611 goto out; 2627 2612 ··· 2668 2653 2669 2654 do_map: 2670 2655 map.m_len = pg_end - map.m_lblk; 2671 - err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT); 2656 + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT); 2672 2657 if (err) 2673 2658 goto clear_out; 2674 2659 ··· 3242 3227 map.m_len = end - map.m_lblk; 3243 3228 3244 3229 f2fs_down_write(&fi->i_gc_rwsem[WRITE]); 3245 - err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_PRECACHE); 3230 + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_PRECACHE); 3246 3231 f2fs_up_write(&fi->i_gc_rwsem[WRITE]); 3247 3232 if (err) 3248 3233 return err; ··· 3253 3238 return 0; 3254 3239 } 3255 3240 3256 - static int f2fs_ioc_precache_extents(struct file *filp, unsigned long arg) 3241 + static int f2fs_ioc_precache_extents(struct file *filp) 3257 3242 { 3258 3243 return f2fs_precache_extents(file_inode(filp)); 3259 3244 } ··· 3957 3942 goto out; 3958 3943 } 3959 3944 3960 - if (inode->i_size != 0) { 3945 + if (F2FS_HAS_BLOCKS(inode)) { 3961 3946 ret = -EFBIG; 3962 3947 goto out; 3963 3948 } ··· 4010 3995 return ret; 4011 3996 } 4012 3997 4013 - static int f2fs_ioc_decompress_file(struct file *filp, unsigned long arg) 3998 + static int f2fs_ioc_decompress_file(struct file *filp) 4014 3999 { 4015 4000 struct inode *inode = file_inode(filp); 4016 4001 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); ··· 4083 4068 return ret; 4084 4069 } 4085 4070 4086 - static int f2fs_ioc_compress_file(struct file *filp, unsigned long arg) 4071 + static int f2fs_ioc_compress_file(struct file *filp) 4087 4072 { 4088 4073 struct inode *inode = file_inode(filp); 4089 4074 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); ··· 4199 4184 case F2FS_IOC_GARBAGE_COLLECT_RANGE: 4200 4185 return f2fs_ioc_gc_range(filp, arg); 4201 4186 case F2FS_IOC_WRITE_CHECKPOINT: 4202 - return f2fs_ioc_write_checkpoint(filp, arg); 4187 + return f2fs_ioc_write_checkpoint(filp); 4203 4188 case F2FS_IOC_DEFRAGMENT: 4204 4189 return f2fs_ioc_defragment(filp, arg); 4205 4190 case F2FS_IOC_MOVE_RANGE: ··· 4213 4198 case F2FS_IOC_SET_PIN_FILE: 4214 4199 return f2fs_ioc_set_pin_file(filp, arg); 4215 4200 case F2FS_IOC_PRECACHE_EXTENTS: 4216 - return f2fs_ioc_precache_extents(filp, arg); 4201 + return f2fs_ioc_precache_extents(filp); 4217 4202 case F2FS_IOC_RESIZE_FS: 4218 4203 return f2fs_ioc_resize_fs(filp, arg); 4219 4204 case FS_IOC_ENABLE_VERITY: ··· 4239 4224 case F2FS_IOC_SET_COMPRESS_OPTION: 4240 4225 return f2fs_ioc_set_compress_option(filp, arg); 4241 4226 case F2FS_IOC_DECOMPRESS_FILE: 4242 - return f2fs_ioc_decompress_file(filp, arg); 4227 + return f2fs_ioc_decompress_file(filp); 4243 4228 case F2FS_IOC_COMPRESS_FILE: 4244 - return f2fs_ioc_compress_file(filp, arg); 4229 + return f2fs_ioc_compress_file(filp); 4245 4230 default: 4246 4231 return -ENOTTY; 4247 4232 } ··· 4356 4341 return ret; 4357 4342 } 4358 4343 4344 + static void f2fs_trace_rw_file_path(struct kiocb *iocb, size_t count, int rw) 4345 + { 4346 + struct inode *inode = file_inode(iocb->ki_filp); 4347 + char *buf, *path; 4348 + 4349 + buf = f2fs_kmalloc(F2FS_I_SB(inode), PATH_MAX, GFP_KERNEL); 4350 + if (!buf) 4351 + return; 4352 + path = dentry_path_raw(file_dentry(iocb->ki_filp), buf, PATH_MAX); 4353 + if (IS_ERR(path)) 4354 + goto free_buf; 4355 + if (rw == WRITE) 4356 + trace_f2fs_datawrite_start(inode, iocb->ki_pos, count, 4357 + current->pid, path, current->comm); 4358 + else 4359 + trace_f2fs_dataread_start(inode, iocb->ki_pos, count, 4360 + current->pid, path, current->comm); 4361 + free_buf: 4362 + kfree(buf); 4363 + } 4364 + 4359 4365 static ssize_t f2fs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 4360 4366 { 4361 4367 struct inode *inode = file_inode(iocb->ki_filp); ··· 4386 4350 if (!f2fs_is_compress_backend_ready(inode)) 4387 4351 return -EOPNOTSUPP; 4388 4352 4389 - if (trace_f2fs_dataread_start_enabled()) { 4390 - char *p = f2fs_kmalloc(F2FS_I_SB(inode), PATH_MAX, GFP_KERNEL); 4391 - char *path; 4353 + if (trace_f2fs_dataread_start_enabled()) 4354 + f2fs_trace_rw_file_path(iocb, iov_iter_count(to), READ); 4392 4355 4393 - if (!p) 4394 - goto skip_read_trace; 4395 - 4396 - path = dentry_path_raw(file_dentry(iocb->ki_filp), p, PATH_MAX); 4397 - if (IS_ERR(path)) { 4398 - kfree(p); 4399 - goto skip_read_trace; 4400 - } 4401 - 4402 - trace_f2fs_dataread_start(inode, pos, iov_iter_count(to), 4403 - current->pid, path, current->comm); 4404 - kfree(p); 4405 - } 4406 - skip_read_trace: 4407 4356 if (f2fs_should_use_dio(inode, iocb, to)) { 4408 4357 ret = f2fs_dio_read_iter(iocb, to); 4409 4358 } else { ··· 4487 4466 flag = F2FS_GET_BLOCK_PRE_AIO; 4488 4467 } 4489 4468 4490 - ret = f2fs_map_blocks(inode, &map, 1, flag); 4469 + ret = f2fs_map_blocks(inode, &map, flag); 4491 4470 /* -ENOSPC|-EDQUOT are fine to report the number of allocated blocks. */ 4492 4471 if (ret < 0 && !((ret == -ENOSPC || ret == -EDQUOT) && map.m_len > 0)) 4493 4472 return ret; ··· 4694 4673 if (preallocated < 0) { 4695 4674 ret = preallocated; 4696 4675 } else { 4697 - if (trace_f2fs_datawrite_start_enabled()) { 4698 - char *p = f2fs_kmalloc(F2FS_I_SB(inode), 4699 - PATH_MAX, GFP_KERNEL); 4700 - char *path; 4676 + if (trace_f2fs_datawrite_start_enabled()) 4677 + f2fs_trace_rw_file_path(iocb, orig_count, WRITE); 4701 4678 4702 - if (!p) 4703 - goto skip_write_trace; 4704 - path = dentry_path_raw(file_dentry(iocb->ki_filp), 4705 - p, PATH_MAX); 4706 - if (IS_ERR(path)) { 4707 - kfree(p); 4708 - goto skip_write_trace; 4709 - } 4710 - trace_f2fs_datawrite_start(inode, orig_pos, orig_count, 4711 - current->pid, path, current->comm); 4712 - kfree(p); 4713 - } 4714 - skip_write_trace: 4715 4679 /* Do the actual write. */ 4716 4680 ret = dio ? 4717 4681 f2fs_dio_write_iter(iocb, from, &may_need_sync) : ··· 4829 4823 case F2FS_IOC32_MOVE_RANGE: 4830 4824 return f2fs_compat_ioc_move_range(file, arg); 4831 4825 case F2FS_IOC_START_ATOMIC_WRITE: 4826 + case F2FS_IOC_START_ATOMIC_REPLACE: 4832 4827 case F2FS_IOC_COMMIT_ATOMIC_WRITE: 4833 4828 case F2FS_IOC_START_VOLATILE_WRITE: 4834 4829 case F2FS_IOC_RELEASE_VOLATILE_WRITE:

+9 -13

fs/f2fs/gc.c

··· 57 57 58 58 /* give it a try one time */ 59 59 if (gc_th->gc_wake) 60 - gc_th->gc_wake = 0; 60 + gc_th->gc_wake = false; 61 61 62 62 if (try_to_freeze()) { 63 63 stat_other_skip_bggc_count(sbi); ··· 72 72 continue; 73 73 } 74 74 75 - if (time_to_inject(sbi, FAULT_CHECKPOINT)) { 76 - f2fs_show_injection_info(sbi, FAULT_CHECKPOINT); 75 + if (time_to_inject(sbi, FAULT_CHECKPOINT)) 77 76 f2fs_stop_checkpoint(sbi, false, 78 77 STOP_CP_REASON_FAULT_INJECT); 79 - } 80 78 81 79 if (!sb_start_write_trylock(sbi->sb)) { 82 80 stat_other_skip_bggc_count(sbi); ··· 183 185 gc_th->max_sleep_time = DEF_GC_THREAD_MAX_SLEEP_TIME; 184 186 gc_th->no_gc_sleep_time = DEF_GC_THREAD_NOGC_SLEEP_TIME; 185 187 186 - gc_th->gc_wake = 0; 188 + gc_th->gc_wake = false; 187 189 188 190 sbi->gc_thread = gc_th; 189 191 init_waitqueue_head(&sbi->gc_thread->gc_wait_queue_head); ··· 1148 1150 struct address_space *mapping = inode->i_mapping; 1149 1151 struct dnode_of_data dn; 1150 1152 struct page *page; 1151 - struct extent_info ei = {0, }; 1152 1153 struct f2fs_io_info fio = { 1153 1154 .sbi = sbi, 1154 1155 .ino = inode->i_ino, ··· 1156 1159 .op = REQ_OP_READ, 1157 1160 .op_flags = 0, 1158 1161 .encrypted_page = NULL, 1159 - .in_list = false, 1160 - .retry = false, 1162 + .in_list = 0, 1163 + .retry = 0, 1161 1164 }; 1162 1165 int err; 1163 1166 ··· 1165 1168 if (!page) 1166 1169 return -ENOMEM; 1167 1170 1168 - if (f2fs_lookup_read_extent_cache(inode, index, &ei)) { 1169 - dn.data_blkaddr = ei.blk + index - ei.fofs; 1171 + if (f2fs_lookup_read_extent_cache_block(inode, index, 1172 + &dn.data_blkaddr)) { 1170 1173 if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr, 1171 1174 DATA_GENERIC_ENHANCE_READ))) { 1172 1175 err = -EFSCORRUPTED; ··· 1245 1248 .op = REQ_OP_READ, 1246 1249 .op_flags = 0, 1247 1250 .encrypted_page = NULL, 1248 - .in_list = false, 1249 - .retry = false, 1251 + .in_list = 0, 1252 + .retry = 0, 1250 1253 }; 1251 1254 struct dnode_of_data dn; 1252 1255 struct f2fs_summary sum; ··· 1362 1365 dec_page_count(fio.sbi, F2FS_DIRTY_META); 1363 1366 1364 1367 set_page_writeback(fio.encrypted_page); 1365 - ClearPageError(page); 1366 1368 1367 1369 fio.op = REQ_OP_WRITE; 1368 1370 fio.op_flags = REQ_SYNC;

+1 -1

fs/f2fs/gc.h

··· 41 41 unsigned int no_gc_sleep_time; 42 42 43 43 /* for changing gc mode */ 44 - unsigned int gc_wake; 44 + bool gc_wake; 45 45 46 46 /* for GC_MERGE mount option */ 47 47 wait_queue_head_t fggc_wq; /*

+6 -8

fs/f2fs/inline.c

··· 174 174 175 175 /* write data page to try to make data consistent */ 176 176 set_page_writeback(page); 177 - ClearPageError(page); 178 177 fio.old_blkaddr = dn->data_blkaddr; 179 178 set_inode_flag(dn->inode, FI_HOT_DATA); 180 179 f2fs_outplace_write_data(dn, &fio); ··· 421 422 422 423 dentry_blk = page_address(page); 423 424 425 + /* 426 + * Start by zeroing the full block, to ensure that all unused space is 427 + * zeroed and no uninitialized memory is leaked to disk. 428 + */ 429 + memset(dentry_blk, 0, F2FS_BLKSIZE); 430 + 424 431 make_dentry_ptr_inline(dir, &src, inline_dentry); 425 432 make_dentry_ptr_block(dir, &dst, dentry_blk); 426 433 427 434 /* copy data from inline dentry block to new dentry block */ 428 435 memcpy(dst.bitmap, src.bitmap, src.nr_bitmap); 429 - memset(dst.bitmap + src.nr_bitmap, 0, dst.nr_bitmap - src.nr_bitmap); 430 - /* 431 - * we do not need to zero out remainder part of dentry and filename 432 - * field, since we have used bitmap for marking the usage status of 433 - * them, besides, we can also ignore copying/zeroing reserved space 434 - * of dentry block, because them haven't been used so far. 435 - */ 436 436 memcpy(dst.dentry, src.dentry, SIZE_OF_DIR_ENTRY * src.max); 437 437 memcpy(dst.filename, src.filename, src.max * F2FS_SLOT_LEN); 438 438

+42 -36

fs/f2fs/inode.c

··· 262 262 return false; 263 263 } 264 264 265 - if (fi->extent_tree[EX_READ]) { 266 - struct extent_info *ei = &fi->extent_tree[EX_READ]->largest; 267 - 268 - if (ei->len && 269 - (!f2fs_is_valid_blkaddr(sbi, ei->blk, 270 - DATA_GENERIC_ENHANCE) || 271 - !f2fs_is_valid_blkaddr(sbi, ei->blk + ei->len - 1, 272 - DATA_GENERIC_ENHANCE))) { 273 - set_sbi_flag(sbi, SBI_NEED_FSCK); 274 - f2fs_warn(sbi, "%s: inode (ino=%lx) extent info [%u, %u, %u] is incorrect, run fsck to fix", 275 - __func__, inode->i_ino, 276 - ei->blk, ei->fofs, ei->len); 277 - return false; 278 - } 279 - } 280 - 281 265 if (f2fs_sanity_check_inline_data(inode)) { 282 266 set_sbi_flag(sbi, SBI_NEED_FSCK); 283 267 f2fs_warn(sbi, "%s: inode (ino=%lx, mode=%u) should not have inline_data, run fsck to fix", ··· 397 413 fi->i_inline_xattr_size = 0; 398 414 } 399 415 400 - if (!sanity_check_inode(inode, node_page)) { 401 - f2fs_put_page(node_page, 1); 402 - f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); 403 - return -EFSCORRUPTED; 404 - } 405 - 406 416 /* check data exist */ 407 417 if (f2fs_has_inline_data(inode) && !f2fs_exist_data(inode)) 408 418 __recover_inline_status(inode, node_page); ··· 444 466 (fi->i_flags & F2FS_COMPR_FL)) { 445 467 if (F2FS_FITS_IN_INODE(ri, fi->i_extra_isize, 446 468 i_log_cluster_size)) { 469 + unsigned short compress_flag; 470 + 447 471 atomic_set(&fi->i_compr_blocks, 448 472 le64_to_cpu(ri->i_compr_blocks)); 449 473 fi->i_compress_algorithm = ri->i_compress_algorithm; 450 474 fi->i_log_cluster_size = ri->i_log_cluster_size; 451 - fi->i_compress_flag = le16_to_cpu(ri->i_compress_flag); 475 + compress_flag = le16_to_cpu(ri->i_compress_flag); 476 + fi->i_compress_level = compress_flag >> 477 + COMPRESS_LEVEL_OFFSET; 478 + fi->i_compress_flag = compress_flag & 479 + (BIT(COMPRESS_LEVEL_OFFSET) - 1); 452 480 fi->i_cluster_size = 1 << fi->i_log_cluster_size; 453 481 set_inode_flag(inode, FI_COMPRESSED_FILE); 454 482 } ··· 465 481 /* Need all the flag bits */ 466 482 f2fs_init_read_extent_tree(inode, node_page); 467 483 f2fs_init_age_extent_tree(inode); 484 + 485 + if (!sanity_check_inode(inode, node_page)) { 486 + f2fs_put_page(node_page, 1); 487 + f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); 488 + return -EFSCORRUPTED; 489 + } 490 + 491 + if (!sanity_check_extent_cache(inode)) { 492 + f2fs_put_page(node_page, 1); 493 + f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); 494 + return -EFSCORRUPTED; 495 + } 468 496 469 497 f2fs_put_page(node_page, 1); 470 498 ··· 682 686 if (f2fs_sb_has_compression(F2FS_I_SB(inode)) && 683 687 F2FS_FITS_IN_INODE(ri, F2FS_I(inode)->i_extra_isize, 684 688 i_log_cluster_size)) { 689 + unsigned short compress_flag; 690 + 685 691 ri->i_compr_blocks = 686 692 cpu_to_le64(atomic_read( 687 693 &F2FS_I(inode)->i_compr_blocks)); 688 694 ri->i_compress_algorithm = 689 695 F2FS_I(inode)->i_compress_algorithm; 690 - ri->i_compress_flag = 691 - cpu_to_le16(F2FS_I(inode)->i_compress_flag); 696 + compress_flag = F2FS_I(inode)->i_compress_flag | 697 + F2FS_I(inode)->i_compress_level << 698 + COMPRESS_LEVEL_OFFSET; 699 + ri->i_compress_flag = cpu_to_le16(compress_flag); 692 700 ri->i_log_cluster_size = 693 701 F2FS_I(inode)->i_log_cluster_size; 694 702 } ··· 714 714 { 715 715 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 716 716 struct page *node_page; 717 + int count = 0; 717 718 retry: 718 719 node_page = f2fs_get_node_page(sbi, inode->i_ino); 719 720 if (IS_ERR(node_page)) { 720 721 int err = PTR_ERR(node_page); 721 722 722 - if (err == -ENOMEM) { 723 - cond_resched(); 723 + /* The node block was truncated. */ 724 + if (err == -ENOENT) 725 + return; 726 + 727 + if (err == -ENOMEM || ++count <= DEFAULT_RETRY_IO_COUNT) 724 728 goto retry; 725 - } else if (err != -ENOENT) { 726 - f2fs_stop_checkpoint(sbi, false, 727 - STOP_CP_REASON_UPDATE_INODE); 728 - } 729 + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_UPDATE_INODE); 729 730 return; 730 731 } 731 732 f2fs_update_inode(inode, node_page); ··· 767 766 void f2fs_evict_inode(struct inode *inode) 768 767 { 769 768 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 770 - nid_t xnid = F2FS_I(inode)->i_xattr_nid; 769 + struct f2fs_inode_info *fi = F2FS_I(inode); 770 + nid_t xnid = fi->i_xattr_nid; 771 771 int err = 0; 772 772 773 773 f2fs_abort_atomic_write(inode, true); 774 + 775 + if (fi->cow_inode) { 776 + clear_inode_flag(fi->cow_inode, FI_COW_FILE); 777 + iput(fi->cow_inode); 778 + fi->cow_inode = NULL; 779 + } 774 780 775 781 trace_f2fs_evict_inode(inode); 776 782 truncate_inode_pages_final(&inode->i_data); ··· 817 809 if (F2FS_HAS_BLOCKS(inode)) 818 810 err = f2fs_truncate(inode); 819 811 820 - if (time_to_inject(sbi, FAULT_EVICT_INODE)) { 821 - f2fs_show_injection_info(sbi, FAULT_EVICT_INODE); 812 + if (time_to_inject(sbi, FAULT_EVICT_INODE)) 822 813 err = -EIO; 823 - } 824 814 825 815 if (!err) { 826 816 f2fs_lock_op(sbi); ··· 863 857 stat_dec_inline_inode(inode); 864 858 stat_dec_compr_inode(inode); 865 859 stat_sub_compr_blocks(inode, 866 - atomic_read(&F2FS_I(inode)->i_compr_blocks)); 860 + atomic_read(&fi->i_compr_blocks)); 867 861 868 862 if (likely(!f2fs_cp_error(sbi) && 869 863 !is_sbi_flag_set(sbi, SBI_CP_DISABLED)))

+90 -96

fs/f2fs/iostat.c

··· 14 14 #include "iostat.h" 15 15 #include <trace/events/f2fs.h> 16 16 17 - #define NUM_PREALLOC_IOSTAT_CTXS 128 18 17 static struct kmem_cache *bio_iostat_ctx_cache; 19 18 static mempool_t *bio_iostat_ctx_pool; 19 + 20 + static inline unsigned long long iostat_get_avg_bytes(struct f2fs_sb_info *sbi, 21 + enum iostat_type type) 22 + { 23 + return sbi->iostat_count[type] ? div64_u64(sbi->iostat_bytes[type], 24 + sbi->iostat_count[type]) : 0; 25 + } 26 + 27 + #define IOSTAT_INFO_SHOW(name, type) \ 28 + seq_printf(seq, "%-23s %-16llu %-16llu %-16llu\n", \ 29 + name":", sbi->iostat_bytes[type], \ 30 + sbi->iostat_count[type], \ 31 + iostat_get_avg_bytes(sbi, type)) 20 32 21 33 int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset) 22 34 { 23 35 struct super_block *sb = seq->private; 24 36 struct f2fs_sb_info *sbi = F2FS_SB(sb); 25 - time64_t now = ktime_get_real_seconds(); 26 37 27 38 if (!sbi->iostat_enable) 28 39 return 0; 29 40 30 - seq_printf(seq, "time: %-16llu\n", now); 41 + seq_printf(seq, "time: %-16llu\n", ktime_get_real_seconds()); 42 + seq_printf(seq, "\t\t\t%-16s %-16s %-16s\n", 43 + "io_bytes", "count", "avg_bytes"); 31 44 32 45 /* print app write IOs */ 33 46 seq_puts(seq, "[WRITE]\n"); 34 - seq_printf(seq, "app buffered data: %-16llu\n", 35 - sbi->rw_iostat[APP_BUFFERED_IO]); 36 - seq_printf(seq, "app direct data: %-16llu\n", 37 - sbi->rw_iostat[APP_DIRECT_IO]); 38 - seq_printf(seq, "app mapped data: %-16llu\n", 39 - sbi->rw_iostat[APP_MAPPED_IO]); 40 - seq_printf(seq, "app buffered cdata: %-16llu\n", 41 - sbi->rw_iostat[APP_BUFFERED_CDATA_IO]); 42 - seq_printf(seq, "app mapped cdata: %-16llu\n", 43 - sbi->rw_iostat[APP_MAPPED_CDATA_IO]); 47 + IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_IO); 48 + IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_IO); 49 + IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_IO); 50 + IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_IO); 51 + IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_IO); 44 52 45 53 /* print fs write IOs */ 46 - seq_printf(seq, "fs data: %-16llu\n", 47 - sbi->rw_iostat[FS_DATA_IO]); 48 - seq_printf(seq, "fs cdata: %-16llu\n", 49 - sbi->rw_iostat[FS_CDATA_IO]); 50 - seq_printf(seq, "fs node: %-16llu\n", 51 - sbi->rw_iostat[FS_NODE_IO]); 52 - seq_printf(seq, "fs meta: %-16llu\n", 53 - sbi->rw_iostat[FS_META_IO]); 54 - seq_printf(seq, "fs gc data: %-16llu\n", 55 - sbi->rw_iostat[FS_GC_DATA_IO]); 56 - seq_printf(seq, "fs gc node: %-16llu\n", 57 - sbi->rw_iostat[FS_GC_NODE_IO]); 58 - seq_printf(seq, "fs cp data: %-16llu\n", 59 - sbi->rw_iostat[FS_CP_DATA_IO]); 60 - seq_printf(seq, "fs cp node: %-16llu\n", 61 - sbi->rw_iostat[FS_CP_NODE_IO]); 62 - seq_printf(seq, "fs cp meta: %-16llu\n", 63 - sbi->rw_iostat[FS_CP_META_IO]); 54 + IOSTAT_INFO_SHOW("fs data", FS_DATA_IO); 55 + IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_IO); 56 + IOSTAT_INFO_SHOW("fs node", FS_NODE_IO); 57 + IOSTAT_INFO_SHOW("fs meta", FS_META_IO); 58 + IOSTAT_INFO_SHOW("fs gc data", FS_GC_DATA_IO); 59 + IOSTAT_INFO_SHOW("fs gc node", FS_GC_NODE_IO); 60 + IOSTAT_INFO_SHOW("fs cp data", FS_CP_DATA_IO); 61 + IOSTAT_INFO_SHOW("fs cp node", FS_CP_NODE_IO); 62 + IOSTAT_INFO_SHOW("fs cp meta", FS_CP_META_IO); 64 63 65 64 /* print app read IOs */ 66 65 seq_puts(seq, "[READ]\n"); 67 - seq_printf(seq, "app buffered data: %-16llu\n", 68 - sbi->rw_iostat[APP_BUFFERED_READ_IO]); 69 - seq_printf(seq, "app direct data: %-16llu\n", 70 - sbi->rw_iostat[APP_DIRECT_READ_IO]); 71 - seq_printf(seq, "app mapped data: %-16llu\n", 72 - sbi->rw_iostat[APP_MAPPED_READ_IO]); 73 - seq_printf(seq, "app buffered cdata: %-16llu\n", 74 - sbi->rw_iostat[APP_BUFFERED_CDATA_READ_IO]); 75 - seq_printf(seq, "app mapped cdata: %-16llu\n", 76 - sbi->rw_iostat[APP_MAPPED_CDATA_READ_IO]); 66 + IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_READ_IO); 67 + IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_READ_IO); 68 + IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_READ_IO); 69 + IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_READ_IO); 70 + IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_READ_IO); 77 71 78 72 /* print fs read IOs */ 79 - seq_printf(seq, "fs data: %-16llu\n", 80 - sbi->rw_iostat[FS_DATA_READ_IO]); 81 - seq_printf(seq, "fs gc data: %-16llu\n", 82 - sbi->rw_iostat[FS_GDATA_READ_IO]); 83 - seq_printf(seq, "fs cdata: %-16llu\n", 84 - sbi->rw_iostat[FS_CDATA_READ_IO]); 85 - seq_printf(seq, "fs node: %-16llu\n", 86 - sbi->rw_iostat[FS_NODE_READ_IO]); 87 - seq_printf(seq, "fs meta: %-16llu\n", 88 - sbi->rw_iostat[FS_META_READ_IO]); 73 + IOSTAT_INFO_SHOW("fs data", FS_DATA_READ_IO); 74 + IOSTAT_INFO_SHOW("fs gc data", FS_GDATA_READ_IO); 75 + IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_READ_IO); 76 + IOSTAT_INFO_SHOW("fs node", FS_NODE_READ_IO); 77 + IOSTAT_INFO_SHOW("fs meta", FS_META_READ_IO); 89 78 90 79 /* print other IOs */ 91 80 seq_puts(seq, "[OTHER]\n"); 92 - seq_printf(seq, "fs discard: %-16llu\n", 93 - sbi->rw_iostat[FS_DISCARD]); 81 + IOSTAT_INFO_SHOW("fs discard", FS_DISCARD_IO); 82 + IOSTAT_INFO_SHOW("fs flush", FS_FLUSH_IO); 94 83 95 84 return 0; 96 85 } 97 86 98 87 static inline void __record_iostat_latency(struct f2fs_sb_info *sbi) 99 88 { 100 - int io, idx = 0; 101 - unsigned int cnt; 89 + int io, idx; 102 90 struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE]; 103 91 struct iostat_lat_info *io_lat = sbi->iostat_io_lat; 104 92 unsigned long flags; ··· 94 106 spin_lock_irqsave(&sbi->iostat_lat_lock, flags); 95 107 for (idx = 0; idx < MAX_IO_TYPE; idx++) { 96 108 for (io = 0; io < NR_PAGE_TYPE; io++) { 97 - cnt = io_lat->bio_cnt[idx][io]; 98 109 iostat_lat[idx][io].peak_lat = 99 110 jiffies_to_msecs(io_lat->peak_lat[idx][io]); 100 - iostat_lat[idx][io].cnt = cnt; 101 - iostat_lat[idx][io].avg_lat = cnt ? 102 - jiffies_to_msecs(io_lat->sum_lat[idx][io]) / cnt : 0; 111 + iostat_lat[idx][io].cnt = io_lat->bio_cnt[idx][io]; 112 + iostat_lat[idx][io].avg_lat = iostat_lat[idx][io].cnt ? 113 + jiffies_to_msecs(io_lat->sum_lat[idx][io]) / iostat_lat[idx][io].cnt : 0; 103 114 io_lat->sum_lat[idx][io] = 0; 104 115 io_lat->peak_lat[idx][io] = 0; 105 116 io_lat->bio_cnt[idx][io] = 0; ··· 128 141 msecs_to_jiffies(sbi->iostat_period_ms); 129 142 130 143 for (i = 0; i < NR_IO_TYPE; i++) { 131 - iostat_diff[i] = sbi->rw_iostat[i] - 132 - sbi->prev_rw_iostat[i]; 133 - sbi->prev_rw_iostat[i] = sbi->rw_iostat[i]; 144 + iostat_diff[i] = sbi->iostat_bytes[i] - 145 + sbi->prev_iostat_bytes[i]; 146 + sbi->prev_iostat_bytes[i] = sbi->iostat_bytes[i]; 134 147 } 135 148 spin_unlock_irqrestore(&sbi->iostat_lock, flags); 136 149 ··· 146 159 147 160 spin_lock_irq(&sbi->iostat_lock); 148 161 for (i = 0; i < NR_IO_TYPE; i++) { 149 - sbi->rw_iostat[i] = 0; 150 - sbi->prev_rw_iostat[i] = 0; 162 + sbi->iostat_count[i] = 0; 163 + sbi->iostat_bytes[i] = 0; 164 + sbi->prev_iostat_bytes[i] = 0; 151 165 } 152 166 spin_unlock_irq(&sbi->iostat_lock); 153 167 154 168 spin_lock_irq(&sbi->iostat_lat_lock); 155 169 memset(io_lat, 0, sizeof(struct iostat_lat_info)); 156 170 spin_unlock_irq(&sbi->iostat_lat_lock); 171 + } 172 + 173 + static inline void __f2fs_update_iostat(struct f2fs_sb_info *sbi, 174 + enum iostat_type type, unsigned long long io_bytes) 175 + { 176 + sbi->iostat_bytes[type] += io_bytes; 177 + sbi->iostat_count[type]++; 157 178 } 158 179 159 180 void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode, ··· 173 178 return; 174 179 175 180 spin_lock_irqsave(&sbi->iostat_lock, flags); 176 - sbi->rw_iostat[type] += io_bytes; 181 + __f2fs_update_iostat(sbi, type, io_bytes); 177 182 178 183 if (type == APP_BUFFERED_IO || type == APP_DIRECT_IO) 179 - sbi->rw_iostat[APP_WRITE_IO] += io_bytes; 184 + __f2fs_update_iostat(sbi, APP_WRITE_IO, io_bytes); 180 185 181 186 if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO) 182 - sbi->rw_iostat[APP_READ_IO] += io_bytes; 187 + __f2fs_update_iostat(sbi, APP_READ_IO, io_bytes); 183 188 184 189 #ifdef CONFIG_F2FS_FS_COMPRESSION 185 190 if (inode && f2fs_compressed_file(inode)) { 186 191 if (type == APP_BUFFERED_IO) 187 - sbi->rw_iostat[APP_BUFFERED_CDATA_IO] += io_bytes; 192 + __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_IO, io_bytes); 188 193 189 194 if (type == APP_BUFFERED_READ_IO) 190 - sbi->rw_iostat[APP_BUFFERED_CDATA_READ_IO] += io_bytes; 195 + __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_READ_IO, io_bytes); 191 196 192 197 if (type == APP_MAPPED_READ_IO) 193 - sbi->rw_iostat[APP_MAPPED_CDATA_READ_IO] += io_bytes; 198 + __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_READ_IO, io_bytes); 194 199 195 200 if (type == APP_MAPPED_IO) 196 - sbi->rw_iostat[APP_MAPPED_CDATA_IO] += io_bytes; 201 + __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_IO, io_bytes); 197 202 198 203 if (type == FS_DATA_READ_IO) 199 - sbi->rw_iostat[FS_CDATA_READ_IO] += io_bytes; 204 + __f2fs_update_iostat(sbi, FS_CDATA_READ_IO, io_bytes); 200 205 201 206 if (type == FS_DATA_IO) 202 - sbi->rw_iostat[FS_CDATA_IO] += io_bytes; 207 + __f2fs_update_iostat(sbi, FS_CDATA_IO, io_bytes); 203 208 } 204 209 #endif 205 210 ··· 209 214 } 210 215 211 216 static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx, 212 - int rw, bool is_sync) 217 + enum iostat_lat_type lat_type) 213 218 { 214 219 unsigned long ts_diff; 215 - unsigned int iotype = iostat_ctx->type; 220 + unsigned int page_type = iostat_ctx->type; 216 221 struct f2fs_sb_info *sbi = iostat_ctx->sbi; 217 222 struct iostat_lat_info *io_lat = sbi->iostat_io_lat; 218 - int idx; 219 223 unsigned long flags; 220 224 221 225 if (!sbi->iostat_enable) 222 226 return; 223 227 224 228 ts_diff = jiffies - iostat_ctx->submit_ts; 225 - if (iotype >= META_FLUSH) 226 - iotype = META; 227 - 228 - if (rw == 0) { 229 - idx = READ_IO; 230 - } else { 231 - if (is_sync) 232 - idx = WRITE_SYNC_IO; 233 - else 234 - idx = WRITE_ASYNC_IO; 229 + if (page_type == META_FLUSH) { 230 + page_type = META; 231 + } else if (page_type >= NR_PAGE_TYPE) { 232 + f2fs_warn(sbi, "%s: %d over NR_PAGE_TYPE", __func__, page_type); 233 + return; 235 234 } 236 235 237 236 spin_lock_irqsave(&sbi->iostat_lat_lock, flags); 238 - io_lat->sum_lat[idx][iotype] += ts_diff; 239 - io_lat->bio_cnt[idx][iotype]++; 240 - if (ts_diff > io_lat->peak_lat[idx][iotype]) 241 - io_lat->peak_lat[idx][iotype] = ts_diff; 237 + io_lat->sum_lat[lat_type][page_type] += ts_diff; 238 + io_lat->bio_cnt[lat_type][page_type]++; 239 + if (ts_diff > io_lat->peak_lat[lat_type][page_type]) 240 + io_lat->peak_lat[lat_type][page_type] = ts_diff; 242 241 spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags); 243 242 } 244 243 245 - void iostat_update_and_unbind_ctx(struct bio *bio, int rw) 244 + void iostat_update_and_unbind_ctx(struct bio *bio) 246 245 { 247 246 struct bio_iostat_ctx *iostat_ctx = bio->bi_private; 248 - bool is_sync = bio->bi_opf & REQ_SYNC; 247 + enum iostat_lat_type lat_type; 249 248 250 - if (rw == 0) 251 - bio->bi_private = iostat_ctx->post_read_ctx; 252 - else 249 + if (op_is_write(bio_op(bio))) { 250 + lat_type = bio->bi_opf & REQ_SYNC ? 251 + WRITE_SYNC_IO : WRITE_ASYNC_IO; 253 252 bio->bi_private = iostat_ctx->sbi; 254 - __update_iostat_latency(iostat_ctx, rw, is_sync); 253 + } else { 254 + lat_type = READ_IO; 255 + bio->bi_private = iostat_ctx->post_read_ctx; 256 + } 257 + 258 + __update_iostat_latency(iostat_ctx, lat_type); 255 259 mempool_free(iostat_ctx, bio_iostat_ctx_pool); 256 260 } 257 261

+12 -11

fs/f2fs/iostat.h

··· 8 8 9 9 struct bio_post_read_ctx; 10 10 11 - #ifdef CONFIG_F2FS_IOSTAT 12 - 13 - #define DEFAULT_IOSTAT_PERIOD_MS 3000 14 - #define MIN_IOSTAT_PERIOD_MS 100 15 - /* maximum period of iostat tracing is 1 day */ 16 - #define MAX_IOSTAT_PERIOD_MS 8640000 17 - 18 - enum { 19 - READ_IO, 11 + enum iostat_lat_type { 12 + READ_IO = 0, 20 13 WRITE_SYNC_IO, 21 14 WRITE_ASYNC_IO, 22 15 MAX_IO_TYPE, 23 16 }; 17 + 18 + #ifdef CONFIG_F2FS_IOSTAT 19 + 20 + #define NUM_PREALLOC_IOSTAT_CTXS 128 21 + #define DEFAULT_IOSTAT_PERIOD_MS 3000 22 + #define MIN_IOSTAT_PERIOD_MS 100 23 + /* maximum period of iostat tracing is 1 day */ 24 + #define MAX_IOSTAT_PERIOD_MS 8640000 24 25 25 26 struct iostat_lat_info { 26 27 unsigned long sum_lat[MAX_IO_TYPE][NR_PAGE_TYPE]; /* sum of io latencies */ ··· 58 57 return iostat_ctx->post_read_ctx; 59 58 } 60 59 61 - extern void iostat_update_and_unbind_ctx(struct bio *bio, int rw); 60 + extern void iostat_update_and_unbind_ctx(struct bio *bio); 62 61 extern void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi, 63 62 struct bio *bio, struct bio_post_read_ctx *ctx); 64 63 extern int f2fs_init_iostat_processing(void); ··· 68 67 #else 69 68 static inline void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode, 70 69 enum iostat_type type, unsigned long long io_bytes) {} 71 - static inline void iostat_update_and_unbind_ctx(struct bio *bio, int rw) {} 70 + static inline void iostat_update_and_unbind_ctx(struct bio *bio) {} 72 71 static inline void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi, 73 72 struct bio *bio, struct bio_post_read_ctx *ctx) {} 74 73 static inline void iostat_update_submit_ctx(struct bio *bio,

+1 -4

fs/f2fs/namei.c

··· 926 926 static int f2fs_create_whiteout(struct mnt_idmap *idmap, 927 927 struct inode *dir, struct inode **whiteout) 928 928 { 929 - if (unlikely(f2fs_cp_error(F2FS_I_SB(dir)))) 930 - return -EIO; 931 - 932 929 return __f2fs_tmpfile(idmap, dir, NULL, 933 930 S_IFCHR | WHITEOUT_MODE, true, whiteout); 934 931 } ··· 963 966 964 967 /* 965 968 * If new_inode is null, the below renaming flow will 966 - * add a link in old_dir which can conver inline_dir. 969 + * add a link in old_dir which can convert inline_dir. 967 970 * After then, if we failed to get the entry due to other 968 971 * reasons like ENOMEM, we had to remove the new entry. 969 972 * Instead of adding such the error handling routine, let's

+2 -7

fs/f2fs/node.c

··· 1587 1587 .op_flags = wbc_to_write_flags(wbc), 1588 1588 .page = page, 1589 1589 .encrypted_page = NULL, 1590 - .submitted = false, 1590 + .submitted = 0, 1591 1591 .io_type = io_type, 1592 1592 .io_wbc = wbc, 1593 1593 }; ··· 1651 1651 } 1652 1652 1653 1653 set_page_writeback(page); 1654 - ClearPageError(page); 1655 1654 1656 1655 fio.old_blkaddr = ni.blk_addr; 1657 1656 f2fs_do_write_node_page(nid, &fio); ··· 2082 2083 spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); 2083 2084 2084 2085 f2fs_wait_on_page_writeback(page, NODE, true, false); 2085 - if (TestClearPageError(page)) 2086 - ret = -EIO; 2087 2086 2088 2087 put_page(page); 2089 2088 ··· 2545 2548 struct f2fs_nm_info *nm_i = NM_I(sbi); 2546 2549 struct free_nid *i = NULL; 2547 2550 retry: 2548 - if (time_to_inject(sbi, FAULT_ALLOC_NID)) { 2549 - f2fs_show_injection_info(sbi, FAULT_ALLOC_NID); 2551 + if (time_to_inject(sbi, FAULT_ALLOC_NID)) 2550 2552 return false; 2551 - } 2552 2553 2553 2554 spin_lock(&nm_i->nid_list_lock); 2554 2555

+101 -124

fs/f2fs/segment.c

··· 192 192 if (!f2fs_is_atomic_file(inode)) 193 193 return; 194 194 195 - clear_inode_flag(fi->cow_inode, FI_COW_FILE); 196 - iput(fi->cow_inode); 197 - fi->cow_inode = NULL; 198 195 release_atomic_write_cnt(inode); 199 196 clear_inode_flag(inode, FI_ATOMIC_COMMITTED); 200 197 clear_inode_flag(inode, FI_ATOMIC_REPLACE); 201 198 clear_inode_flag(inode, FI_ATOMIC_FILE); 202 199 stat_dec_atomic_inode(inode); 203 200 201 + F2FS_I(inode)->atomic_write_task = NULL; 202 + 204 203 if (clean) { 205 204 truncate_inode_pages_final(inode->i_mapping); 206 205 f2fs_i_size_write(inode, fi->original_i_size); 206 + fi->original_i_size = 0; 207 207 } 208 208 } 209 209 ··· 255 255 } 256 256 257 257 f2fs_put_dnode(&dn); 258 + 259 + trace_f2fs_replace_atomic_write_block(inode, F2FS_I(inode)->cow_inode, 260 + index, *old_addr, new_addr, recover); 258 261 return 0; 259 262 } 260 263 ··· 265 262 bool revoke) 266 263 { 267 264 struct revoke_entry *cur, *tmp; 265 + pgoff_t start_index = 0; 268 266 bool truncate = is_inode_flag_set(inode, FI_ATOMIC_REPLACE); 269 267 270 268 list_for_each_entry_safe(cur, tmp, head, list) { 271 - if (revoke) 269 + if (revoke) { 272 270 __replace_atomic_write_block(inode, cur->index, 273 271 cur->old_addr, NULL, true); 272 + } else if (truncate) { 273 + f2fs_truncate_hole(inode, start_index, cur->index); 274 + start_index = cur->index + 1; 275 + } 274 276 275 277 list_del(&cur->list); 276 278 kmem_cache_free(revoke_entry_slab, cur); 277 279 } 278 280 279 281 if (!revoke && truncate) 280 - f2fs_do_truncate_blocks(inode, 0, false); 282 + f2fs_do_truncate_blocks(inode, start_index * PAGE_SIZE, false); 281 283 } 282 284 283 285 static int __f2fs_commit_atomic_write(struct inode *inode) ··· 392 384 */ 393 385 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) 394 386 { 395 - if (time_to_inject(sbi, FAULT_CHECKPOINT)) { 396 - f2fs_show_injection_info(sbi, FAULT_CHECKPOINT); 387 + if (time_to_inject(sbi, FAULT_CHECKPOINT)) 397 388 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT); 398 - } 399 389 400 390 /* balance_fs_bg is able to be pending */ 401 391 if (need && excess_cached_nats(sbi)) ··· 514 508 515 509 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), 516 510 test_opt(sbi, FLUSH_MERGE), ret); 511 + if (!ret) 512 + f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0); 517 513 return ret; 518 514 } 519 515 ··· 1067 1059 dpolicy->granularity = granularity; 1068 1060 1069 1061 dpolicy->max_requests = dcc->max_discard_request; 1070 - dpolicy->io_aware_gran = MAX_PLIST_NUM; 1062 + dpolicy->io_aware_gran = dcc->discard_io_aware_gran; 1071 1063 dpolicy->timeout = false; 1072 1064 1073 1065 if (discard_type == DPOLICY_BG) { ··· 1103 1095 block_t start, block_t len); 1104 1096 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */ 1105 1097 static int __submit_discard_cmd(struct f2fs_sb_info *sbi, 1106 - struct discard_policy *dpolicy, 1107 - struct discard_cmd *dc, 1108 - unsigned int *issued) 1098 + struct discard_policy *dpolicy, 1099 + struct discard_cmd *dc, int *issued) 1109 1100 { 1110 1101 struct block_device *bdev = dc->bdev; 1111 1102 unsigned int max_discard_blocks = ··· 1148 1141 dc->len += len; 1149 1142 1150 1143 if (time_to_inject(sbi, FAULT_DISCARD)) { 1151 - f2fs_show_injection_info(sbi, FAULT_DISCARD); 1152 1144 err = -EIO; 1153 1145 } else { 1154 1146 err = __blkdev_issue_discard(bdev, ··· 1192 1186 1193 1187 atomic_inc(&dcc->issued_discard); 1194 1188 1195 - f2fs_update_iostat(sbi, NULL, FS_DISCARD, len * F2FS_BLKSIZE); 1189 + f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, len * F2FS_BLKSIZE); 1196 1190 1197 1191 lstart += len; 1198 1192 start += len; ··· 1384 1378 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); 1385 1379 } 1386 1380 1387 - static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, 1388 - struct discard_policy *dpolicy) 1381 + static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, 1382 + struct discard_policy *dpolicy, int *issued) 1389 1383 { 1390 1384 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1391 1385 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; ··· 1393 1387 struct discard_cmd *dc; 1394 1388 struct blk_plug plug; 1395 1389 unsigned int pos = dcc->next_pos; 1396 - unsigned int issued = 0; 1397 1390 bool io_interrupted = false; 1398 1391 1399 1392 mutex_lock(&dcc->cmd_lock); ··· 1419 1414 } 1420 1415 1421 1416 dcc->next_pos = dc->lstart + dc->len; 1422 - err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); 1417 + err = __submit_discard_cmd(sbi, dpolicy, dc, issued); 1423 1418 1424 - if (issued >= dpolicy->max_requests) 1419 + if (*issued >= dpolicy->max_requests) 1425 1420 break; 1426 1421 next: 1427 1422 node = rb_next(&dc->rb_node); ··· 1437 1432 1438 1433 mutex_unlock(&dcc->cmd_lock); 1439 1434 1440 - if (!issued && io_interrupted) 1441 - issued = -1; 1442 - 1443 - return issued; 1435 + if (!(*issued) && io_interrupted) 1436 + *issued = -1; 1444 1437 } 1445 1438 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, 1446 1439 struct discard_policy *dpolicy); ··· 1466 1463 if (i + 1 < dpolicy->granularity) 1467 1464 break; 1468 1465 1469 - if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) 1470 - return __issue_discard_cmd_orderly(sbi, dpolicy); 1466 + if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) { 1467 + __issue_discard_cmd_orderly(sbi, dpolicy, &issued); 1468 + return issued; 1469 + } 1471 1470 1472 1471 pend_list = &dcc->pend_list[i]; 1473 1472 ··· 1614 1609 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); 1615 1610 1616 1611 /* wait all */ 1617 - __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1); 1612 + __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, MIN_DISCARD_GRANULARITY); 1618 1613 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1619 - __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1); 1614 + __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, MIN_DISCARD_GRANULARITY); 1620 1615 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1621 1616 1622 1617 return discard_blks; ··· 1658 1653 } 1659 1654 } 1660 1655 1661 - /* This comes from f2fs_put_super */ 1656 + /** 1657 + * f2fs_issue_discard_timeout() - Issue all discard cmd within UMOUNT_DISCARD_TIMEOUT 1658 + * @sbi: the f2fs_sb_info data for discard cmd to issue 1659 + * 1660 + * When UMOUNT_DISCARD_TIMEOUT is exceeded, all remaining discard commands will be dropped 1661 + * 1662 + * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false. 1663 + */ 1662 1664 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) 1663 1665 { 1664 1666 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; ··· 1673 1661 bool dropped; 1674 1662 1675 1663 if (!atomic_read(&dcc->discard_cmd_cnt)) 1676 - return false; 1664 + return true; 1677 1665 1678 1666 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, 1679 1667 dcc->discard_granularity); ··· 1684 1672 __wait_all_discard_cmd(sbi, NULL); 1685 1673 1686 1674 f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt)); 1687 - return dropped; 1675 + return !dropped; 1688 1676 } 1689 1677 1690 1678 static int issue_discard_thread(void *data) ··· 1706 1694 1707 1695 if (sbi->gc_mode == GC_URGENT_HIGH || 1708 1696 !f2fs_available_free_memory(sbi, DISCARD_CACHE)) 1709 - __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1); 1697 + __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1698 + MIN_DISCARD_GRANULARITY); 1710 1699 else 1711 1700 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, 1712 1701 dcc->discard_granularity); 1713 1702 1714 1703 if (dcc->discard_wake) 1715 - dcc->discard_wake = 0; 1704 + dcc->discard_wake = false; 1716 1705 1717 1706 /* clean up pending candidates before going to sleep */ 1718 1707 if (atomic_read(&dcc->queued_discard)) ··· 2078 2065 if (!dcc) 2079 2066 return -ENOMEM; 2080 2067 2068 + dcc->discard_io_aware_gran = MAX_PLIST_NUM; 2081 2069 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY; 2082 2070 dcc->max_ordered_discard = DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY; 2083 2071 if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT) ··· 2341 2327 return is_cp; 2342 2328 } 2343 2329 2344 - /* 2345 - * This function should be resided under the curseg_mutex lock 2346 - */ 2347 - static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, 2348 - struct f2fs_summary *sum) 2330 + static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type) 2349 2331 { 2350 2332 struct curseg_info *curseg = CURSEG_I(sbi, type); 2351 - void *addr = curseg->sum_blk; 2352 2333 2353 - addr += curseg->next_blkoff * sizeof(struct f2fs_summary); 2354 - memcpy(addr, sum, sizeof(struct f2fs_summary)); 2334 + if (sbi->ckpt->alloc_type[type] == SSR) 2335 + return sbi->blocks_per_seg; 2336 + return curseg->next_blkoff; 2355 2337 } 2356 2338 2357 2339 /* ··· 2359 2349 int i, sum_in_page; 2360 2350 2361 2351 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 2362 - if (sbi->ckpt->alloc_type[i] == SSR) 2363 - valid_sum_count += sbi->blocks_per_seg; 2364 - else { 2365 - if (for_ra) 2366 - valid_sum_count += le16_to_cpu( 2367 - F2FS_CKPT(sbi)->cur_data_blkoff[i]); 2368 - else 2369 - valid_sum_count += curseg_blkoff(sbi, i); 2370 - } 2352 + if (sbi->ckpt->alloc_type[i] != SSR && for_ra) 2353 + valid_sum_count += 2354 + le16_to_cpu(F2FS_CKPT(sbi)->cur_data_blkoff[i]); 2355 + else 2356 + valid_sum_count += f2fs_curseg_valid_blocks(sbi, i); 2371 2357 } 2372 2358 2373 2359 sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE - ··· 2634 2628 return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); 2635 2629 } 2636 2630 2637 - /* 2638 - * If a segment is written by LFS manner, next block offset is just obtained 2639 - * by increasing the current block offset. However, if a segment is written by 2640 - * SSR manner, next block offset obtained by calling __next_free_blkoff 2641 - */ 2642 - static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, 2643 - struct curseg_info *seg) 2631 + static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi, 2632 + struct curseg_info *seg) 2644 2633 { 2645 - if (seg->alloc_type == SSR) { 2646 - seg->next_blkoff = 2647 - __next_free_blkoff(sbi, seg->segno, 2648 - seg->next_blkoff + 1); 2649 - } else { 2650 - seg->next_blkoff++; 2651 - if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) { 2652 - /* To allocate block chunks in different sizes, use random number */ 2653 - if (--seg->fragment_remained_chunk <= 0) { 2654 - seg->fragment_remained_chunk = 2655 - get_random_u32_inclusive(1, sbi->max_fragment_chunk); 2656 - seg->next_blkoff += 2657 - get_random_u32_inclusive(1, sbi->max_fragment_hole); 2658 - } 2659 - } 2660 - } 2634 + return __next_free_blkoff(sbi, seg->segno, seg->next_blkoff + 1); 2661 2635 } 2662 2636 2663 2637 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno) ··· 2895 2909 struct curseg_info *curseg = CURSEG_I(sbi, type); 2896 2910 unsigned int old_segno; 2897 2911 2898 - if (!curseg->inited) 2899 - goto alloc; 2900 - 2901 - if (force || curseg->next_blkoff || 2902 - get_valid_blocks(sbi, curseg->segno, new_sec)) 2903 - goto alloc; 2904 - 2905 - if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec)) 2912 + if (!force && curseg->inited && 2913 + !curseg->next_blkoff && 2914 + !get_valid_blocks(sbi, curseg->segno, new_sec) && 2915 + !get_ckpt_valid_blocks(sbi, curseg->segno, new_sec)) 2906 2916 return; 2907 - alloc: 2917 + 2908 2918 old_segno = curseg->segno; 2909 2919 new_curseg(sbi, type, true); 2910 2920 stat_inc_seg_type(sbi, curseg); 2911 2921 locate_dirty_segment(sbi, old_segno); 2912 2922 } 2913 2923 2914 - static void __allocate_new_section(struct f2fs_sb_info *sbi, 2915 - int type, bool force) 2916 - { 2917 - __allocate_new_segment(sbi, type, true, force); 2918 - } 2919 - 2920 2924 void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force) 2921 2925 { 2922 2926 f2fs_down_read(&SM_I(sbi)->curseg_lock); 2923 2927 down_write(&SIT_I(sbi)->sentry_lock); 2924 - __allocate_new_section(sbi, type, force); 2928 + __allocate_new_segment(sbi, type, true, force); 2925 2929 up_write(&SIT_I(sbi)->sentry_lock); 2926 2930 f2fs_up_read(&SM_I(sbi)->curseg_lock); 2927 2931 } ··· 3089 3113 return err; 3090 3114 } 3091 3115 3092 - static bool __has_curseg_space(struct f2fs_sb_info *sbi, 3093 - struct curseg_info *curseg) 3094 - { 3095 - return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi, 3096 - curseg->segno); 3097 - } 3098 - 3099 3116 int f2fs_rw_hint_to_seg_type(enum rw_hint hint) 3100 3117 { 3101 3118 switch (hint) { ··· 3207 3238 return type; 3208 3239 } 3209 3240 3241 + static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi, 3242 + struct curseg_info *seg) 3243 + { 3244 + /* To allocate block chunks in different sizes, use random number */ 3245 + if (--seg->fragment_remained_chunk > 0) 3246 + return; 3247 + 3248 + seg->fragment_remained_chunk = 3249 + get_random_u32_inclusive(1, sbi->max_fragment_chunk); 3250 + seg->next_blkoff += 3251 + get_random_u32_inclusive(1, sbi->max_fragment_hole); 3252 + } 3253 + 3210 3254 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, 3211 3255 block_t old_blkaddr, block_t *new_blkaddr, 3212 3256 struct f2fs_summary *sum, int type, ··· 3230 3248 unsigned long long old_mtime; 3231 3249 bool from_gc = (type == CURSEG_ALL_DATA_ATGC); 3232 3250 struct seg_entry *se = NULL; 3251 + bool segment_full = false; 3233 3252 3234 3253 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3235 3254 ··· 3249 3266 3250 3267 f2fs_wait_discard_bio(sbi, *new_blkaddr); 3251 3268 3252 - /* 3253 - * __add_sum_entry should be resided under the curseg_mutex 3254 - * because, this function updates a summary entry in the 3255 - * current summary block. 3256 - */ 3257 - __add_sum_entry(sbi, type, sum); 3258 - 3259 - __refresh_next_blkoff(sbi, curseg); 3260 - 3269 + curseg->sum_blk->entries[curseg->next_blkoff] = *sum; 3270 + if (curseg->alloc_type == SSR) { 3271 + curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); 3272 + } else { 3273 + curseg->next_blkoff++; 3274 + if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) 3275 + f2fs_randomize_chunk(sbi, curseg); 3276 + } 3277 + if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno)) 3278 + segment_full = true; 3261 3279 stat_inc_block_count(sbi, curseg); 3262 3280 3263 3281 if (from_gc) { ··· 3277 3293 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 3278 3294 update_sit_entry(sbi, old_blkaddr, -1); 3279 3295 3280 - if (!__has_curseg_space(sbi, curseg)) { 3281 - /* 3282 - * Flush out current segment and replace it with new segment. 3283 - */ 3296 + /* 3297 + * If the current segment is full, flush it out and replace it with a 3298 + * new segment. 3299 + */ 3300 + if (segment_full) { 3284 3301 if (from_gc) { 3285 3302 get_atssr_segment(sbi, type, se->type, 3286 3303 AT_SSR, se->mtime); ··· 3316 3331 struct f2fs_bio_info *io; 3317 3332 3318 3333 if (F2FS_IO_ALIGNED(sbi)) 3319 - fio->retry = false; 3334 + fio->retry = 0; 3320 3335 3321 3336 INIT_LIST_HEAD(&fio->list); 3322 - fio->in_list = true; 3337 + fio->in_list = 1; 3323 3338 io = sbi->write_io[fio->type] + fio->temp; 3324 3339 spin_lock(&io->io_lock); 3325 3340 list_add_tail(&fio->list, &io->io_list); ··· 3400 3415 .new_blkaddr = page->index, 3401 3416 .page = page, 3402 3417 .encrypted_page = NULL, 3403 - .in_list = false, 3418 + .in_list = 0, 3404 3419 }; 3405 3420 3406 3421 if (unlikely(page->index >= MAIN_BLKADDR(sbi))) 3407 3422 fio.op_flags &= ~REQ_META; 3408 3423 3409 3424 set_page_writeback(page); 3410 - ClearPageError(page); 3411 3425 f2fs_submit_page_write(&fio); 3412 3426 3413 3427 stat_inc_meta_count(sbi, page->index); ··· 3471 3487 3472 3488 stat_inc_inplace_blocks(fio->sbi); 3473 3489 3474 - if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE))) 3490 + if (fio->bio && !IS_F2FS_IPU_NOCACHE(sbi)) 3475 3491 err = f2fs_merge_page_bio(fio); 3476 3492 else 3477 3493 err = f2fs_submit_page_bio(fio); ··· 3560 3576 } 3561 3577 3562 3578 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); 3563 - __add_sum_entry(sbi, type, sum); 3579 + curseg->sum_blk->entries[curseg->next_blkoff] = *sum; 3564 3580 3565 3581 if (!recover_curseg || recover_newaddr) { 3566 3582 if (!from_gc) ··· 3618 3634 3619 3635 /* submit cached LFS IO */ 3620 3636 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type); 3621 - /* sbumit cached IPU IO */ 3637 + /* submit cached IPU IO */ 3622 3638 f2fs_submit_merged_ipu_write(sbi, NULL, page); 3623 3639 if (ordered) { 3624 3640 wait_on_page_writeback(page); ··· 3869 3885 3870 3886 /* Step 3: write summary entries */ 3871 3887 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 3872 - unsigned short blkoff; 3873 - 3874 3888 seg_i = CURSEG_I(sbi, i); 3875 - if (sbi->ckpt->alloc_type[i] == SSR) 3876 - blkoff = sbi->blocks_per_seg; 3877 - else 3878 - blkoff = curseg_blkoff(sbi, i); 3879 - 3880 - for (j = 0; j < blkoff; j++) { 3889 + for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) { 3881 3890 if (!page) { 3882 3891 page = f2fs_grab_meta_page(sbi, blkaddr++); 3883 3892 kaddr = (unsigned char *)page_address(page); ··· 5103 5126 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS; 5104 5127 5105 5128 if (!f2fs_lfs_mode(sbi)) 5106 - sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC; 5129 + sm_info->ipu_policy = BIT(F2FS_IPU_FSYNC); 5107 5130 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL; 5108 5131 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS; 5109 5132 sm_info->min_seq_blocks = sbi->blocks_per_seg;

+30 -11

fs/f2fs/segment.h

··· 670 670 671 671 #define SMALL_VOLUME_SEGMENTS (16 * 512) /* 16GB */ 672 672 673 + #define F2FS_IPU_DISABLE 0 674 + 675 + /* Modification on enum should be synchronized with ipu_mode_names array */ 673 676 enum { 674 677 F2FS_IPU_FORCE, 675 678 F2FS_IPU_SSR, ··· 682 679 F2FS_IPU_ASYNC, 683 680 F2FS_IPU_NOCACHE, 684 681 F2FS_IPU_HONOR_OPU_WRITE, 682 + F2FS_IPU_MAX, 685 683 }; 684 + 685 + static inline bool IS_F2FS_IPU_DISABLE(struct f2fs_sb_info *sbi) 686 + { 687 + return SM_I(sbi)->ipu_policy == F2FS_IPU_DISABLE; 688 + } 689 + 690 + #define F2FS_IPU_POLICY(name) \ 691 + static inline bool IS_##name(struct f2fs_sb_info *sbi) \ 692 + { \ 693 + return SM_I(sbi)->ipu_policy & BIT(name); \ 694 + } 695 + 696 + F2FS_IPU_POLICY(F2FS_IPU_FORCE); 697 + F2FS_IPU_POLICY(F2FS_IPU_SSR); 698 + F2FS_IPU_POLICY(F2FS_IPU_UTIL); 699 + F2FS_IPU_POLICY(F2FS_IPU_SSR_UTIL); 700 + F2FS_IPU_POLICY(F2FS_IPU_FSYNC); 701 + F2FS_IPU_POLICY(F2FS_IPU_ASYNC); 702 + F2FS_IPU_POLICY(F2FS_IPU_NOCACHE); 703 + F2FS_IPU_POLICY(F2FS_IPU_HONOR_OPU_WRITE); 686 704 687 705 static inline unsigned int curseg_segno(struct f2fs_sb_info *sbi, 688 706 int type) ··· 719 695 return curseg->alloc_type; 720 696 } 721 697 722 - static inline unsigned short curseg_blkoff(struct f2fs_sb_info *sbi, int type) 698 + static inline bool valid_main_segno(struct f2fs_sb_info *sbi, 699 + unsigned int segno) 723 700 { 724 - struct curseg_info *curseg = CURSEG_I(sbi, type); 725 - return curseg->next_blkoff; 726 - } 727 - 728 - static inline void check_seg_range(struct f2fs_sb_info *sbi, unsigned int segno) 729 - { 730 - f2fs_bug_on(sbi, segno > TOTAL_SEGS(sbi) - 1); 701 + return segno <= (MAIN_SEGS(sbi) - 1); 731 702 } 732 703 733 704 static inline void verify_fio_blkaddr(struct f2fs_io_info *fio) ··· 777 758 778 759 /* check segment usage, and check boundary of a given segment number */ 779 760 if (unlikely(GET_SIT_VBLOCKS(raw_sit) > usable_blks_per_seg 780 - || segno > TOTAL_SEGS(sbi) - 1)) { 761 + || !valid_main_segno(sbi, segno))) { 781 762 f2fs_err(sbi, "Wrong valid blocks %d or segno %u", 782 763 GET_SIT_VBLOCKS(raw_sit), segno); 783 764 set_sbi_flag(sbi, SBI_NEED_FSCK); ··· 794 775 unsigned int offset = SIT_BLOCK_OFFSET(start); 795 776 block_t blk_addr = sit_i->sit_base_addr + offset; 796 777 797 - check_seg_range(sbi, start); 778 + f2fs_bug_on(sbi, !valid_main_segno(sbi, start)); 798 779 799 780 #ifdef CONFIG_F2FS_CHECK_FS 800 781 if (f2fs_test_bit(offset, sit_i->sit_bitmap) != ··· 943 924 if (!wakeup || !is_idle(sbi, DISCARD_TIME)) 944 925 return; 945 926 wake_up: 946 - dcc->discard_wake = 1; 927 + dcc->discard_wake = true; 947 928 wake_up_interruptible_all(&dcc->discard_wait_queue); 948 929 }

+32 -31

fs/f2fs/super.c

··· 1288 1288 * zone alignment optimization. This is optional for host-aware 1289 1289 * devices, but mandatory for host-managed zoned block devices. 1290 1290 */ 1291 - #ifndef CONFIG_BLK_DEV_ZONED 1292 1291 if (f2fs_sb_has_blkzoned(sbi)) { 1293 - f2fs_err(sbi, "Zoned block device support is not enabled"); 1294 - return -EINVAL; 1295 - } 1296 - #endif 1297 - if (f2fs_sb_has_blkzoned(sbi)) { 1292 + #ifdef CONFIG_BLK_DEV_ZONED 1298 1293 if (F2FS_OPTION(sbi).discard_unit != 1299 1294 DISCARD_UNIT_SECTION) { 1300 1295 f2fs_info(sbi, "Zoned block device doesn't need small discard, set discard_unit=section by default"); 1301 1296 F2FS_OPTION(sbi).discard_unit = 1302 1297 DISCARD_UNIT_SECTION; 1303 1298 } 1299 + #else 1300 + f2fs_err(sbi, "Zoned block device support is not enabled"); 1301 + return -EINVAL; 1302 + #endif 1304 1303 } 1305 1304 1306 1305 #ifdef CONFIG_F2FS_FS_COMPRESSION ··· 1340 1341 } 1341 1342 1342 1343 if (test_opt(sbi, DISABLE_CHECKPOINT) && f2fs_lfs_mode(sbi)) { 1343 - f2fs_err(sbi, "LFS not compatible with checkpoint=disable"); 1344 + f2fs_err(sbi, "LFS is not compatible with checkpoint=disable"); 1344 1345 return -EINVAL; 1345 1346 } 1346 1347 1347 1348 if (test_opt(sbi, ATGC) && f2fs_lfs_mode(sbi)) { 1348 - f2fs_err(sbi, "LFS not compatible with ATGC"); 1349 + f2fs_err(sbi, "LFS is not compatible with ATGC"); 1349 1350 return -EINVAL; 1350 1351 } 1351 1352 ··· 1365 1366 { 1366 1367 struct f2fs_inode_info *fi; 1367 1368 1368 - if (time_to_inject(F2FS_SB(sb), FAULT_SLAB_ALLOC)) { 1369 - f2fs_show_injection_info(F2FS_SB(sb), FAULT_SLAB_ALLOC); 1369 + if (time_to_inject(F2FS_SB(sb), FAULT_SLAB_ALLOC)) 1370 1370 return NULL; 1371 - } 1372 1371 1373 1372 fi = alloc_inode_sb(sb, f2fs_inode_cachep, GFP_F2FS_ZERO); 1374 1373 if (!fi) ··· 1420 1423 /* to avoid evict_inode call simultaneously */ 1421 1424 atomic_inc(&inode->i_count); 1422 1425 spin_unlock(&inode->i_lock); 1423 - 1424 - f2fs_abort_atomic_write(inode, true); 1425 1426 1426 1427 /* should remain fi->extent_tree for writepage */ 1427 1428 f2fs_destroy_extent_node(inode); ··· 1538 1543 { 1539 1544 struct f2fs_sb_info *sbi = F2FS_SB(sb); 1540 1545 int i; 1541 - bool dropped; 1546 + bool done; 1542 1547 1543 1548 /* unregister procfs/sysfs entries in advance to avoid race case */ 1544 1549 f2fs_unregister_sysfs(sbi); ··· 1568 1573 } 1569 1574 1570 1575 /* be sure to wait for any on-going discard commands */ 1571 - dropped = f2fs_issue_discard_timeout(sbi); 1572 - 1573 - if (f2fs_realtime_discard_enable(sbi) && !sbi->discard_blks && !dropped) { 1576 + done = f2fs_issue_discard_timeout(sbi); 1577 + if (f2fs_realtime_discard_enable(sbi) && !sbi->discard_blks && done) { 1574 1578 struct cp_control cpc = { 1575 1579 .reason = CP_UMOUNT | CP_TRIMMED, 1576 1580 }; ··· 1894 1900 1895 1901 if (test_opt(sbi, GC_MERGE)) 1896 1902 seq_puts(seq, ",gc_merge"); 1903 + else 1904 + seq_puts(seq, ",nogc_merge"); 1897 1905 1898 1906 if (test_opt(sbi, DISABLE_ROLL_FORWARD)) 1899 1907 seq_puts(seq, ",disable_roll_forward"); 1900 1908 if (test_opt(sbi, NORECOVERY)) 1901 1909 seq_puts(seq, ",norecovery"); 1902 - if (test_opt(sbi, DISCARD)) 1910 + if (test_opt(sbi, DISCARD)) { 1903 1911 seq_puts(seq, ",discard"); 1904 - else 1912 + if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_BLOCK) 1913 + seq_printf(seq, ",discard_unit=%s", "block"); 1914 + else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT) 1915 + seq_printf(seq, ",discard_unit=%s", "segment"); 1916 + else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION) 1917 + seq_printf(seq, ",discard_unit=%s", "section"); 1918 + } else { 1905 1919 seq_puts(seq, ",nodiscard"); 1920 + } 1906 1921 if (test_opt(sbi, NOHEAP)) 1907 1922 seq_puts(seq, ",no_heap"); 1908 1923 else ··· 2034 2031 2035 2032 if (test_opt(sbi, ATGC)) 2036 2033 seq_puts(seq, ",atgc"); 2037 - 2038 - if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_BLOCK) 2039 - seq_printf(seq, ",discard_unit=%s", "block"); 2040 - else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT) 2041 - seq_printf(seq, ",discard_unit=%s", "segment"); 2042 - else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION) 2043 - seq_printf(seq, ",discard_unit=%s", "section"); 2044 2034 2045 2035 if (F2FS_OPTION(sbi).memory_mode == MEMORY_MODE_NORMAL) 2046 2036 seq_printf(seq, ",memory=%s", "normal"); ··· 2296 2300 } 2297 2301 } 2298 2302 #endif 2303 + if (f2fs_lfs_mode(sbi) && !IS_F2FS_IPU_DISABLE(sbi)) { 2304 + err = -EINVAL; 2305 + f2fs_warn(sbi, "LFS is not compatible with IPU"); 2306 + goto restore_opts; 2307 + } 2308 + 2299 2309 /* disallow enable atgc dynamically */ 2300 2310 if (no_atgc == !!test_opt(sbi, ATGC)) { 2301 2311 err = -EINVAL; ··· 2591 2589 2592 2590 int f2fs_dquot_initialize(struct inode *inode) 2593 2591 { 2594 - if (time_to_inject(F2FS_I_SB(inode), FAULT_DQUOT_INIT)) { 2595 - f2fs_show_injection_info(F2FS_I_SB(inode), FAULT_DQUOT_INIT); 2592 + if (time_to_inject(F2FS_I_SB(inode), FAULT_DQUOT_INIT)) 2596 2593 return -ESRCH; 2597 - } 2598 2594 2599 2595 return dquot_initialize(inode); 2600 2596 } ··· 4083 4083 if (f2fs_block_unit_discard(sbi)) 4084 4084 SM_I(sbi)->dcc_info->discard_granularity = 4085 4085 MIN_DISCARD_GRANULARITY; 4086 - SM_I(sbi)->ipu_policy = 1 << F2FS_IPU_FORCE | 4087 - 1 << F2FS_IPU_HONOR_OPU_WRITE; 4086 + if (!f2fs_lfs_mode(sbi)) 4087 + SM_I(sbi)->ipu_policy = BIT(F2FS_IPU_FORCE) | 4088 + BIT(F2FS_IPU_HONOR_OPU_WRITE); 4088 4089 } 4089 4090 4090 4091 sbi->readdir_ra = true;

+41 -8

fs/f2fs/sysfs.c

··· 473 473 return count; 474 474 } 475 475 476 + if (!strcmp(a->attr.name, "discard_io_aware_gran")) { 477 + if (t > MAX_PLIST_NUM) 478 + return -EINVAL; 479 + if (!f2fs_block_unit_discard(sbi)) 480 + return -EINVAL; 481 + if (t == *ui) 482 + return count; 483 + *ui = t; 484 + return count; 485 + } 486 + 476 487 if (!strcmp(a->attr.name, "discard_granularity")) { 477 488 if (t == 0 || t > MAX_PLIST_NUM) 478 489 return -EINVAL; ··· 522 511 } else if (t == 1) { 523 512 sbi->gc_mode = GC_URGENT_HIGH; 524 513 if (sbi->gc_thread) { 525 - sbi->gc_thread->gc_wake = 1; 514 + sbi->gc_thread->gc_wake = true; 526 515 wake_up_interruptible_all( 527 516 &sbi->gc_thread->gc_wait_queue_head); 528 517 wake_up_discard_thread(sbi, true); ··· 532 521 } else if (t == 3) { 533 522 sbi->gc_mode = GC_URGENT_MID; 534 523 if (sbi->gc_thread) { 535 - sbi->gc_thread->gc_wake = 1; 524 + sbi->gc_thread->gc_wake = true; 536 525 wake_up_interruptible_all( 537 526 &sbi->gc_thread->gc_wait_queue_head); 538 527 } ··· 689 678 } 690 679 691 680 if (!strcmp(a->attr.name, "warm_data_age_threshold")) { 692 - if (t == 0 || t <= sbi->hot_data_age_threshold) 681 + if (t <= sbi->hot_data_age_threshold) 693 682 return -EINVAL; 694 683 if (t == *ui) 695 684 return count; 696 685 *ui = (unsigned int)t; 686 + return count; 687 + } 688 + 689 + if (!strcmp(a->attr.name, "last_age_weight")) { 690 + if (t > 100) 691 + return -EINVAL; 692 + if (t == *ui) 693 + return count; 694 + *ui = (unsigned int)t; 695 + return count; 696 + } 697 + 698 + if (!strcmp(a->attr.name, "ipu_policy")) { 699 + if (t >= BIT(F2FS_IPU_MAX)) 700 + return -EINVAL; 701 + if (t && f2fs_lfs_mode(sbi)) 702 + return -EINVAL; 703 + SM_I(sbi)->ipu_policy = (unsigned int)t; 697 704 return count; 698 705 } 699 706 ··· 854 825 F2FS_RW_ATTR(DCC_INFO, discard_cmd_control, min_discard_issue_time, min_discard_issue_time); 855 826 F2FS_RW_ATTR(DCC_INFO, discard_cmd_control, mid_discard_issue_time, mid_discard_issue_time); 856 827 F2FS_RW_ATTR(DCC_INFO, discard_cmd_control, max_discard_issue_time, max_discard_issue_time); 828 + F2FS_RW_ATTR(DCC_INFO, discard_cmd_control, discard_io_aware_gran, discard_io_aware_gran); 857 829 F2FS_RW_ATTR(DCC_INFO, discard_cmd_control, discard_urgent_util, discard_urgent_util); 858 830 F2FS_RW_ATTR(DCC_INFO, discard_cmd_control, discard_granularity, discard_granularity); 859 831 F2FS_RW_ATTR(DCC_INFO, discard_cmd_control, max_ordered_discard, max_ordered_discard); ··· 974 944 /* For block age extent cache */ 975 945 F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, hot_data_age_threshold, hot_data_age_threshold); 976 946 F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, warm_data_age_threshold, warm_data_age_threshold); 947 + F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, last_age_weight, last_age_weight); 977 948 978 949 #define ATTR_LIST(name) (&f2fs_attr_##name.attr) 979 950 static struct attribute *f2fs_attrs[] = { ··· 991 960 ATTR_LIST(min_discard_issue_time), 992 961 ATTR_LIST(mid_discard_issue_time), 993 962 ATTR_LIST(max_discard_issue_time), 963 + ATTR_LIST(discard_io_aware_gran), 994 964 ATTR_LIST(discard_urgent_util), 995 965 ATTR_LIST(discard_granularity), 996 966 ATTR_LIST(max_ordered_discard), ··· 1074 1042 ATTR_LIST(revoked_atomic_block), 1075 1043 ATTR_LIST(hot_data_age_threshold), 1076 1044 ATTR_LIST(warm_data_age_threshold), 1045 + ATTR_LIST(last_age_weight), 1077 1046 NULL, 1078 1047 }; 1079 1048 ATTRIBUTE_GROUPS(f2fs); ··· 1162 1129 .store = f2fs_attr_store, 1163 1130 }; 1164 1131 1165 - static struct kobj_type f2fs_sb_ktype = { 1132 + static const struct kobj_type f2fs_sb_ktype = { 1166 1133 .default_groups = f2fs_groups, 1167 1134 .sysfs_ops = &f2fs_attr_ops, 1168 1135 .release = f2fs_sb_release, 1169 1136 }; 1170 1137 1171 - static struct kobj_type f2fs_ktype = { 1138 + static const struct kobj_type f2fs_ktype = { 1172 1139 .sysfs_ops = &f2fs_attr_ops, 1173 1140 }; 1174 1141 ··· 1176 1143 .kobj = {.ktype = &f2fs_ktype}, 1177 1144 }; 1178 1145 1179 - static struct kobj_type f2fs_feat_ktype = { 1146 + static const struct kobj_type f2fs_feat_ktype = { 1180 1147 .default_groups = f2fs_feat_groups, 1181 1148 .sysfs_ops = &f2fs_attr_ops, 1182 1149 }; ··· 1217 1184 .store = f2fs_stat_attr_store, 1218 1185 }; 1219 1186 1220 - static struct kobj_type f2fs_stat_ktype = { 1187 + static const struct kobj_type f2fs_stat_ktype = { 1221 1188 .default_groups = f2fs_stat_groups, 1222 1189 .sysfs_ops = &f2fs_stat_attr_ops, 1223 1190 .release = f2fs_stat_kobj_release, ··· 1244 1211 .show = f2fs_sb_feat_attr_show, 1245 1212 }; 1246 1213 1247 - static struct kobj_type f2fs_feature_list_ktype = { 1214 + static const struct kobj_type f2fs_feature_list_ktype = { 1248 1215 .default_groups = f2fs_sb_feat_groups, 1249 1216 .sysfs_ops = &f2fs_feature_list_attr_ops, 1250 1217 .release = f2fs_feature_list_kobj_release,

+1 -1

fs/f2fs/verity.c

··· 81 81 size_t n = min_t(size_t, count, 82 82 PAGE_SIZE - offset_in_page(pos)); 83 83 struct page *page; 84 - void *fsdata; 84 + void *fsdata = NULL; 85 85 int res; 86 86 87 87 res = aops->write_begin(NULL, mapping, pos, n, &page, &fsdata);

+1 -1

include/linux/f2fs_fs.h

··· 315 315 __u8 i_log_cluster_size; /* log of cluster size */ 316 316 __le16 i_compress_flag; /* compress flag */ 317 317 /* 0 bit: chksum flag 318 - * [10,15] bits: compress level 318 + * [8,15] bits: compress level 319 319 */ 320 320 __le32 i_extra_end[0]; /* for attribute size calculation */ 321 321 } __packed;

+69 -35

include/trace/events/f2fs.h

··· 569 569 ); 570 570 571 571 TRACE_EVENT(f2fs_map_blocks, 572 - TP_PROTO(struct inode *inode, struct f2fs_map_blocks *map, 573 - int create, int flag, int ret), 572 + TP_PROTO(struct inode *inode, struct f2fs_map_blocks *map, int flag, 573 + int ret), 574 574 575 - TP_ARGS(inode, map, create, flag, ret), 575 + TP_ARGS(inode, map, flag, ret), 576 576 577 577 TP_STRUCT__entry( 578 578 __field(dev_t, dev) ··· 584 584 __field(int, m_seg_type) 585 585 __field(bool, m_may_create) 586 586 __field(bool, m_multidev_dio) 587 - __field(int, create) 588 587 __field(int, flag) 589 588 __field(int, ret) 590 589 ), ··· 598 599 __entry->m_seg_type = map->m_seg_type; 599 600 __entry->m_may_create = map->m_may_create; 600 601 __entry->m_multidev_dio = map->m_multidev_dio; 601 - __entry->create = create; 602 602 __entry->flag = flag; 603 603 __entry->ret = ret; 604 604 ), ··· 605 607 TP_printk("dev = (%d,%d), ino = %lu, file offset = %llu, " 606 608 "start blkaddr = 0x%llx, len = 0x%llx, flags = %u, " 607 609 "seg_type = %d, may_create = %d, multidevice = %d, " 608 - "create = %d, flag = %d, err = %d", 610 + "flag = %d, err = %d", 609 611 show_dev_ino(__entry), 610 612 (unsigned long long)__entry->m_lblk, 611 613 (unsigned long long)__entry->m_pblk, ··· 614 616 __entry->m_seg_type, 615 617 __entry->m_may_create, 616 618 __entry->m_multidev_dio, 617 - __entry->create, 618 619 __entry->flag, 619 620 __entry->ret) 620 621 ); ··· 1288 1291 TP_PROTO(struct page *page, int type), 1289 1292 1290 1293 TP_ARGS(page, type) 1294 + ); 1295 + 1296 + TRACE_EVENT(f2fs_replace_atomic_write_block, 1297 + 1298 + TP_PROTO(struct inode *inode, struct inode *cow_inode, pgoff_t index, 1299 + block_t old_addr, block_t new_addr, bool recovery), 1300 + 1301 + TP_ARGS(inode, cow_inode, index, old_addr, new_addr, recovery), 1302 + 1303 + TP_STRUCT__entry( 1304 + __field(dev_t, dev) 1305 + __field(ino_t, ino) 1306 + __field(ino_t, cow_ino) 1307 + __field(pgoff_t, index) 1308 + __field(block_t, old_addr) 1309 + __field(block_t, new_addr) 1310 + __field(bool, recovery) 1311 + ), 1312 + 1313 + TP_fast_assign( 1314 + __entry->dev = inode->i_sb->s_dev; 1315 + __entry->ino = inode->i_ino; 1316 + __entry->cow_ino = cow_inode->i_ino; 1317 + __entry->index = index; 1318 + __entry->old_addr = old_addr; 1319 + __entry->new_addr = new_addr; 1320 + __entry->recovery = recovery; 1321 + ), 1322 + 1323 + TP_printk("dev = (%d,%d), ino = %lu, cow_ino = %lu, index = %lu, " 1324 + "old_addr = 0x%llx, new_addr = 0x%llx, recovery = %d", 1325 + show_dev_ino(__entry), 1326 + __entry->cow_ino, 1327 + (unsigned long)__entry->index, 1328 + (unsigned long long)__entry->old_addr, 1329 + (unsigned long long)__entry->new_addr, 1330 + __entry->recovery) 1291 1331 ); 1292 1332 1293 1333 TRACE_EVENT(f2fs_filemap_fault, ··· 2009 1975 __entry->fs_cdrio = iostat[FS_CDATA_READ_IO]; 2010 1976 __entry->fs_nrio = iostat[FS_NODE_READ_IO]; 2011 1977 __entry->fs_mrio = iostat[FS_META_READ_IO]; 2012 - __entry->fs_discard = iostat[FS_DISCARD]; 1978 + __entry->fs_discard = iostat[FS_DISCARD_IO]; 2013 1979 ), 2014 1980 2015 1981 TP_printk("dev = (%d,%d), " ··· 2082 2048 2083 2049 TP_fast_assign( 2084 2050 __entry->dev = sbi->sb->s_dev; 2085 - __entry->d_rd_peak = iostat_lat[0][DATA].peak_lat; 2086 - __entry->d_rd_avg = iostat_lat[0][DATA].avg_lat; 2087 - __entry->d_rd_cnt = iostat_lat[0][DATA].cnt; 2088 - __entry->n_rd_peak = iostat_lat[0][NODE].peak_lat; 2089 - __entry->n_rd_avg = iostat_lat[0][NODE].avg_lat; 2090 - __entry->n_rd_cnt = iostat_lat[0][NODE].cnt; 2091 - __entry->m_rd_peak = iostat_lat[0][META].peak_lat; 2092 - __entry->m_rd_avg = iostat_lat[0][META].avg_lat; 2093 - __entry->m_rd_cnt = iostat_lat[0][META].cnt; 2094 - __entry->d_wr_s_peak = iostat_lat[1][DATA].peak_lat; 2095 - __entry->d_wr_s_avg = iostat_lat[1][DATA].avg_lat; 2096 - __entry->d_wr_s_cnt = iostat_lat[1][DATA].cnt; 2097 - __entry->n_wr_s_peak = iostat_lat[1][NODE].peak_lat; 2098 - __entry->n_wr_s_avg = iostat_lat[1][NODE].avg_lat; 2099 - __entry->n_wr_s_cnt = iostat_lat[1][NODE].cnt; 2100 - __entry->m_wr_s_peak = iostat_lat[1][META].peak_lat; 2101 - __entry->m_wr_s_avg = iostat_lat[1][META].avg_lat; 2102 - __entry->m_wr_s_cnt = iostat_lat[1][META].cnt; 2103 - __entry->d_wr_as_peak = iostat_lat[2][DATA].peak_lat; 2104 - __entry->d_wr_as_avg = iostat_lat[2][DATA].avg_lat; 2105 - __entry->d_wr_as_cnt = iostat_lat[2][DATA].cnt; 2106 - __entry->n_wr_as_peak = iostat_lat[2][NODE].peak_lat; 2107 - __entry->n_wr_as_avg = iostat_lat[2][NODE].avg_lat; 2108 - __entry->n_wr_as_cnt = iostat_lat[2][NODE].cnt; 2109 - __entry->m_wr_as_peak = iostat_lat[2][META].peak_lat; 2110 - __entry->m_wr_as_avg = iostat_lat[2][META].avg_lat; 2111 - __entry->m_wr_as_cnt = iostat_lat[2][META].cnt; 2051 + __entry->d_rd_peak = iostat_lat[READ_IO][DATA].peak_lat; 2052 + __entry->d_rd_avg = iostat_lat[READ_IO][DATA].avg_lat; 2053 + __entry->d_rd_cnt = iostat_lat[READ_IO][DATA].cnt; 2054 + __entry->n_rd_peak = iostat_lat[READ_IO][NODE].peak_lat; 2055 + __entry->n_rd_avg = iostat_lat[READ_IO][NODE].avg_lat; 2056 + __entry->n_rd_cnt = iostat_lat[READ_IO][NODE].cnt; 2057 + __entry->m_rd_peak = iostat_lat[READ_IO][META].peak_lat; 2058 + __entry->m_rd_avg = iostat_lat[READ_IO][META].avg_lat; 2059 + __entry->m_rd_cnt = iostat_lat[READ_IO][META].cnt; 2060 + __entry->d_wr_s_peak = iostat_lat[WRITE_SYNC_IO][DATA].peak_lat; 2061 + __entry->d_wr_s_avg = iostat_lat[WRITE_SYNC_IO][DATA].avg_lat; 2062 + __entry->d_wr_s_cnt = iostat_lat[WRITE_SYNC_IO][DATA].cnt; 2063 + __entry->n_wr_s_peak = iostat_lat[WRITE_SYNC_IO][NODE].peak_lat; 2064 + __entry->n_wr_s_avg = iostat_lat[WRITE_SYNC_IO][NODE].avg_lat; 2065 + __entry->n_wr_s_cnt = iostat_lat[WRITE_SYNC_IO][NODE].cnt; 2066 + __entry->m_wr_s_peak = iostat_lat[WRITE_SYNC_IO][META].peak_lat; 2067 + __entry->m_wr_s_avg = iostat_lat[WRITE_SYNC_IO][META].avg_lat; 2068 + __entry->m_wr_s_cnt = iostat_lat[WRITE_SYNC_IO][META].cnt; 2069 + __entry->d_wr_as_peak = iostat_lat[WRITE_ASYNC_IO][DATA].peak_lat; 2070 + __entry->d_wr_as_avg = iostat_lat[WRITE_ASYNC_IO][DATA].avg_lat; 2071 + __entry->d_wr_as_cnt = iostat_lat[WRITE_ASYNC_IO][DATA].cnt; 2072 + __entry->n_wr_as_peak = iostat_lat[WRITE_ASYNC_IO][NODE].peak_lat; 2073 + __entry->n_wr_as_avg = iostat_lat[WRITE_ASYNC_IO][NODE].avg_lat; 2074 + __entry->n_wr_as_cnt = iostat_lat[WRITE_ASYNC_IO][NODE].cnt; 2075 + __entry->m_wr_as_peak = iostat_lat[WRITE_ASYNC_IO][META].peak_lat; 2076 + __entry->m_wr_as_avg = iostat_lat[WRITE_ASYNC_IO][META].avg_lat; 2077 + __entry->m_wr_as_cnt = iostat_lat[WRITE_ASYNC_IO][META].cnt; 2112 2078 ), 2113 2079 2114 2080 TP_printk("dev = (%d,%d), "

Configure Feed

Configure Feed