Merge branch 'libnvdimm-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm

+157 -92

fs/dax.c

··· 451 451 __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key); 452 452 } 453 453 454 + static int __dax_invalidate_mapping_entry(struct address_space *mapping, 455 + pgoff_t index, bool trunc) 456 + { 457 + int ret = 0; 458 + void *entry; 459 + struct radix_tree_root *page_tree = &mapping->page_tree; 460 + 461 + spin_lock_irq(&mapping->tree_lock); 462 + entry = get_unlocked_mapping_entry(mapping, index, NULL); 463 + if (!entry || !radix_tree_exceptional_entry(entry)) 464 + goto out; 465 + if (!trunc && 466 + (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) || 467 + radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))) 468 + goto out; 469 + radix_tree_delete(page_tree, index); 470 + mapping->nrexceptional--; 471 + ret = 1; 472 + out: 473 + put_unlocked_mapping_entry(mapping, index, entry); 474 + spin_unlock_irq(&mapping->tree_lock); 475 + return ret; 476 + } 454 477 /* 455 478 * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree 456 479 * entry to get unlocked before deleting it. 457 480 */ 458 481 int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index) 459 482 { 460 - void *entry; 483 + int ret = __dax_invalidate_mapping_entry(mapping, index, true); 461 484 462 - spin_lock_irq(&mapping->tree_lock); 463 - entry = get_unlocked_mapping_entry(mapping, index, NULL); 464 485 /* 465 486 * This gets called from truncate / punch_hole path. As such, the caller 466 487 * must hold locks protecting against concurrent modifications of the ··· 489 468 * caller has seen exceptional entry for this index, we better find it 490 469 * at that index as well... 491 470 */ 492 - if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry))) { 493 - spin_unlock_irq(&mapping->tree_lock); 494 - return 0; 495 - } 496 - radix_tree_delete(&mapping->page_tree, index); 497 - mapping->nrexceptional--; 498 - spin_unlock_irq(&mapping->tree_lock); 499 - dax_wake_mapping_entry_waiter(mapping, index, entry, true); 471 + WARN_ON_ONCE(!ret); 472 + return ret; 473 + } 500 474 501 - return 1; 475 + /* 476 + * Invalidate exceptional DAX entry if easily possible. This handles DAX 477 + * entries for invalidate_inode_pages() so we evict the entry only if we can 478 + * do so without blocking. 479 + */ 480 + int dax_invalidate_mapping_entry(struct address_space *mapping, pgoff_t index) 481 + { 482 + int ret = 0; 483 + void *entry, **slot; 484 + struct radix_tree_root *page_tree = &mapping->page_tree; 485 + 486 + spin_lock_irq(&mapping->tree_lock); 487 + entry = __radix_tree_lookup(page_tree, index, NULL, &slot); 488 + if (!entry || !radix_tree_exceptional_entry(entry) || 489 + slot_locked(mapping, slot)) 490 + goto out; 491 + if (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) || 492 + radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)) 493 + goto out; 494 + radix_tree_delete(page_tree, index); 495 + mapping->nrexceptional--; 496 + ret = 1; 497 + out: 498 + spin_unlock_irq(&mapping->tree_lock); 499 + if (ret) 500 + dax_wake_mapping_entry_waiter(mapping, index, entry, true); 501 + return ret; 502 + } 503 + 504 + /* 505 + * Invalidate exceptional DAX entry if it is clean. 506 + */ 507 + int dax_invalidate_mapping_entry_sync(struct address_space *mapping, 508 + pgoff_t index) 509 + { 510 + return __dax_invalidate_mapping_entry(mapping, index, false); 502 511 } 503 512 504 513 /* ··· 539 488 * otherwise it will simply fall out of the page cache under memory 540 489 * pressure without ever having been dirtied. 541 490 */ 542 - static int dax_load_hole(struct address_space *mapping, void *entry, 491 + static int dax_load_hole(struct address_space *mapping, void **entry, 543 492 struct vm_fault *vmf) 544 493 { 545 494 struct page *page; 495 + int ret; 546 496 547 497 /* Hole page already exists? Return it... */ 548 - if (!radix_tree_exceptional_entry(entry)) { 549 - vmf->page = entry; 550 - return VM_FAULT_LOCKED; 498 + if (!radix_tree_exceptional_entry(*entry)) { 499 + page = *entry; 500 + goto out; 551 501 } 552 502 553 503 /* This will replace locked radix tree entry with a hole page */ ··· 556 504 vmf->gfp_mask | __GFP_ZERO); 557 505 if (!page) 558 506 return VM_FAULT_OOM; 507 + out: 559 508 vmf->page = page; 560 - return VM_FAULT_LOCKED; 509 + ret = finish_fault(vmf); 510 + vmf->page = NULL; 511 + *entry = page; 512 + if (!ret) { 513 + /* Grab reference for PTE that is now referencing the page */ 514 + get_page(page); 515 + return VM_FAULT_NOPAGE; 516 + } 517 + return ret; 561 518 } 562 519 563 520 static int copy_user_dax(struct block_device *bdev, sector_t sector, size_t size, ··· 995 934 if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED)) 996 935 return -EIO; 997 936 937 + /* 938 + * Write can allocate block for an area which has a hole page mapped 939 + * into page tables. We have to tear down these mappings so that data 940 + * written by write(2) is visible in mmap. 941 + */ 942 + if ((iomap->flags & IOMAP_F_NEW) && inode->i_mapping->nrpages) { 943 + invalidate_inode_pages2_range(inode->i_mapping, 944 + pos >> PAGE_SHIFT, 945 + (end - 1) >> PAGE_SHIFT); 946 + } 947 + 998 948 while (pos < end) { 999 949 unsigned offset = pos & (PAGE_SIZE - 1); 1000 950 struct blk_dax_ctl dax = { 0 }; ··· 1064 992 if (iov_iter_rw(iter) == WRITE) 1065 993 flags |= IOMAP_WRITE; 1066 994 1067 - /* 1068 - * Yes, even DAX files can have page cache attached to them: A zeroed 1069 - * page is inserted into the pagecache when we have to serve a write 1070 - * fault on a hole. It should never be dirtied and can simply be 1071 - * dropped from the pagecache once we get real data for the page. 1072 - * 1073 - * XXX: This is racy against mmap, and there's nothing we can do about 1074 - * it. We'll eventually need to shift this down even further so that 1075 - * we can check if we allocated blocks over a hole first. 1076 - */ 1077 - if (mapping->nrpages) { 1078 - ret = invalidate_inode_pages2_range(mapping, 1079 - pos >> PAGE_SHIFT, 1080 - (pos + iov_iter_count(iter) - 1) >> PAGE_SHIFT); 1081 - WARN_ON_ONCE(ret); 1082 - } 1083 - 1084 995 while (iov_iter_count(iter)) { 1085 996 ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops, 1086 997 iter, dax_iomap_actor); ··· 1077 1022 return done ? done : ret; 1078 1023 } 1079 1024 EXPORT_SYMBOL_GPL(dax_iomap_rw); 1025 + 1026 + static int dax_fault_return(int error) 1027 + { 1028 + if (error == 0) 1029 + return VM_FAULT_NOPAGE; 1030 + if (error == -ENOMEM) 1031 + return VM_FAULT_OOM; 1032 + return VM_FAULT_SIGBUS; 1033 + } 1080 1034 1081 1035 /** 1082 1036 * dax_iomap_fault - handle a page fault on a DAX file ··· 1119 1055 if (pos >= i_size_read(inode)) 1120 1056 return VM_FAULT_SIGBUS; 1121 1057 1122 - entry = grab_mapping_entry(mapping, vmf->pgoff, 0); 1123 - if (IS_ERR(entry)) { 1124 - error = PTR_ERR(entry); 1125 - goto out; 1126 - } 1127 - 1128 1058 if ((vmf->flags & FAULT_FLAG_WRITE) && !vmf->cow_page) 1129 1059 flags |= IOMAP_WRITE; 1130 1060 ··· 1129 1071 */ 1130 1072 error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap); 1131 1073 if (error) 1132 - goto unlock_entry; 1074 + return dax_fault_return(error); 1133 1075 if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) { 1134 - error = -EIO; /* fs corruption? */ 1076 + vmf_ret = dax_fault_return(-EIO); /* fs corruption? */ 1077 + goto finish_iomap; 1078 + } 1079 + 1080 + entry = grab_mapping_entry(mapping, vmf->pgoff, 0); 1081 + if (IS_ERR(entry)) { 1082 + vmf_ret = dax_fault_return(PTR_ERR(entry)); 1135 1083 goto finish_iomap; 1136 1084 } 1137 1085 ··· 1160 1096 } 1161 1097 1162 1098 if (error) 1163 - goto finish_iomap; 1099 + goto error_unlock_entry; 1164 1100 1165 1101 __SetPageUptodate(vmf->cow_page); 1166 1102 vmf_ret = finish_fault(vmf); 1167 1103 if (!vmf_ret) 1168 1104 vmf_ret = VM_FAULT_DONE_COW; 1169 - goto finish_iomap; 1105 + goto unlock_entry; 1170 1106 } 1171 1107 1172 1108 switch (iomap.type) { ··· 1178 1114 } 1179 1115 error = dax_insert_mapping(mapping, iomap.bdev, sector, 1180 1116 PAGE_SIZE, &entry, vma, vmf); 1117 + /* -EBUSY is fine, somebody else faulted on the same PTE */ 1118 + if (error == -EBUSY) 1119 + error = 0; 1181 1120 break; 1182 1121 case IOMAP_UNWRITTEN: 1183 1122 case IOMAP_HOLE: 1184 1123 if (!(vmf->flags & FAULT_FLAG_WRITE)) { 1185 - vmf_ret = dax_load_hole(mapping, entry, vmf); 1186 - break; 1124 + vmf_ret = dax_load_hole(mapping, &entry, vmf); 1125 + goto unlock_entry; 1187 1126 } 1188 1127 /*FALLTHRU*/ 1189 1128 default: ··· 1195 1128 break; 1196 1129 } 1197 1130 1131 + error_unlock_entry: 1132 + vmf_ret = dax_fault_return(error) | major; 1133 + unlock_entry: 1134 + put_locked_mapping_entry(mapping, vmf->pgoff, entry); 1198 1135 finish_iomap: 1199 1136 if (ops->iomap_end) { 1200 - if (error || (vmf_ret & VM_FAULT_ERROR)) { 1201 - /* keep previous error */ 1202 - ops->iomap_end(inode, pos, PAGE_SIZE, 0, flags, 1203 - &iomap); 1204 - } else { 1205 - error = ops->iomap_end(inode, pos, PAGE_SIZE, 1206 - PAGE_SIZE, flags, &iomap); 1207 - } 1137 + int copied = PAGE_SIZE; 1138 + 1139 + if (vmf_ret & VM_FAULT_ERROR) 1140 + copied = 0; 1141 + /* 1142 + * The fault is done by now and there's no way back (other 1143 + * thread may be already happily using PTE we have installed). 1144 + * Just ignore error from ->iomap_end since we cannot do much 1145 + * with it. 1146 + */ 1147 + ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap); 1208 1148 } 1209 - unlock_entry: 1210 - if (vmf_ret != VM_FAULT_LOCKED || error) 1211 - put_locked_mapping_entry(mapping, vmf->pgoff, entry); 1212 - out: 1213 - if (error == -ENOMEM) 1214 - return VM_FAULT_OOM | major; 1215 - /* -EBUSY is fine, somebody else faulted on the same PTE */ 1216 - if (error < 0 && error != -EBUSY) 1217 - return VM_FAULT_SIGBUS | major; 1218 - if (vmf_ret) { 1219 - WARN_ON_ONCE(error); /* -EBUSY from ops->iomap_end? */ 1220 - return vmf_ret; 1221 - } 1222 - return VM_FAULT_NOPAGE | major; 1149 + return vmf_ret; 1223 1150 } 1224 1151 EXPORT_SYMBOL_GPL(dax_iomap_fault); 1225 1152 ··· 1338 1277 goto fallback; 1339 1278 1340 1279 /* 1341 - * grab_mapping_entry() will make sure we get a 2M empty entry, a DAX 1342 - * PMD or a HZP entry. If it can't (because a 4k page is already in 1343 - * the tree, for instance), it will return -EEXIST and we just fall 1344 - * back to 4k entries. 1345 - */ 1346 - entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD); 1347 - if (IS_ERR(entry)) 1348 - goto fallback; 1349 - 1350 - /* 1351 1280 * Note that we don't use iomap_apply here. We aren't doing I/O, only 1352 1281 * setting up a mapping, so really we're using iomap_begin() as a way 1353 1282 * to look up our filesystem block. ··· 1345 1294 pos = (loff_t)pgoff << PAGE_SHIFT; 1346 1295 error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap); 1347 1296 if (error) 1348 - goto unlock_entry; 1297 + goto fallback; 1298 + 1349 1299 if (iomap.offset + iomap.length < pos + PMD_SIZE) 1300 + goto finish_iomap; 1301 + 1302 + /* 1303 + * grab_mapping_entry() will make sure we get a 2M empty entry, a DAX 1304 + * PMD or a HZP entry. If it can't (because a 4k page is already in 1305 + * the tree, for instance), it will return -EEXIST and we just fall 1306 + * back to 4k entries. 1307 + */ 1308 + entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD); 1309 + if (IS_ERR(entry)) 1350 1310 goto finish_iomap; 1351 1311 1352 1312 vmf.pgoff = pgoff; ··· 1372 1310 case IOMAP_UNWRITTEN: 1373 1311 case IOMAP_HOLE: 1374 1312 if (WARN_ON_ONCE(write)) 1375 - goto finish_iomap; 1313 + goto unlock_entry; 1376 1314 result = dax_pmd_load_hole(vma, pmd, &vmf, address, &iomap, 1377 1315 &entry); 1378 1316 break; ··· 1381 1319 break; 1382 1320 } 1383 1321 1384 - finish_iomap: 1385 - if (ops->iomap_end) { 1386 - if (result == VM_FAULT_FALLBACK) { 1387 - ops->iomap_end(inode, pos, PMD_SIZE, 0, iomap_flags, 1388 - &iomap); 1389 - } else { 1390 - error = ops->iomap_end(inode, pos, PMD_SIZE, PMD_SIZE, 1391 - iomap_flags, &iomap); 1392 - if (error) 1393 - result = VM_FAULT_FALLBACK; 1394 - } 1395 - } 1396 1322 unlock_entry: 1397 1323 put_locked_mapping_entry(mapping, pgoff, entry); 1324 + finish_iomap: 1325 + if (ops->iomap_end) { 1326 + int copied = PMD_SIZE; 1327 + 1328 + if (result == VM_FAULT_FALLBACK) 1329 + copied = 0; 1330 + /* 1331 + * The fault is done by now and there's no way back (other 1332 + * thread may be already happily using PMD we have installed). 1333 + * Just ignore error from ->iomap_end since we cannot do much 1334 + * with it. 1335 + */ 1336 + ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags, 1337 + &iomap); 1338 + } 1398 1339 fallback: 1399 1340 if (result == VM_FAULT_FALLBACK) { 1400 1341 split_huge_pmd(vma, pmd, address);

+1 -2

fs/ext2/inode.c

··· 751 751 mutex_unlock(&ei->truncate_mutex); 752 752 goto cleanup; 753 753 } 754 - } else { 755 - *new = true; 756 754 } 755 + *new = true; 757 756 758 757 ext2_splice_branch(inode, iblock, partial, indirect_blks, count); 759 758 mutex_unlock(&ei->truncate_mutex);

+10 -38

fs/ext4/file.c

··· 258 258 static int ext4_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 259 259 { 260 260 int result; 261 - handle_t *handle = NULL; 262 261 struct inode *inode = file_inode(vma->vm_file); 263 262 struct super_block *sb = inode->i_sb; 264 263 bool write = vmf->flags & FAULT_FLAG_WRITE; ··· 265 266 if (write) { 266 267 sb_start_pagefault(sb); 267 268 file_update_time(vma->vm_file); 268 - down_read(&EXT4_I(inode)->i_mmap_sem); 269 - handle = ext4_journal_start_sb(sb, EXT4_HT_WRITE_PAGE, 270 - EXT4_DATA_TRANS_BLOCKS(sb)); 271 - } else 272 - down_read(&EXT4_I(inode)->i_mmap_sem); 273 - 274 - if (IS_ERR(handle)) 275 - result = VM_FAULT_SIGBUS; 276 - else 277 - result = dax_iomap_fault(vma, vmf, &ext4_iomap_ops); 278 - 279 - if (write) { 280 - if (!IS_ERR(handle)) 281 - ext4_journal_stop(handle); 282 - up_read(&EXT4_I(inode)->i_mmap_sem); 269 + } 270 + down_read(&EXT4_I(inode)->i_mmap_sem); 271 + result = dax_iomap_fault(vma, vmf, &ext4_iomap_ops); 272 + up_read(&EXT4_I(inode)->i_mmap_sem); 273 + if (write) 283 274 sb_end_pagefault(sb); 284 - } else 285 - up_read(&EXT4_I(inode)->i_mmap_sem); 286 275 287 276 return result; 288 277 } ··· 279 292 pmd_t *pmd, unsigned int flags) 280 293 { 281 294 int result; 282 - handle_t *handle = NULL; 283 295 struct inode *inode = file_inode(vma->vm_file); 284 296 struct super_block *sb = inode->i_sb; 285 297 bool write = flags & FAULT_FLAG_WRITE; ··· 286 300 if (write) { 287 301 sb_start_pagefault(sb); 288 302 file_update_time(vma->vm_file); 289 - down_read(&EXT4_I(inode)->i_mmap_sem); 290 - handle = ext4_journal_start_sb(sb, EXT4_HT_WRITE_PAGE, 291 - ext4_chunk_trans_blocks(inode, 292 - PMD_SIZE / PAGE_SIZE)); 293 - } else 294 - down_read(&EXT4_I(inode)->i_mmap_sem); 295 - 296 - if (IS_ERR(handle)) 297 - result = VM_FAULT_SIGBUS; 298 - else { 299 - result = dax_iomap_pmd_fault(vma, addr, pmd, flags, 300 - &ext4_iomap_ops); 301 303 } 302 - 303 - if (write) { 304 - if (!IS_ERR(handle)) 305 - ext4_journal_stop(handle); 306 - up_read(&EXT4_I(inode)->i_mmap_sem); 304 + down_read(&EXT4_I(inode)->i_mmap_sem); 305 + result = dax_iomap_pmd_fault(vma, addr, pmd, flags, 306 + &ext4_iomap_ops); 307 + up_read(&EXT4_I(inode)->i_mmap_sem); 308 + if (write) 307 309 sb_end_pagefault(sb); 308 - } else 309 - up_read(&EXT4_I(inode)->i_mmap_sem); 310 310 311 311 return result; 312 312 }

+3

include/linux/dax.h

··· 41 41 int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf, 42 42 struct iomap_ops *ops); 43 43 int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); 44 + int dax_invalidate_mapping_entry(struct address_space *mapping, pgoff_t index); 45 + int dax_invalidate_mapping_entry_sync(struct address_space *mapping, 46 + pgoff_t index); 44 47 void dax_wake_mapping_entry_waiter(struct address_space *mapping, 45 48 pgoff_t index, void *entry, bool wake_all); 46 49

+61 -14

mm/truncate.c

··· 24 24 #include <linux/rmap.h> 25 25 #include "internal.h" 26 26 27 - static void clear_exceptional_entry(struct address_space *mapping, 28 - pgoff_t index, void *entry) 27 + static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, 28 + void *entry) 29 29 { 30 30 struct radix_tree_node *node; 31 31 void **slot; 32 32 33 - /* Handled by shmem itself */ 34 - if (shmem_mapping(mapping)) 35 - return; 36 - 37 - if (dax_mapping(mapping)) { 38 - dax_delete_mapping_entry(mapping, index); 39 - return; 40 - } 41 33 spin_lock_irq(&mapping->tree_lock); 42 34 /* 43 35 * Regular page slots are stabilized by the page lock even ··· 45 53 mapping->nrexceptional--; 46 54 unlock: 47 55 spin_unlock_irq(&mapping->tree_lock); 56 + } 57 + 58 + /* 59 + * Unconditionally remove exceptional entry. Usually called from truncate path. 60 + */ 61 + static void truncate_exceptional_entry(struct address_space *mapping, 62 + pgoff_t index, void *entry) 63 + { 64 + /* Handled by shmem itself */ 65 + if (shmem_mapping(mapping)) 66 + return; 67 + 68 + if (dax_mapping(mapping)) { 69 + dax_delete_mapping_entry(mapping, index); 70 + return; 71 + } 72 + clear_shadow_entry(mapping, index, entry); 73 + } 74 + 75 + /* 76 + * Invalidate exceptional entry if easily possible. This handles exceptional 77 + * entries for invalidate_inode_pages() so for DAX it evicts only unlocked and 78 + * clean entries. 79 + */ 80 + static int invalidate_exceptional_entry(struct address_space *mapping, 81 + pgoff_t index, void *entry) 82 + { 83 + /* Handled by shmem itself */ 84 + if (shmem_mapping(mapping)) 85 + return 1; 86 + if (dax_mapping(mapping)) 87 + return dax_invalidate_mapping_entry(mapping, index); 88 + clear_shadow_entry(mapping, index, entry); 89 + return 1; 90 + } 91 + 92 + /* 93 + * Invalidate exceptional entry if clean. This handles exceptional entries for 94 + * invalidate_inode_pages2() so for DAX it evicts only clean entries. 95 + */ 96 + static int invalidate_exceptional_entry2(struct address_space *mapping, 97 + pgoff_t index, void *entry) 98 + { 99 + /* Handled by shmem itself */ 100 + if (shmem_mapping(mapping)) 101 + return 1; 102 + if (dax_mapping(mapping)) 103 + return dax_invalidate_mapping_entry_sync(mapping, index); 104 + clear_shadow_entry(mapping, index, entry); 105 + return 1; 48 106 } 49 107 50 108 /** ··· 304 262 break; 305 263 306 264 if (radix_tree_exceptional_entry(page)) { 307 - clear_exceptional_entry(mapping, index, page); 265 + truncate_exceptional_entry(mapping, index, 266 + page); 308 267 continue; 309 268 } 310 269 ··· 394 351 } 395 352 396 353 if (radix_tree_exceptional_entry(page)) { 397 - clear_exceptional_entry(mapping, index, page); 354 + truncate_exceptional_entry(mapping, index, 355 + page); 398 356 continue; 399 357 } 400 358 ··· 514 470 break; 515 471 516 472 if (radix_tree_exceptional_entry(page)) { 517 - clear_exceptional_entry(mapping, index, page); 473 + invalidate_exceptional_entry(mapping, index, 474 + page); 518 475 continue; 519 476 } 520 477 ··· 637 592 break; 638 593 639 594 if (radix_tree_exceptional_entry(page)) { 640 - clear_exceptional_entry(mapping, index, page); 595 + if (!invalidate_exceptional_entry2(mapping, 596 + index, page)) 597 + ret = -EBUSY; 641 598 continue; 642 599 } 643 600

Configure Feed

Configure Feed