+1 -1

fs/binfmt_elf.c

reviewed

··· 1117 1117 * without MAP_FIXED nor MAP_FIXED_NOREPLACE). 1118 1118 */ 1119 1119 alignment = maximum_alignment(elf_phdata, elf_ex->e_phnum); 1120 1120 - if (alignment > ELF_MIN_ALIGN) { 1120 1120 + if (interpreter || alignment > ELF_MIN_ALIGN) { 1121 1121 load_bias = ELF_ET_DYN_BASE; 1122 1122 if (current->flags & PF_RANDOMIZE) 1123 1123 load_bias += arch_mmap_rnd();

+31 -9

fs/proc/task_mmu.c

reviewed

··· 440 440 } 441 441 442 442 static void smaps_account(struct mem_size_stats *mss, struct page *page, 443 443 - bool compound, bool young, bool dirty, bool locked) 443 443 + bool compound, bool young, bool dirty, bool locked, 444 444 + bool migration) 444 445 { 445 446 int i, nr = compound ? compound_nr(page) : 1; 446 447 unsigned long size = nr * PAGE_SIZE; ··· 468 467 * page_count(page) == 1 guarantees the page is mapped exactly once. 469 468 * If any subpage of the compound page mapped with PTE it would elevate 470 469 * page_count(). 470 470 + * 471 471 + * The page_mapcount() is called to get a snapshot of the mapcount. 472 472 + * Without holding the page lock this snapshot can be slightly wrong as 473 473 + * we cannot always read the mapcount atomically. It is not safe to 474 474 + * call page_mapcount() even with PTL held if the page is not mapped, 475 475 + * especially for migration entries. Treat regular migration entries 476 476 + * as mapcount == 1. 471 477 */ 472 472 - if (page_count(page) == 1) { 478 478 + if ((page_count(page) == 1) || migration) { 473 479 smaps_page_accumulate(mss, page, size, size << PSS_SHIFT, dirty, 474 480 locked, true); 475 481 return; ··· 525 517 struct vm_area_struct *vma = walk->vma; 526 518 bool locked = !!(vma->vm_flags & VM_LOCKED); 527 519 struct page *page = NULL; 520 520 + bool migration = false; 528 521 529 522 if (pte_present(*pte)) { 530 523 page = vm_normal_page(vma, addr, *pte); ··· 545 536 } else { 546 537 mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; 547 538 } 548 548 - } else if (is_pfn_swap_entry(swpent)) 539 539 + } else if (is_pfn_swap_entry(swpent)) { 540 540 + if (is_migration_entry(swpent)) 541 541 + migration = true; 549 542 page = pfn_swap_entry_to_page(swpent); 543 543 + } 550 544 } else { 551 545 smaps_pte_hole_lookup(addr, walk); 552 546 return; ··· 558 546 if (!page) 559 547 return; 560 548 561 561 - smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte), locked); 549 549 + smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte), 550 550 + locked, migration); 562 551 } 563 552 564 553 #ifdef CONFIG_TRANSPARENT_HUGEPAGE ··· 570 557 struct vm_area_struct *vma = walk->vma; 571 558 bool locked = !!(vma->vm_flags & VM_LOCKED); 572 559 struct page *page = NULL; 560 560 + bool migration = false; 573 561 574 562 if (pmd_present(*pmd)) { 575 563 /* FOLL_DUMP will return -EFAULT on huge zero page */ ··· 578 564 } else if (unlikely(thp_migration_supported() && is_swap_pmd(*pmd))) { 579 565 swp_entry_t entry = pmd_to_swp_entry(*pmd); 580 566 581 581 - if (is_migration_entry(entry)) 567 567 + if (is_migration_entry(entry)) { 568 568 + migration = true; 582 569 page = pfn_swap_entry_to_page(entry); 570 570 + } 583 571 } 584 572 if (IS_ERR_OR_NULL(page)) 585 573 return; ··· 593 577 /* pass */; 594 578 else 595 579 mss->file_thp += HPAGE_PMD_SIZE; 596 596 - smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd), locked); 580 580 + 581 581 + smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd), 582 582 + locked, migration); 597 583 } 598 584 #else 599 585 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, ··· 1396 1378 { 1397 1379 u64 frame = 0, flags = 0; 1398 1380 struct page *page = NULL; 1381 1381 + bool migration = false; 1399 1382 1400 1383 if (pte_present(pte)) { 1401 1384 if (pm->show_pfn) ··· 1418 1399 frame = swp_type(entry) | 1419 1400 (swp_offset(entry) << MAX_SWAPFILES_SHIFT); 1420 1401 flags |= PM_SWAP; 1402 1402 + migration = is_migration_entry(entry); 1421 1403 if (is_pfn_swap_entry(entry)) 1422 1404 page = pfn_swap_entry_to_page(entry); 1423 1405 } 1424 1406 1425 1407 if (page && !PageAnon(page)) 1426 1408 flags |= PM_FILE; 1427 1427 - if (page && page_mapcount(page) == 1) 1409 1409 + if (page && !migration && page_mapcount(page) == 1) 1428 1410 flags |= PM_MMAP_EXCLUSIVE; 1429 1411 if (vma->vm_flags & VM_SOFTDIRTY) 1430 1412 flags |= PM_SOFT_DIRTY; ··· 1441 1421 spinlock_t *ptl; 1442 1422 pte_t *pte, *orig_pte; 1443 1423 int err = 0; 1444 1444 - 1445 1424 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1425 1425 + bool migration = false; 1426 1426 + 1446 1427 ptl = pmd_trans_huge_lock(pmdp, vma); 1447 1428 if (ptl) { 1448 1429 u64 flags = 0, frame = 0; ··· 1482 1461 if (pmd_swp_uffd_wp(pmd)) 1483 1462 flags |= PM_UFFD_WP; 1484 1463 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 1464 1464 + migration = is_migration_entry(entry); 1485 1465 page = pfn_swap_entry_to_page(entry); 1486 1466 } 1487 1467 #endif 1488 1468 1489 1489 - if (page && page_mapcount(page) == 1) 1469 1469 + if (page && !migration && page_mapcount(page) == 1) 1490 1470 flags |= PM_MMAP_EXCLUSIVE; 1491 1471 1492 1472 for (; addr != end; addr += PAGE_SIZE) {

+2

include/linux/kfence.h

reviewed

··· 17 17 #include <linux/atomic.h> 18 18 #include <linux/static_key.h> 19 19 20 20 + extern unsigned long kfence_sample_interval; 21 21 + 20 22 /* 21 23 * We allocate an even number of pages, as it simplifies calculations to map 22 24 * address to metadata indices; effectively, the very first page serves as an

+3 -2

include/linux/memcontrol.h

reviewed

··· 219 219 struct mem_cgroup *memcg; 220 220 atomic_t nr_charged_bytes; 221 221 union { 222 222 - struct list_head list; 222 222 + struct list_head list; /* protected by objcg_lock */ 223 223 struct rcu_head rcu; 224 224 }; 225 225 }; ··· 315 315 #ifdef CONFIG_MEMCG_KMEM 316 316 int kmemcg_id; 317 317 struct obj_cgroup __rcu *objcg; 318 318 - struct list_head objcg_list; /* list of inherited objcgs */ 318 318 + /* list of inherited objcgs, protected by objcg_lock */ 319 319 + struct list_head objcg_list; 319 320 #endif 320 321 321 322 MEMCG_PADDING(_pad2_);

+2 -1

mm/kfence/core.c

reviewed

··· 47 47 48 48 static bool kfence_enabled __read_mostly; 49 49 50 50 - static unsigned long kfence_sample_interval __read_mostly = CONFIG_KFENCE_SAMPLE_INTERVAL; 50 50 + unsigned long kfence_sample_interval __read_mostly = CONFIG_KFENCE_SAMPLE_INTERVAL; 51 51 + EXPORT_SYMBOL_GPL(kfence_sample_interval); /* Export for test modules. */ 51 52 52 53 #ifdef MODULE_PARAM_PREFIX 53 54 #undef MODULE_PARAM_PREFIX

+4 -4

mm/kfence/kfence_test.c

reviewed

··· 268 268 * 100x the sample interval should be more than enough to ensure we get 269 269 * a KFENCE allocation eventually. 270 270 */ 271 271 - timeout = jiffies + msecs_to_jiffies(100 * CONFIG_KFENCE_SAMPLE_INTERVAL); 271 271 + timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval); 272 272 /* 273 273 * Especially for non-preemption kernels, ensure the allocation-gate 274 274 * timer can catch up: after @resched_after, every failed allocation 275 275 * attempt yields, to ensure the allocation-gate timer is scheduled. 276 276 */ 277 277 - resched_after = jiffies + msecs_to_jiffies(CONFIG_KFENCE_SAMPLE_INTERVAL); 277 277 + resched_after = jiffies + msecs_to_jiffies(kfence_sample_interval); 278 278 do { 279 279 if (test_cache) 280 280 alloc = kmem_cache_alloc(test_cache, gfp); ··· 608 608 int i; 609 609 610 610 /* Skip if we think it'd take too long. */ 611 611 - KFENCE_TEST_REQUIRES(test, CONFIG_KFENCE_SAMPLE_INTERVAL <= 100); 611 611 + KFENCE_TEST_REQUIRES(test, kfence_sample_interval <= 100); 612 612 613 613 setup_test_cache(test, size, 0, NULL); 614 614 buf1 = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY); ··· 739 739 * 100x the sample interval should be more than enough to ensure we get 740 740 * a KFENCE allocation eventually. 741 741 */ 742 742 - timeout = jiffies + msecs_to_jiffies(100 * CONFIG_KFENCE_SAMPLE_INTERVAL); 742 742 + timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval); 743 743 do { 744 744 void *objects[100]; 745 745 int i, num = kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC, ARRAY_SIZE(objects),

+5 -5

mm/memcontrol.c

reviewed

··· 254 254 } 255 255 256 256 #ifdef CONFIG_MEMCG_KMEM 257 257 - extern spinlock_t css_set_lock; 257 257 + static DEFINE_SPINLOCK(objcg_lock); 258 258 259 259 bool mem_cgroup_kmem_disabled(void) 260 260 { ··· 298 298 if (nr_pages) 299 299 obj_cgroup_uncharge_pages(objcg, nr_pages); 300 300 301 301 - spin_lock_irqsave(&css_set_lock, flags); 301 301 + spin_lock_irqsave(&objcg_lock, flags); 302 302 list_del(&objcg->list); 303 303 - spin_unlock_irqrestore(&css_set_lock, flags); 303 303 + spin_unlock_irqrestore(&objcg_lock, flags); 304 304 305 305 percpu_ref_exit(ref); 306 306 kfree_rcu(objcg, rcu); ··· 332 332 333 333 objcg = rcu_replace_pointer(memcg->objcg, NULL, true); 334 334 335 335 - spin_lock_irq(&css_set_lock); 335 335 + spin_lock_irq(&objcg_lock); 336 336 337 337 /* 1) Ready to reparent active objcg. */ 338 338 list_add(&objcg->list, &memcg->objcg_list); ··· 342 342 /* 3) Move already reparented objcgs to the parent's list */ 343 343 list_splice(&memcg->objcg_list, &parent->objcg_list); 344 344 345 345 - spin_unlock_irq(&css_set_lock); 345 345 + spin_unlock_irq(&objcg_lock); 346 346 347 347 percpu_ref_kill(&objcg->refcnt); 348 348 }

+3 -1

mm/vmscan.c

reviewed

··· 1066 1066 * forward progress (e.g. journalling workqueues or kthreads). 1067 1067 */ 1068 1068 if (!current_is_kswapd() && 1069 1069 - current->flags & (PF_IO_WORKER|PF_KTHREAD)) 1069 1069 + current->flags & (PF_IO_WORKER|PF_KTHREAD)) { 1070 1070 + cond_resched(); 1070 1071 return; 1072 1072 + } 1071 1073 1072 1074 /* 1073 1075 * These figures are pulled out of thin air.