mm/huge_memory: separate out the folio part of zap_huge_pmd()

Place the part of the logic that manipulates counters and possibly updates
the accessed bit of the folio into its own function to make zap_huge_pmd()
more readable.

Also rename flush_needed to is_present as we only require a flush for
present entries.

Additionally add comments as to why we're doing what we're doing with
respect to softleaf entries.

This also lays the ground for further refactoring.

Link: https://lkml.kernel.org/r/6c4db67952f5529da4db102a6149b9050b5dda4e.1774029655.git.ljs@kernel.org
Signed-off-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Cc: Barry Song <baohua@kernel.org>
Cc: David Hildenbrand <david@kernel.org>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nico Pache <npache@redhat.com>
Cc: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Lorenzo Stoakes (Oracle) and committed by

Andrew Morton 1 month ago f87854c9 1c6b7ff6

+35 -26

1 changed file

expand all

huge_memory.c

+35 -26

mm/huge_memory.c

··· 2402 2402 mm_dec_nr_ptes(mm); 2403 2403 } 2404 2404 2405 + static void zap_huge_pmd_folio(struct mm_struct *mm, struct vm_area_struct *vma, 2406 + pmd_t pmdval, struct folio *folio, bool is_present, 2407 + bool *has_deposit) 2408 + { 2409 + const bool is_device_private = folio_is_device_private(folio); 2410 + 2411 + /* Present and device private folios are rmappable. */ 2412 + if (is_present || is_device_private) 2413 + folio_remove_rmap_pmd(folio, &folio->page, vma); 2414 + 2415 + if (folio_test_anon(folio)) { 2416 + *has_deposit = true; 2417 + add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); 2418 + } else { 2419 + add_mm_counter(mm, mm_counter_file(folio), 2420 + -HPAGE_PMD_NR); 2421 + 2422 + /* 2423 + * Use flush_needed to indicate whether the PMD entry 2424 + * is present, instead of checking pmd_present() again. 2425 + */ 2426 + if (is_present && pmd_young(pmdval) && 2427 + likely(vma_has_recency(vma))) 2428 + folio_mark_accessed(folio); 2429 + } 2430 + 2431 + /* Device private folios are pinned. */ 2432 + if (is_device_private) 2433 + folio_put(folio); 2434 + } 2435 + 2405 2436 /** 2406 2437 * zap_huge_pmd - Zap a huge THP which is of PMD size. 2407 2438 * @tlb: The MMU gather TLB state associated with the operation. ··· 2448 2417 bool has_deposit = arch_needs_pgtable_deposit(); 2449 2418 struct mm_struct *mm = tlb->mm; 2450 2419 struct folio *folio = NULL; 2451 - bool flush_needed = false; 2420 + bool is_present = false; 2452 2421 spinlock_t *ptl; 2453 2422 pmd_t orig_pmd; 2454 2423 ··· 2477 2446 2478 2447 if (pmd_present(orig_pmd)) { 2479 2448 folio = pmd_folio(orig_pmd); 2480 - 2481 - flush_needed = true; 2482 - folio_remove_rmap_pmd(folio, &folio->page, vma); 2449 + is_present = true; 2483 2450 } else if (pmd_is_valid_softleaf(orig_pmd)) { 2484 2451 const softleaf_t entry = softleaf_from_pmd(orig_pmd); 2485 2452 2486 2453 folio = softleaf_to_folio(entry); 2487 - 2488 2454 if (!thp_migration_supported()) 2489 2455 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 2490 2456 } else { ··· 2489 2461 goto out; 2490 2462 } 2491 2463 2492 - if (folio_test_anon(folio)) { 2493 - has_deposit = true; 2494 - add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); 2495 - } else { 2496 - add_mm_counter(mm, mm_counter_file(folio), 2497 - -HPAGE_PMD_NR); 2498 - 2499 - /* 2500 - * Use flush_needed to indicate whether the PMD entry 2501 - * is present, instead of checking pmd_present() again. 2502 - */ 2503 - if (flush_needed && pmd_young(orig_pmd) && 2504 - likely(vma_has_recency(vma))) 2505 - folio_mark_accessed(folio); 2506 - } 2507 - 2508 - if (folio_is_device_private(folio)) { 2509 - folio_remove_rmap_pmd(folio, &folio->page, vma); 2510 - folio_put(folio); 2511 - } 2464 + zap_huge_pmd_folio(mm, vma, orig_pmd, folio, is_present, &has_deposit); 2512 2465 2513 2466 out: 2514 2467 if (has_deposit) 2515 2468 zap_deposited_table(mm, pmd); 2516 2469 2517 2470 spin_unlock(ptl); 2518 - if (flush_needed) 2471 + if (is_present) 2519 2472 tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE); 2520 2473 return true; 2521 2474 }

Configure Feed

Configure Feed