mm-khugepaged-refine-scan-progress-number-fix

+35 -43

1 changed file

expand all

khugepaged.c

+35 -43

mm/khugepaged.c

··· 69 69 static DEFINE_MUTEX(khugepaged_mutex); 70 70 71 71 /* 72 - * default scan 8*HPAGE_PMD_NR ptes, pmd_mapped, no_pte_table or vmas 73 - * every 10 second. 72 + * default scan 8*HPAGE_PMD_NR ptes, pte_mapped_hugepage, pmd_mapped, 73 + * no_pte_table or vmas every 10 second. 74 74 */ 75 75 static unsigned int khugepaged_pages_to_scan __read_mostly; 76 76 static unsigned int khugepaged_pages_collapsed; ··· 102 102 103 103 /* Num pages scanned per node */ 104 104 u32 node_load[MAX_NUMNODES]; 105 + 106 + /* Num pages scanned (see khugepaged_pages_to_scan) */ 107 + unsigned int progress; 105 108 106 109 /* nodemask for allocation fallback */ 107 110 nodemask_t alloc_nmask; ··· 1238 1235 1239 1236 static enum scan_result hpage_collapse_scan_pmd(struct mm_struct *mm, 1240 1237 struct vm_area_struct *vma, unsigned long start_addr, 1241 - bool *mmap_locked, unsigned int *cur_progress, 1242 - struct collapse_control *cc) 1238 + bool *mmap_locked, struct collapse_control *cc) 1243 1239 { 1244 1240 pmd_t *pmd; 1245 1241 pte_t *pte, *_pte; ··· 1254 1252 1255 1253 result = find_pmd_or_thp_or_none(mm, start_addr, &pmd); 1256 1254 if (result != SCAN_SUCCEED) { 1257 - if (cur_progress) 1258 - *cur_progress = 1; 1255 + cc->progress++; 1259 1256 goto out; 1260 1257 } 1261 1258 ··· 1262 1261 nodes_clear(cc->alloc_nmask); 1263 1262 pte = pte_offset_map_lock(mm, pmd, start_addr, &ptl); 1264 1263 if (!pte) { 1265 - if (cur_progress) 1266 - *cur_progress = 1; 1264 + cc->progress++; 1267 1265 result = SCAN_NO_PTE_TABLE; 1268 1266 goto out; 1269 1267 } 1270 1268 1271 1269 for (addr = start_addr, _pte = pte; _pte < pte + HPAGE_PMD_NR; 1272 1270 _pte++, addr += PAGE_SIZE) { 1273 - if (cur_progress) 1274 - *cur_progress += 1; 1271 + cc->progress++; 1275 1272 1276 1273 pte_t pteval = ptep_get(_pte); 1277 1274 if (pte_none_or_zero(pteval)) { ··· 2292 2293 2293 2294 static enum scan_result hpage_collapse_scan_file(struct mm_struct *mm, 2294 2295 unsigned long addr, struct file *file, pgoff_t start, 2295 - unsigned int *cur_progress, struct collapse_control *cc) 2296 + struct collapse_control *cc) 2296 2297 { 2297 2298 struct folio *folio = NULL; 2298 2299 struct address_space *mapping = file->f_mapping; ··· 2382 2383 } 2383 2384 } 2384 2385 rcu_read_unlock(); 2385 - if (cur_progress) { 2386 - if (result == SCAN_PTE_MAPPED_HUGEPAGE) 2387 - *cur_progress = 1; 2388 - else 2389 - *cur_progress = HPAGE_PMD_NR; 2390 - } 2386 + if (result == SCAN_PTE_MAPPED_HUGEPAGE) 2387 + cc->progress++; 2388 + else 2389 + cc->progress += HPAGE_PMD_NR; 2391 2390 2392 2391 if (result == SCAN_SUCCEED) { 2393 2392 if (cc->is_khugepaged && ··· 2401 2404 return result; 2402 2405 } 2403 2406 2404 - static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result *result, 2405 - struct collapse_control *cc) 2407 + static void khugepaged_scan_mm_slot(unsigned int progress_max, 2408 + enum scan_result *result, struct collapse_control *cc) 2406 2409 __releases(&khugepaged_mm_lock) 2407 2410 __acquires(&khugepaged_mm_lock) 2408 2411 { ··· 2410 2413 struct mm_slot *slot; 2411 2414 struct mm_struct *mm; 2412 2415 struct vm_area_struct *vma; 2413 - int progress = 0; 2416 + unsigned int progress_prev = cc->progress; 2414 2417 2415 - VM_BUG_ON(!pages); 2416 2418 lockdep_assert_held(&khugepaged_mm_lock); 2417 2419 *result = SCAN_FAIL; 2418 2420 ··· 2434 2438 if (unlikely(!mmap_read_trylock(mm))) 2435 2439 goto breakouterloop_mmap_lock; 2436 2440 2437 - progress++; 2441 + cc->progress++; 2438 2442 if (unlikely(hpage_collapse_test_exit_or_disable(mm))) 2439 2443 goto breakouterloop; 2440 2444 ··· 2444 2448 2445 2449 cond_resched(); 2446 2450 if (unlikely(hpage_collapse_test_exit_or_disable(mm))) { 2447 - progress++; 2451 + cc->progress++; 2448 2452 break; 2449 2453 } 2450 2454 if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_KHUGEPAGED, PMD_ORDER)) { 2451 - progress++; 2455 + cc->progress++; 2452 2456 continue; 2453 2457 } 2454 2458 hstart = round_up(vma->vm_start, HPAGE_PMD_SIZE); 2455 2459 hend = round_down(vma->vm_end, HPAGE_PMD_SIZE); 2456 2460 if (khugepaged_scan.address > hend) { 2457 - progress++; 2461 + cc->progress++; 2458 2462 continue; 2459 2463 } 2460 2464 if (khugepaged_scan.address < hstart) ··· 2463 2467 2464 2468 while (khugepaged_scan.address < hend) { 2465 2469 bool mmap_locked = true; 2466 - unsigned int cur_progress = 0; 2467 2470 2468 2471 cond_resched(); 2469 2472 if (unlikely(hpage_collapse_test_exit_or_disable(mm))) ··· 2479 2484 mmap_read_unlock(mm); 2480 2485 mmap_locked = false; 2481 2486 *result = hpage_collapse_scan_file(mm, 2482 - khugepaged_scan.address, file, pgoff, 2483 - &cur_progress, cc); 2487 + khugepaged_scan.address, file, pgoff, cc); 2484 2488 fput(file); 2485 2489 if (*result == SCAN_PTE_MAPPED_HUGEPAGE) { 2486 2490 mmap_read_lock(mm); ··· 2493 2499 } 2494 2500 } else { 2495 2501 *result = hpage_collapse_scan_pmd(mm, vma, 2496 - khugepaged_scan.address, &mmap_locked, 2497 - &cur_progress, cc); 2502 + khugepaged_scan.address, &mmap_locked, cc); 2498 2503 } 2499 2504 2500 2505 if (*result == SCAN_SUCCEED) ··· 2501 2508 2502 2509 /* move to next address */ 2503 2510 khugepaged_scan.address += HPAGE_PMD_SIZE; 2504 - progress += cur_progress; 2505 2511 if (!mmap_locked) 2506 2512 /* 2507 2513 * We released mmap_lock so break loop. Note ··· 2510 2518 * correct result back to caller. 2511 2519 */ 2512 2520 goto breakouterloop_mmap_lock; 2513 - if (progress >= pages) 2521 + if (cc->progress >= progress_max) 2514 2522 goto breakouterloop; 2515 2523 } 2516 2524 } ··· 2541 2549 collect_mm_slot(slot); 2542 2550 } 2543 2551 2544 - trace_mm_khugepaged_scan(mm, progress, khugepaged_scan.mm_slot == NULL); 2545 - 2546 - return progress; 2552 + trace_mm_khugepaged_scan(mm, cc->progress - progress_prev, 2553 + khugepaged_scan.mm_slot == NULL); 2547 2554 } 2548 2555 2549 2556 static int khugepaged_has_work(void) ··· 2558 2567 2559 2568 static void khugepaged_do_scan(struct collapse_control *cc) 2560 2569 { 2561 - unsigned int progress = 0, pass_through_head = 0; 2562 - unsigned int pages = READ_ONCE(khugepaged_pages_to_scan); 2570 + const unsigned int progress_max = READ_ONCE(khugepaged_pages_to_scan); 2571 + unsigned int pass_through_head = 0; 2563 2572 bool wait = true; 2564 2573 enum scan_result result = SCAN_SUCCEED; 2565 2574 2566 2575 lru_add_drain_all(); 2567 2576 2577 + cc->progress = 0; 2568 2578 while (true) { 2569 2579 cond_resched(); 2570 2580 ··· 2577 2585 pass_through_head++; 2578 2586 if (khugepaged_has_work() && 2579 2587 pass_through_head < 2) 2580 - progress += khugepaged_scan_mm_slot(pages - progress, 2581 - &result, cc); 2588 + khugepaged_scan_mm_slot(progress_max, &result, cc); 2582 2589 else 2583 - progress = pages; 2590 + cc->progress = progress_max; 2584 2591 spin_unlock(&khugepaged_mm_lock); 2585 2592 2586 - if (progress >= pages) 2593 + if (cc->progress >= progress_max) 2587 2594 break; 2588 2595 2589 2596 if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) { ··· 2788 2797 if (!cc) 2789 2798 return -ENOMEM; 2790 2799 cc->is_khugepaged = false; 2800 + cc->progress = 0; 2791 2801 2792 2802 mmgrab(mm); 2793 2803 lru_add_drain_all(); ··· 2823 2831 mmap_locked = false; 2824 2832 *lock_dropped = true; 2825 2833 result = hpage_collapse_scan_file(mm, addr, file, pgoff, 2826 - NULL, cc); 2834 + cc); 2827 2835 2828 2836 if (result == SCAN_PAGE_DIRTY_OR_WRITEBACK && !triggered_wb && 2829 2837 mapping_can_writeback(file->f_mapping)) { ··· 2838 2846 fput(file); 2839 2847 } else { 2840 2848 result = hpage_collapse_scan_pmd(mm, vma, addr, 2841 - &mmap_locked, NULL, cc); 2849 + &mmap_locked, cc); 2842 2850 } 2843 2851 if (!mmap_locked) 2844 2852 *lock_dropped = true;

Configure Feed

Configure Feed