+1

.mailmap

reviewed

··· 133 133 Jan Glauber <jan.glauber@gmail.com> <jang@de.ibm.com> 134 134 Jan Glauber <jan.glauber@gmail.com> <jang@linux.vnet.ibm.com> 135 135 Jan Glauber <jan.glauber@gmail.com> <jglauber@cavium.com> 136 136 + Jarkko Sakkinen <jarkko@kernel.org> <jarkko.sakkinen@linux.intel.com> 136 137 Jason Gunthorpe <jgg@ziepe.ca> <jgg@mellanox.com> 137 138 Jason Gunthorpe <jgg@ziepe.ca> <jgg@nvidia.com> 138 139 Jason Gunthorpe <jgg@ziepe.ca> <jgunthorpe@obsidianresearch.com>

+1

Documentation/admin-guide/kernel-parameters.txt

reviewed

··· 1343 1343 current integrity status. 1344 1344 1345 1345 failslab= 1346 1346 + fail_usercopy= 1346 1347 fail_page_alloc= 1347 1348 fail_make_request=[KNL] 1348 1349 General fault injection mechanism.

+8 -6

Documentation/core-api/xarray.rst

reviewed

··· 475 475 Each entry will only be returned once, no matter how many indices it 476 476 occupies. 477 477 478 478 - Using xas_next() or xas_prev() with a multi-index xa_state 479 479 - is not supported. Using either of these functions on a multi-index entry 480 480 - will reveal sibling entries; these should be skipped over by the caller. 478 478 + Using xas_next() or xas_prev() with a multi-index xa_state is not 479 479 + supported. Using either of these functions on a multi-index entry will 480 480 + reveal sibling entries; these should be skipped over by the caller. 481 481 482 482 - Storing ``NULL`` into any index of a multi-index entry will set the entry 483 483 - at every index to ``NULL`` and dissolve the tie. Splitting a multi-index 484 484 - entry into entries occupying smaller ranges is not yet supported. 482 482 + Storing ``NULL`` into any index of a multi-index entry will set the 483 483 + entry at every index to ``NULL`` and dissolve the tie. A multi-index 484 484 + entry can be split into entries occupying smaller ranges by calling 485 485 + xas_split_alloc() without the xa_lock held, followed by taking the lock 486 486 + and calling xas_split(). 485 487 486 488 Functions and structures 487 489 ========================

+6 -1

Documentation/fault-injection/fault-injection.rst

reviewed

··· 16 16 17 17 injects page allocation failures. (alloc_pages(), get_free_pages(), ...) 18 18 19 19 + - fail_usercopy 20 20 + 21 21 + injects failures in user memory access functions. (copy_from_user(), get_user(), ...) 22 22 + 19 23 - fail_futex 20 24 21 25 injects futex deadlock and uaddr fault errors. ··· 181 177 182 178 failslab= 183 179 fail_page_alloc= 180 180 + fail_usercopy= 184 181 fail_make_request= 185 182 fail_futex= 186 183 mmc_core.fail_request=<interval>,<probability>,<space>,<times> ··· 227 222 228 223 - debugfs entries 229 224 230 230 - failslab, fail_page_alloc, and fail_make_request use this way. 225 225 + failslab, fail_page_alloc, fail_usercopy, and fail_make_request use this way. 231 226 Helper functions: 232 227 233 228 fault_create_debugfs_attr(name, parent, attr);

+3 -3

MAINTAINERS

reviewed

··· 9715 9715 9716 9716 KEYS-TRUSTED 9717 9717 M: James Bottomley <jejb@linux.ibm.com> 9718 9718 - M: Jarkko Sakkinen <jarkko.sakkinen@linux.intel.com> 9718 9718 + M: Jarkko Sakkinen <jarkko@kernel.org> 9719 9719 M: Mimi Zohar <zohar@linux.ibm.com> 9720 9720 L: linux-integrity@vger.kernel.org 9721 9721 L: keyrings@vger.kernel.org ··· 9727 9727 9728 9728 KEYS/KEYRINGS 9729 9729 M: David Howells <dhowells@redhat.com> 9730 9730 - M: Jarkko Sakkinen <jarkko.sakkinen@linux.intel.com> 9730 9730 + M: Jarkko Sakkinen <jarkko@kernel.org> 9731 9731 L: keyrings@vger.kernel.org 9732 9732 S: Maintained 9733 9733 F: Documentation/security/keys/core.rst ··· 17717 17717 17718 17718 TPM DEVICE DRIVER 17719 17719 M: Peter Huewe <peterhuewe@gmx.de> 17720 17720 - M: Jarkko Sakkinen <jarkko.sakkinen@linux.intel.com> 17720 17720 + M: Jarkko Sakkinen <jarkko@kernel.org> 17721 17721 R: Jason Gunthorpe <jgg@ziepe.ca> 17722 17722 L: linux-integrity@vger.kernel.org 17723 17723 S: Maintained

+2 -2

arch/ia64/mm/init.c

reviewed

··· 537 537 if (map_start < map_end) 538 538 memmap_init_zone((unsigned long)(map_end - map_start), 539 539 args->nid, args->zone, page_to_pfn(map_start), 540 540 - MEMINIT_EARLY, NULL); 540 540 + MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 541 541 return 0; 542 542 } 543 543 ··· 547 547 { 548 548 if (!vmem_map) { 549 549 memmap_init_zone(size, nid, zone, start_pfn, 550 550 - MEMINIT_EARLY, NULL); 550 550 + MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 551 551 } else { 552 552 struct page *start; 553 553 struct memmap_init_callback_data args;

+23 -6

arch/powerpc/include/asm/book3s/64/pgtable.h

reviewed

··· 615 615 VM_BUG_ON(pfn >> (64 - PAGE_SHIFT)); 616 616 VM_BUG_ON((pfn << PAGE_SHIFT) & ~PTE_RPN_MASK); 617 617 618 618 - return __pte(((pte_basic_t)pfn << PAGE_SHIFT) | pgprot_val(pgprot)); 618 618 + return __pte(((pte_basic_t)pfn << PAGE_SHIFT) | pgprot_val(pgprot) | _PAGE_PTE); 619 619 } 620 620 621 621 static inline unsigned long pte_pfn(pte_t pte) ··· 649 649 static inline pte_t pte_mkexec(pte_t pte) 650 650 { 651 651 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_EXEC)); 652 652 - } 653 653 - 654 654 - static inline pte_t pte_mkpte(pte_t pte) 655 655 - { 656 656 - return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PTE)); 657 652 } 658 653 659 654 static inline pte_t pte_mkwrite(pte_t pte) ··· 814 819 static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, 815 820 pte_t *ptep, pte_t pte, int percpu) 816 821 { 822 822 + 823 823 + VM_WARN_ON(!(pte_raw(pte) & cpu_to_be64(_PAGE_PTE))); 824 824 + /* 825 825 + * Keep the _PAGE_PTE added till we are sure we handle _PAGE_PTE 826 826 + * in all the callers. 827 827 + */ 828 828 + pte = __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PTE)); 829 829 + 817 830 if (radix_enabled()) 818 831 return radix__set_pte_at(mm, addr, ptep, pte, percpu); 819 832 return hash__set_pte_at(mm, addr, ptep, pte, percpu); ··· 869 866 870 867 static inline void pmd_clear(pmd_t *pmdp) 871 868 { 869 869 + if (IS_ENABLED(CONFIG_DEBUG_VM) && !radix_enabled()) { 870 870 + /* 871 871 + * Don't use this if we can possibly have a hash page table 872 872 + * entry mapping this. 873 873 + */ 874 874 + WARN_ON((pmd_val(*pmdp) & (H_PAGE_HASHPTE | _PAGE_PTE)) == (H_PAGE_HASHPTE | _PAGE_PTE)); 875 875 + } 872 876 *pmdp = __pmd(0); 873 877 } 874 878 ··· 924 914 925 915 static inline void pud_clear(pud_t *pudp) 926 916 { 917 917 + if (IS_ENABLED(CONFIG_DEBUG_VM) && !radix_enabled()) { 918 918 + /* 919 919 + * Don't use this if we can possibly have a hash page table 920 920 + * entry mapping this. 921 921 + */ 922 922 + WARN_ON((pud_val(*pudp) & (H_PAGE_HASHPTE | _PAGE_PTE)) == (H_PAGE_HASHPTE | _PAGE_PTE)); 923 923 + } 927 924 *pudp = __pud(0); 928 925 } 929 926

-5

arch/powerpc/include/asm/nohash/pgtable.h

reviewed

··· 140 140 return __pte(pte_val(pte) & ~_PAGE_ACCESSED); 141 141 } 142 142 143 143 - static inline pte_t pte_mkpte(pte_t pte) 144 144 - { 145 145 - return pte; 146 146 - } 147 147 - 148 143 static inline pte_t pte_mkspecial(pte_t pte) 149 144 { 150 145 return __pte(pte_val(pte) | _PAGE_SPECIAL);

-5

arch/powerpc/mm/pgtable.c

reviewed

··· 184 184 */ 185 185 VM_WARN_ON(pte_hw_valid(*ptep) && !pte_protnone(*ptep)); 186 186 187 187 - /* Add the pte bit when trying to set a pte */ 188 188 - pte = pte_mkpte(pte); 189 189 - 190 187 /* Note: mm->context.id might not yet have been assigned as 191 188 * this context might not have been activated yet when this 192 189 * is called. ··· 271 274 * tlb flush for this update. 272 275 */ 273 276 VM_WARN_ON(pte_hw_valid(*ptep) && !pte_protnone(*ptep)); 274 274 - 275 275 - pte = pte_mkpte(pte); 276 277 277 278 pte = set_pte_filter(pte); 278 279

+1 -1

arch/powerpc/platforms/powernv/memtrace.c

reviewed

··· 224 224 ent->mem = 0; 225 225 } 226 226 227 227 - if (add_memory(ent->nid, ent->start, ent->size)) { 227 227 + if (add_memory(ent->nid, ent->start, ent->size, MHP_NONE)) { 228 228 pr_err("Failed to add trace memory to node %d\n", 229 229 ent->nid); 230 230 ret += 1;

+1 -1

arch/powerpc/platforms/pseries/hotplug-memory.c

reviewed

··· 606 606 block_sz = memory_block_size_bytes(); 607 607 608 608 /* Add the memory */ 609 609 - rc = __add_memory(lmb->nid, lmb->base_addr, block_sz); 609 609 + rc = __add_memory(lmb->nid, lmb->base_addr, block_sz, MHP_NONE); 610 610 if (rc) { 611 611 invalidate_lmb_associativity_index(lmb); 612 612 return rc;

+2 -1

drivers/acpi/acpi_memhotplug.c

reviewed

··· 194 194 if (node < 0) 195 195 node = memory_add_physaddr_to_nid(info->start_addr); 196 196 197 197 - result = __add_memory(node, info->start_addr, info->length); 197 197 + result = __add_memory(node, info->start_addr, info->length, 198 198 + MHP_NONE); 198 199 199 200 /* 200 201 * If the memory block has been used by the kernel, add_memory()

+2 -1

drivers/base/memory.c

reviewed

··· 432 432 433 433 nid = memory_add_physaddr_to_nid(phys_addr); 434 434 ret = __add_memory(nid, phys_addr, 435 435 - MIN_MEMORY_BLOCK_SIZE * sections_per_block); 435 435 + MIN_MEMORY_BLOCK_SIZE * sections_per_block, 436 436 + MHP_NONE); 436 437 437 438 if (ret) 438 439 goto out;

+21 -12

drivers/base/node.c

reviewed

··· 772 772 return pfn_to_nid(pfn); 773 773 } 774 774 775 775 - static int do_register_memory_block_under_node(int nid, 776 776 - struct memory_block *mem_blk) 775 775 + static void do_register_memory_block_under_node(int nid, 776 776 + struct memory_block *mem_blk) 777 777 { 778 778 int ret; 779 779 ··· 786 786 ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj, 787 787 &mem_blk->dev.kobj, 788 788 kobject_name(&mem_blk->dev.kobj)); 789 789 - if (ret) 790 790 - return ret; 789 789 + if (ret && ret != -EEXIST) 790 790 + dev_err_ratelimited(&node_devices[nid]->dev, 791 791 + "can't create link to %s in sysfs (%d)\n", 792 792 + kobject_name(&mem_blk->dev.kobj), ret); 791 793 792 792 - return sysfs_create_link_nowarn(&mem_blk->dev.kobj, 794 794 + ret = sysfs_create_link_nowarn(&mem_blk->dev.kobj, 793 795 &node_devices[nid]->dev.kobj, 794 796 kobject_name(&node_devices[nid]->dev.kobj)); 797 797 + if (ret && ret != -EEXIST) 798 798 + dev_err_ratelimited(&mem_blk->dev, 799 799 + "can't create link to %s in sysfs (%d)\n", 800 800 + kobject_name(&node_devices[nid]->dev.kobj), 801 801 + ret); 795 802 } 796 803 797 804 /* register memory section under specified node if it spans that node */ ··· 834 827 if (page_nid != nid) 835 828 continue; 836 829 837 837 - return do_register_memory_block_under_node(nid, mem_blk); 830 830 + do_register_memory_block_under_node(nid, mem_blk); 831 831 + return 0; 838 832 } 839 833 /* mem section does not span the specified node */ 840 834 return 0; ··· 850 842 { 851 843 int nid = *(int *)arg; 852 844 853 853 - return do_register_memory_block_under_node(nid, mem_blk); 845 845 + do_register_memory_block_under_node(nid, mem_blk); 846 846 + return 0; 854 847 } 855 848 856 849 /* ··· 869 860 kobject_name(&node_devices[mem_blk->nid]->dev.kobj)); 870 861 } 871 862 872 872 - int link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn, 873 873 - enum meminit_context context) 863 863 + void link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn, 864 864 + enum meminit_context context) 874 865 { 875 866 walk_memory_blocks_func_t func; 876 867 ··· 879 870 else 880 871 func = register_mem_block_under_node_early; 881 872 882 882 - return walk_memory_blocks(PFN_PHYS(start_pfn), 883 883 - PFN_PHYS(end_pfn - start_pfn), (void *)&nid, 884 884 - func); 873 873 + walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn), 874 874 + (void *)&nid, func); 875 875 + return; 885 876 } 886 877 887 878 #ifdef CONFIG_HUGETLBFS

+1 -1

drivers/block/zram/zram_drv.c

reviewed

··· 1270 1270 zram_slot_unlock(zram, index); 1271 1271 1272 1272 /* Should NEVER happen. Return bio error if it does. */ 1273 1273 - if (unlikely(ret)) 1273 1273 + if (WARN_ON(ret)) 1274 1274 pr_err("Decompression failed! err=%d, page=%u\n", ret, index); 1275 1275 1276 1276 return ret;

+35 -15

drivers/dax/kmem.c

reviewed

··· 35 35 return 0; 36 36 } 37 37 38 38 + struct dax_kmem_data { 39 39 + const char *res_name; 40 40 + struct resource *res[]; 41 41 + }; 42 42 + 38 43 static int dev_dax_kmem_probe(struct dev_dax *dev_dax) 39 44 { 40 45 struct device *dev = &dev_dax->dev; 46 46 + struct dax_kmem_data *data; 47 47 + int rc = -ENOMEM; 41 48 int i, mapped = 0; 42 42 - char *res_name; 43 49 int numa_node; 44 50 45 51 /* ··· 61 55 return -EINVAL; 62 56 } 63 57 64 64 - res_name = kstrdup(dev_name(dev), GFP_KERNEL); 65 65 - if (!res_name) 58 58 + data = kzalloc(sizeof(*data) + sizeof(struct resource *) * dev_dax->nr_range, GFP_KERNEL); 59 59 + if (!data) 66 60 return -ENOMEM; 61 61 + 62 62 + data->res_name = kstrdup(dev_name(dev), GFP_KERNEL); 63 63 + if (!data->res_name) 64 64 + goto err_res_name; 67 65 68 66 for (i = 0; i < dev_dax->nr_range; i++) { 69 67 struct resource *res; 70 68 struct range range; 71 71 - int rc; 72 69 73 70 rc = dax_kmem_range(dev_dax, i, &range); 74 71 if (rc) { ··· 81 72 } 82 73 83 74 /* Region is permanently reserved if hotremove fails. */ 84 84 - res = request_mem_region(range.start, range_len(&range), res_name); 75 75 + res = request_mem_region(range.start, range_len(&range), data->res_name); 85 76 if (!res) { 86 77 dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n", 87 78 i, range.start, range.end); ··· 91 82 */ 92 83 if (mapped) 93 84 continue; 94 94 - kfree(res_name); 95 95 - return -EBUSY; 85 85 + rc = -EBUSY; 86 86 + goto err_request_mem; 96 87 } 88 88 + data->res[i] = res; 97 89 98 90 /* 99 91 * Set flags appropriate for System RAM. Leave ..._BUSY clear ··· 109 99 * this as RAM automatically. 110 100 */ 111 101 rc = add_memory_driver_managed(numa_node, range.start, 112 112 - range_len(&range), kmem_name); 102 102 + range_len(&range), kmem_name, MHP_NONE); 113 103 114 104 if (rc) { 115 105 dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n", 116 106 i, range.start, range.end); 117 117 - release_mem_region(range.start, range_len(&range)); 107 107 + release_resource(res); 108 108 + kfree(res); 109 109 + data->res[i] = NULL; 118 110 if (mapped) 119 111 continue; 120 120 - kfree(res_name); 121 121 - return rc; 112 112 + goto err_request_mem; 122 113 } 123 114 mapped++; 124 115 } 125 116 126 126 - dev_set_drvdata(dev, res_name); 117 117 + dev_set_drvdata(dev, data); 127 118 128 119 return 0; 120 120 + 121 121 + err_request_mem: 122 122 + kfree(data->res_name); 123 123 + err_res_name: 124 124 + kfree(data); 125 125 + return rc; 129 126 } 130 127 131 128 #ifdef CONFIG_MEMORY_HOTREMOVE ··· 140 123 { 141 124 int i, success = 0; 142 125 struct device *dev = &dev_dax->dev; 143 143 - const char *res_name = dev_get_drvdata(dev); 126 126 + struct dax_kmem_data *data = dev_get_drvdata(dev); 144 127 145 128 /* 146 129 * We have one shot for removing memory, if some memory blocks were not ··· 159 142 rc = remove_memory(dev_dax->target_node, range.start, 160 143 range_len(&range)); 161 144 if (rc == 0) { 162 162 - release_mem_region(range.start, range_len(&range)); 145 145 + release_resource(data->res[i]); 146 146 + kfree(data->res[i]); 147 147 + data->res[i] = NULL; 163 148 success++; 164 149 continue; 165 150 } ··· 172 153 } 173 154 174 155 if (success >= dev_dax->nr_range) { 175 175 - kfree(res_name); 156 156 + kfree(data->res_name); 157 157 + kfree(data); 176 158 dev_set_drvdata(dev, NULL); 177 159 } 178 160

+1 -1

drivers/hv/hv_balloon.c

reviewed

··· 726 726 727 727 nid = memory_add_physaddr_to_nid(PFN_PHYS(start_pfn)); 728 728 ret = add_memory(nid, PFN_PHYS((start_pfn)), 729 729 - (HA_CHUNK << PAGE_SHIFT)); 729 729 + (HA_CHUNK << PAGE_SHIFT), MEMHP_MERGE_RESOURCE); 730 730 731 731 if (ret) { 732 732 pr_err("hot_add memory failed error is %d\n", ret);

-3

drivers/infiniband/core/uverbs_main.c

reviewed

··· 845 845 * will only be one mm, so no big deal. 846 846 */ 847 847 mmap_read_lock(mm); 848 848 - if (!mmget_still_valid(mm)) 849 849 - goto skip_mm; 850 848 mutex_lock(&ufile->umap_lock); 851 849 list_for_each_entry_safe (priv, next_priv, &ufile->umaps, 852 850 list) { ··· 863 865 } 864 866 } 865 867 mutex_unlock(&ufile->umap_lock); 866 866 - skip_mm: 867 868 mmap_read_unlock(mm); 868 869 mmput(mm); 869 870 }

+11 -7

drivers/rapidio/devices/rio_mport_cdev.c

reviewed

··· 871 871 rmcd_error("pin_user_pages_fast err=%ld", 872 872 pinned); 873 873 nr_pages = 0; 874 874 - } else 874 874 + } else { 875 875 rmcd_error("pinned %ld out of %ld pages", 876 876 pinned, nr_pages); 877 877 + /* 878 878 + * Set nr_pages up to mean "how many pages to unpin, in 879 879 + * the error handler: 880 880 + */ 881 881 + nr_pages = pinned; 882 882 + } 877 883 ret = -EFAULT; 878 878 - /* 879 879 - * Set nr_pages up to mean "how many pages to unpin, in 880 880 - * the error handler: 881 881 - */ 882 882 - nr_pages = pinned; 883 884 goto err_pg; 884 885 } 885 886 ··· 1680 1679 struct rio_dev *rdev; 1681 1680 struct rio_switch *rswitch = NULL; 1682 1681 struct rio_mport *mport; 1682 1682 + struct device *dev; 1683 1683 size_t size; 1684 1684 u32 rval; 1685 1685 u32 swpinfo = 0; ··· 1695 1693 rmcd_debug(RDEV, "name:%s ct:0x%x did:0x%x hc:0x%x", dev_info.name, 1696 1694 dev_info.comptag, dev_info.destid, dev_info.hopcount); 1697 1695 1698 1698 - if (bus_find_device_by_name(&rio_bus_type, NULL, dev_info.name)) { 1696 1696 + dev = bus_find_device_by_name(&rio_bus_type, NULL, dev_info.name); 1697 1697 + if (dev) { 1699 1698 rmcd_debug(RDEV, "device %s already exists", dev_info.name); 1699 1699 + put_device(dev); 1700 1700 return -EEXIST; 1701 1701 } 1702 1702

+1 -1

drivers/s390/char/sclp_cmd.c

reviewed

··· 406 406 if (!size) 407 407 goto skip_add; 408 408 for (addr = start; addr < start + size; addr += block_size) 409 409 - add_memory(0, addr, block_size); 409 409 + add_memory(0, addr, block_size, MHP_NONE); 410 410 skip_add: 411 411 first_rn = rn; 412 412 num = 1;

+21 -23

drivers/vfio/pci/vfio_pci.c

reviewed

··· 1480 1480 } else { 1481 1481 mmap_read_lock(mm); 1482 1482 } 1483 1483 - if (mmget_still_valid(mm)) { 1484 1484 - if (try) { 1485 1485 - if (!mutex_trylock(&vdev->vma_lock)) { 1486 1486 - mmap_read_unlock(mm); 1487 1487 - mmput(mm); 1488 1488 - return 0; 1489 1489 - } 1490 1490 - } else { 1491 1491 - mutex_lock(&vdev->vma_lock); 1483 1483 + if (try) { 1484 1484 + if (!mutex_trylock(&vdev->vma_lock)) { 1485 1485 + mmap_read_unlock(mm); 1486 1486 + mmput(mm); 1487 1487 + return 0; 1492 1488 } 1493 1493 - list_for_each_entry_safe(mmap_vma, tmp, 1494 1494 - &vdev->vma_list, vma_next) { 1495 1495 - struct vm_area_struct *vma = mmap_vma->vma; 1496 1496 - 1497 1497 - if (vma->vm_mm != mm) 1498 1498 - continue; 1499 1499 - 1500 1500 - list_del(&mmap_vma->vma_next); 1501 1501 - kfree(mmap_vma); 1502 1502 - 1503 1503 - zap_vma_ptes(vma, vma->vm_start, 1504 1504 - vma->vm_end - vma->vm_start); 1505 1505 - } 1506 1506 - mutex_unlock(&vdev->vma_lock); 1489 1489 + } else { 1490 1490 + mutex_lock(&vdev->vma_lock); 1507 1491 } 1492 1492 + list_for_each_entry_safe(mmap_vma, tmp, 1493 1493 + &vdev->vma_list, vma_next) { 1494 1494 + struct vm_area_struct *vma = mmap_vma->vma; 1495 1495 + 1496 1496 + if (vma->vm_mm != mm) 1497 1497 + continue; 1498 1498 + 1499 1499 + list_del(&mmap_vma->vma_next); 1500 1500 + kfree(mmap_vma); 1501 1501 + 1502 1502 + zap_vma_ptes(vma, vma->vm_start, 1503 1503 + vma->vm_end - vma->vm_start); 1504 1504 + } 1505 1505 + mutex_unlock(&vdev->vma_lock); 1508 1506 mmap_read_unlock(mm); 1509 1507 mmput(mm); 1510 1508 }

+2 -1

drivers/virtio/virtio_mem.c

reviewed

··· 424 424 425 425 dev_dbg(&vm->vdev->dev, "adding memory block: %lu\n", mb_id); 426 426 return add_memory_driver_managed(nid, addr, memory_block_size_bytes(), 427 427 - vm->resource_name); 427 427 + vm->resource_name, 428 428 + MEMHP_MERGE_RESOURCE); 428 429 } 429 430 430 431 /*

+1 -1

drivers/xen/balloon.c

reviewed

··· 331 331 mutex_unlock(&balloon_mutex); 332 332 /* add_memory_resource() requires the device_hotplug lock */ 333 333 lock_device_hotplug(); 334 334 - rc = add_memory_resource(nid, resource); 334 334 + rc = add_memory_resource(nid, resource, MEMHP_MERGE_RESOURCE); 335 335 unlock_device_hotplug(); 336 336 mutex_lock(&balloon_mutex); 337 337

+6 -2

fs/autofs/dev-ioctl.c

reviewed

··· 8 8 #include <linux/compat.h> 9 9 #include <linux/syscalls.h> 10 10 #include <linux/magic.h> 11 11 + #include <linux/nospec.h> 11 12 12 13 #include "autofs_i.h" 13 14 ··· 564 563 565 564 static ioctl_fn lookup_dev_ioctl(unsigned int cmd) 566 565 { 567 567 - static ioctl_fn _ioctls[] = { 566 566 + static const ioctl_fn _ioctls[] = { 568 567 autofs_dev_ioctl_version, 569 568 autofs_dev_ioctl_protover, 570 569 autofs_dev_ioctl_protosubver, ··· 582 581 }; 583 582 unsigned int idx = cmd_idx(cmd); 584 583 585 585 - return (idx >= ARRAY_SIZE(_ioctls)) ? NULL : _ioctls[idx]; 584 584 + if (idx >= ARRAY_SIZE(_ioctls)) 585 585 + return NULL; 586 586 + idx = array_index_nospec(idx, ARRAY_SIZE(_ioctls)); 587 587 + return _ioctls[idx]; 586 588 } 587 589 588 590 /* ioctl dispatcher */

+47 -216

fs/binfmt_elf.c

reviewed

··· 13 13 #include <linux/module.h> 14 14 #include <linux/kernel.h> 15 15 #include <linux/fs.h> 16 16 + #include <linux/log2.h> 16 17 #include <linux/mm.h> 17 18 #include <linux/mman.h> 18 19 #include <linux/errno.h> ··· 420 419 return (rv < 0) ? rv : -EIO; 421 420 } 422 421 return 0; 422 422 + } 423 423 + 424 424 + static unsigned long maximum_alignment(struct elf_phdr *cmds, int nr) 425 425 + { 426 426 + unsigned long alignment = 0; 427 427 + int i; 428 428 + 429 429 + for (i = 0; i < nr; i++) { 430 430 + if (cmds[i].p_type == PT_LOAD) { 431 431 + unsigned long p_align = cmds[i].p_align; 432 432 + 433 433 + /* skip non-power of two alignments as invalid */ 434 434 + if (!is_power_of_2(p_align)) 435 435 + continue; 436 436 + alignment = max(alignment, p_align); 437 437 + } 438 438 + } 439 439 + 440 440 + /* ensure we align to at least one page */ 441 441 + return ELF_PAGEALIGN(alignment); 423 442 } 424 443 425 444 /** ··· 1029 1008 int elf_prot, elf_flags; 1030 1009 unsigned long k, vaddr; 1031 1010 unsigned long total_size = 0; 1011 1011 + unsigned long alignment; 1032 1012 1033 1013 if (elf_ppnt->p_type != PT_LOAD) 1034 1014 continue; ··· 1108 1086 load_bias = ELF_ET_DYN_BASE; 1109 1087 if (current->flags & PF_RANDOMIZE) 1110 1088 load_bias += arch_mmap_rnd(); 1089 1089 + alignment = maximum_alignment(elf_phdata, elf_ex->e_phnum); 1090 1090 + if (alignment) 1091 1091 + load_bias &= ~(alignment - 1); 1111 1092 elf_flags |= MAP_FIXED; 1112 1093 } else 1113 1094 load_bias = 0; ··· 1413 1388 * Modelled on fs/exec.c:aout_core_dump() 1414 1389 * Jeremy Fitzhardinge <jeremy@sw.oz.au> 1415 1390 */ 1416 1416 - 1417 1417 - /* 1418 1418 - * The purpose of always_dump_vma() is to make sure that special kernel mappings 1419 1419 - * that are useful for post-mortem analysis are included in every core dump. 1420 1420 - * In that way we ensure that the core dump is fully interpretable later 1421 1421 - * without matching up the same kernel and hardware config to see what PC values 1422 1422 - * meant. These special mappings include - vDSO, vsyscall, and other 1423 1423 - * architecture specific mappings 1424 1424 - */ 1425 1425 - static bool always_dump_vma(struct vm_area_struct *vma) 1426 1426 - { 1427 1427 - /* Any vsyscall mappings? */ 1428 1428 - if (vma == get_gate_vma(vma->vm_mm)) 1429 1429 - return true; 1430 1430 - 1431 1431 - /* 1432 1432 - * Assume that all vmas with a .name op should always be dumped. 1433 1433 - * If this changes, a new vm_ops field can easily be added. 1434 1434 - */ 1435 1435 - if (vma->vm_ops && vma->vm_ops->name && vma->vm_ops->name(vma)) 1436 1436 - return true; 1437 1437 - 1438 1438 - /* 1439 1439 - * arch_vma_name() returns non-NULL for special architecture mappings, 1440 1440 - * such as vDSO sections. 1441 1441 - */ 1442 1442 - if (arch_vma_name(vma)) 1443 1443 - return true; 1444 1444 - 1445 1445 - return false; 1446 1446 - } 1447 1447 - 1448 1448 - /* 1449 1449 - * Decide what to dump of a segment, part, all or none. 1450 1450 - */ 1451 1451 - static unsigned long vma_dump_size(struct vm_area_struct *vma, 1452 1452 - unsigned long mm_flags) 1453 1453 - { 1454 1454 - #define FILTER(type) (mm_flags & (1UL << MMF_DUMP_##type)) 1455 1455 - 1456 1456 - /* always dump the vdso and vsyscall sections */ 1457 1457 - if (always_dump_vma(vma)) 1458 1458 - goto whole; 1459 1459 - 1460 1460 - if (vma->vm_flags & VM_DONTDUMP) 1461 1461 - return 0; 1462 1462 - 1463 1463 - /* support for DAX */ 1464 1464 - if (vma_is_dax(vma)) { 1465 1465 - if ((vma->vm_flags & VM_SHARED) && FILTER(DAX_SHARED)) 1466 1466 - goto whole; 1467 1467 - if (!(vma->vm_flags & VM_SHARED) && FILTER(DAX_PRIVATE)) 1468 1468 - goto whole; 1469 1469 - return 0; 1470 1470 - } 1471 1471 - 1472 1472 - /* Hugetlb memory check */ 1473 1473 - if (is_vm_hugetlb_page(vma)) { 1474 1474 - if ((vma->vm_flags & VM_SHARED) && FILTER(HUGETLB_SHARED)) 1475 1475 - goto whole; 1476 1476 - if (!(vma->vm_flags & VM_SHARED) && FILTER(HUGETLB_PRIVATE)) 1477 1477 - goto whole; 1478 1478 - return 0; 1479 1479 - } 1480 1480 - 1481 1481 - /* Do not dump I/O mapped devices or special mappings */ 1482 1482 - if (vma->vm_flags & VM_IO) 1483 1483 - return 0; 1484 1484 - 1485 1485 - /* By default, dump shared memory if mapped from an anonymous file. */ 1486 1486 - if (vma->vm_flags & VM_SHARED) { 1487 1487 - if (file_inode(vma->vm_file)->i_nlink == 0 ? 1488 1488 - FILTER(ANON_SHARED) : FILTER(MAPPED_SHARED)) 1489 1489 - goto whole; 1490 1490 - return 0; 1491 1491 - } 1492 1492 - 1493 1493 - /* Dump segments that have been written to. */ 1494 1494 - if (vma->anon_vma && FILTER(ANON_PRIVATE)) 1495 1495 - goto whole; 1496 1496 - if (vma->vm_file == NULL) 1497 1497 - return 0; 1498 1498 - 1499 1499 - if (FILTER(MAPPED_PRIVATE)) 1500 1500 - goto whole; 1501 1501 - 1502 1502 - /* 1503 1503 - * If this looks like the beginning of a DSO or executable mapping, 1504 1504 - * check for an ELF header. If we find one, dump the first page to 1505 1505 - * aid in determining what was mapped here. 1506 1506 - */ 1507 1507 - if (FILTER(ELF_HEADERS) && 1508 1508 - vma->vm_pgoff == 0 && (vma->vm_flags & VM_READ)) { 1509 1509 - u32 __user *header = (u32 __user *) vma->vm_start; 1510 1510 - u32 word; 1511 1511 - /* 1512 1512 - * Doing it this way gets the constant folded by GCC. 1513 1513 - */ 1514 1514 - union { 1515 1515 - u32 cmp; 1516 1516 - char elfmag[SELFMAG]; 1517 1517 - } magic; 1518 1518 - BUILD_BUG_ON(SELFMAG != sizeof word); 1519 1519 - magic.elfmag[EI_MAG0] = ELFMAG0; 1520 1520 - magic.elfmag[EI_MAG1] = ELFMAG1; 1521 1521 - magic.elfmag[EI_MAG2] = ELFMAG2; 1522 1522 - magic.elfmag[EI_MAG3] = ELFMAG3; 1523 1523 - if (unlikely(get_user(word, header))) 1524 1524 - word = 0; 1525 1525 - if (word == magic.cmp) 1526 1526 - return PAGE_SIZE; 1527 1527 - } 1528 1528 - 1529 1529 - #undef FILTER 1530 1530 - 1531 1531 - return 0; 1532 1532 - 1533 1533 - whole: 1534 1534 - return vma->vm_end - vma->vm_start; 1535 1535 - } 1536 1391 1537 1392 /* An ELF note in memory */ 1538 1393 struct memelfnote ··· 2125 2220 2126 2221 #endif 2127 2222 2128 2128 - static struct vm_area_struct *first_vma(struct task_struct *tsk, 2129 2129 - struct vm_area_struct *gate_vma) 2130 2130 - { 2131 2131 - struct vm_area_struct *ret = tsk->mm->mmap; 2132 2132 - 2133 2133 - if (ret) 2134 2134 - return ret; 2135 2135 - return gate_vma; 2136 2136 - } 2137 2137 - /* 2138 2138 - * Helper function for iterating across a vma list. It ensures that the caller 2139 2139 - * will visit `gate_vma' prior to terminating the search. 2140 2140 - */ 2141 2141 - static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma, 2142 2142 - struct vm_area_struct *gate_vma) 2143 2143 - { 2144 2144 - struct vm_area_struct *ret; 2145 2145 - 2146 2146 - ret = this_vma->vm_next; 2147 2147 - if (ret) 2148 2148 - return ret; 2149 2149 - if (this_vma == gate_vma) 2150 2150 - return NULL; 2151 2151 - return gate_vma; 2152 2152 - } 2153 2153 - 2154 2223 static void fill_extnum_info(struct elfhdr *elf, struct elf_shdr *shdr4extnum, 2155 2224 elf_addr_t e_shoff, int segs) 2156 2225 { ··· 2151 2272 static int elf_core_dump(struct coredump_params *cprm) 2152 2273 { 2153 2274 int has_dumped = 0; 2154 2154 - int segs, i; 2155 2155 - size_t vma_data_size = 0; 2156 2156 - struct vm_area_struct *vma, *gate_vma; 2275 2275 + int vma_count, segs, i; 2276 2276 + size_t vma_data_size; 2157 2277 struct elfhdr elf; 2158 2278 loff_t offset = 0, dataoff; 2159 2279 struct elf_note_info info = { }; ··· 2160 2282 struct elf_shdr *shdr4extnum = NULL; 2161 2283 Elf_Half e_phnum; 2162 2284 elf_addr_t e_shoff; 2163 2163 - elf_addr_t *vma_filesz = NULL; 2285 2285 + struct core_vma_metadata *vma_meta; 2164 2286 2165 2165 - /* 2166 2166 - * We no longer stop all VM operations. 2167 2167 - * 2168 2168 - * This is because those proceses that could possibly change map_count 2169 2169 - * or the mmap / vma pages are now blocked in do_exit on current 2170 2170 - * finishing this core dump. 2171 2171 - * 2172 2172 - * Only ptrace can touch these memory addresses, but it doesn't change 2173 2173 - * the map_count or the pages allocated. So no possibility of crashing 2174 2174 - * exists while dumping the mm->vm_next areas to the core file. 2175 2175 - */ 2176 2176 - 2287 2287 + if (dump_vma_snapshot(cprm, &vma_count, &vma_meta, &vma_data_size)) 2288 2288 + return 0; 2289 2289 + 2177 2290 /* 2178 2291 * The number of segs are recored into ELF header as 16bit value. 2179 2292 * Please check DEFAULT_MAX_MAP_COUNT definition when you modify here. 2180 2293 */ 2181 2181 - segs = current->mm->map_count; 2182 2182 - segs += elf_core_extra_phdrs(); 2183 2183 - 2184 2184 - gate_vma = get_gate_vma(current->mm); 2185 2185 - if (gate_vma != NULL) 2186 2186 - segs++; 2294 2294 + segs = vma_count + elf_core_extra_phdrs(); 2187 2295 2188 2296 /* for notes section */ 2189 2297 segs++; ··· 2207 2343 2208 2344 dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE); 2209 2345 2210 2210 - /* 2211 2211 - * Zero vma process will get ZERO_SIZE_PTR here. 2212 2212 - * Let coredump continue for register state at least. 2213 2213 - */ 2214 2214 - vma_filesz = kvmalloc(array_size(sizeof(*vma_filesz), (segs - 1)), 2215 2215 - GFP_KERNEL); 2216 2216 - if (!vma_filesz) 2217 2217 - goto end_coredump; 2218 2218 - 2219 2219 - for (i = 0, vma = first_vma(current, gate_vma); vma != NULL; 2220 2220 - vma = next_vma(vma, gate_vma)) { 2221 2221 - unsigned long dump_size; 2222 2222 - 2223 2223 - dump_size = vma_dump_size(vma, cprm->mm_flags); 2224 2224 - vma_filesz[i++] = dump_size; 2225 2225 - vma_data_size += dump_size; 2226 2226 - } 2227 2227 - 2228 2346 offset += vma_data_size; 2229 2347 offset += elf_core_extra_data_size(); 2230 2348 e_shoff = offset; ··· 2227 2381 goto end_coredump; 2228 2382 2229 2383 /* Write program headers for segments dump */ 2230 2230 - for (i = 0, vma = first_vma(current, gate_vma); vma != NULL; 2231 2231 - vma = next_vma(vma, gate_vma)) { 2384 2384 + for (i = 0; i < vma_count; i++) { 2385 2385 + struct core_vma_metadata *meta = vma_meta + i; 2232 2386 struct elf_phdr phdr; 2233 2387 2234 2388 phdr.p_type = PT_LOAD; 2235 2389 phdr.p_offset = offset; 2236 2236 - phdr.p_vaddr = vma->vm_start; 2390 2390 + phdr.p_vaddr = meta->start; 2237 2391 phdr.p_paddr = 0; 2238 2238 - phdr.p_filesz = vma_filesz[i++]; 2239 2239 - phdr.p_memsz = vma->vm_end - vma->vm_start; 2392 2392 + phdr.p_filesz = meta->dump_size; 2393 2393 + phdr.p_memsz = meta->end - meta->start; 2240 2394 offset += phdr.p_filesz; 2241 2241 - phdr.p_flags = vma->vm_flags & VM_READ ? PF_R : 0; 2242 2242 - if (vma->vm_flags & VM_WRITE) 2395 2395 + phdr.p_flags = 0; 2396 2396 + if (meta->flags & VM_READ) 2397 2397 + phdr.p_flags |= PF_R; 2398 2398 + if (meta->flags & VM_WRITE) 2243 2399 phdr.p_flags |= PF_W; 2244 2244 - if (vma->vm_flags & VM_EXEC) 2400 2400 + if (meta->flags & VM_EXEC) 2245 2401 phdr.p_flags |= PF_X; 2246 2402 phdr.p_align = ELF_EXEC_PAGESIZE; 2247 2403 ··· 2265 2417 if (!dump_skip(cprm, dataoff - cprm->pos)) 2266 2418 goto end_coredump; 2267 2419 2268 2268 - for (i = 0, vma = first_vma(current, gate_vma); vma != NULL; 2269 2269 - vma = next_vma(vma, gate_vma)) { 2270 2270 - unsigned long addr; 2271 2271 - unsigned long end; 2420 2420 + for (i = 0; i < vma_count; i++) { 2421 2421 + struct core_vma_metadata *meta = vma_meta + i; 2272 2422 2273 2273 - end = vma->vm_start + vma_filesz[i++]; 2274 2274 - 2275 2275 - for (addr = vma->vm_start; addr < end; addr += PAGE_SIZE) { 2276 2276 - struct page *page; 2277 2277 - int stop; 2278 2278 - 2279 2279 - page = get_dump_page(addr); 2280 2280 - if (page) { 2281 2281 - void *kaddr = kmap(page); 2282 2282 - stop = !dump_emit(cprm, kaddr, PAGE_SIZE); 2283 2283 - kunmap(page); 2284 2284 - put_page(page); 2285 2285 - } else 2286 2286 - stop = !dump_skip(cprm, PAGE_SIZE); 2287 2287 - if (stop) 2288 2288 - goto end_coredump; 2289 2289 - } 2423 2423 + if (!dump_user_range(cprm, meta->start, meta->dump_size)) 2424 2424 + goto end_coredump; 2290 2425 } 2291 2426 dump_truncate(cprm); 2292 2427 ··· 2284 2453 end_coredump: 2285 2454 free_note_info(&info); 2286 2455 kfree(shdr4extnum); 2287 2287 - kvfree(vma_filesz); 2456 2456 + kvfree(vma_meta); 2288 2457 kfree(phdr4note); 2289 2458 return has_dumped; 2290 2459 }

+27 -135

fs/binfmt_elf_fdpic.c

reviewed

··· 1215 1215 int pr_fpvalid; /* True if math co-processor being used. */ 1216 1216 }; 1217 1217 1218 1218 - /* 1219 1219 - * Decide whether a segment is worth dumping; default is yes to be 1220 1220 - * sure (missing info is worse than too much; etc). 1221 1221 - * Personally I'd include everything, and use the coredump limit... 1222 1222 - * 1223 1223 - * I think we should skip something. But I am not sure how. H.J. 1224 1224 - */ 1225 1225 - static int maydump(struct vm_area_struct *vma, unsigned long mm_flags) 1226 1226 - { 1227 1227 - int dump_ok; 1228 1228 - 1229 1229 - /* Do not dump I/O mapped devices or special mappings */ 1230 1230 - if (vma->vm_flags & VM_IO) { 1231 1231 - kdcore("%08lx: %08lx: no (IO)", vma->vm_start, vma->vm_flags); 1232 1232 - return 0; 1233 1233 - } 1234 1234 - 1235 1235 - /* If we may not read the contents, don't allow us to dump 1236 1236 - * them either. "dump_write()" can't handle it anyway. 1237 1237 - */ 1238 1238 - if (!(vma->vm_flags & VM_READ)) { 1239 1239 - kdcore("%08lx: %08lx: no (!read)", vma->vm_start, vma->vm_flags); 1240 1240 - return 0; 1241 1241 - } 1242 1242 - 1243 1243 - /* support for DAX */ 1244 1244 - if (vma_is_dax(vma)) { 1245 1245 - if (vma->vm_flags & VM_SHARED) { 1246 1246 - dump_ok = test_bit(MMF_DUMP_DAX_SHARED, &mm_flags); 1247 1247 - kdcore("%08lx: %08lx: %s (DAX shared)", vma->vm_start, 1248 1248 - vma->vm_flags, dump_ok ? "yes" : "no"); 1249 1249 - } else { 1250 1250 - dump_ok = test_bit(MMF_DUMP_DAX_PRIVATE, &mm_flags); 1251 1251 - kdcore("%08lx: %08lx: %s (DAX private)", vma->vm_start, 1252 1252 - vma->vm_flags, dump_ok ? "yes" : "no"); 1253 1253 - } 1254 1254 - return dump_ok; 1255 1255 - } 1256 1256 - 1257 1257 - /* By default, dump shared memory if mapped from an anonymous file. */ 1258 1258 - if (vma->vm_flags & VM_SHARED) { 1259 1259 - if (file_inode(vma->vm_file)->i_nlink == 0) { 1260 1260 - dump_ok = test_bit(MMF_DUMP_ANON_SHARED, &mm_flags); 1261 1261 - kdcore("%08lx: %08lx: %s (share)", vma->vm_start, 1262 1262 - vma->vm_flags, dump_ok ? "yes" : "no"); 1263 1263 - return dump_ok; 1264 1264 - } 1265 1265 - 1266 1266 - dump_ok = test_bit(MMF_DUMP_MAPPED_SHARED, &mm_flags); 1267 1267 - kdcore("%08lx: %08lx: %s (share)", vma->vm_start, 1268 1268 - vma->vm_flags, dump_ok ? "yes" : "no"); 1269 1269 - return dump_ok; 1270 1270 - } 1271 1271 - 1272 1272 - #ifdef CONFIG_MMU 1273 1273 - /* By default, if it hasn't been written to, don't write it out */ 1274 1274 - if (!vma->anon_vma) { 1275 1275 - dump_ok = test_bit(MMF_DUMP_MAPPED_PRIVATE, &mm_flags); 1276 1276 - kdcore("%08lx: %08lx: %s (!anon)", vma->vm_start, 1277 1277 - vma->vm_flags, dump_ok ? "yes" : "no"); 1278 1278 - return dump_ok; 1279 1279 - } 1280 1280 - #endif 1281 1281 - 1282 1282 - dump_ok = test_bit(MMF_DUMP_ANON_PRIVATE, &mm_flags); 1283 1283 - kdcore("%08lx: %08lx: %s", vma->vm_start, vma->vm_flags, 1284 1284 - dump_ok ? "yes" : "no"); 1285 1285 - return dump_ok; 1286 1286 - } 1287 1287 - 1288 1218 /* An ELF note in memory */ 1289 1219 struct memelfnote 1290 1220 { ··· 1454 1524 /* 1455 1525 * dump the segments for an MMU process 1456 1526 */ 1457 1457 - static bool elf_fdpic_dump_segments(struct coredump_params *cprm) 1527 1527 + static bool elf_fdpic_dump_segments(struct coredump_params *cprm, 1528 1528 + struct core_vma_metadata *vma_meta, 1529 1529 + int vma_count) 1458 1530 { 1459 1459 - struct vm_area_struct *vma; 1531 1531 + int i; 1460 1532 1461 1461 - for (vma = current->mm->mmap; vma; vma = vma->vm_next) { 1462 1462 - #ifdef CONFIG_MMU 1463 1463 - unsigned long addr; 1464 1464 - #endif 1533 1533 + for (i = 0; i < vma_count; i++) { 1534 1534 + struct core_vma_metadata *meta = vma_meta + i; 1465 1535 1466 1466 - if (!maydump(vma, cprm->mm_flags)) 1467 1467 - continue; 1468 1468 - 1469 1469 - #ifdef CONFIG_MMU 1470 1470 - for (addr = vma->vm_start; addr < vma->vm_end; 1471 1471 - addr += PAGE_SIZE) { 1472 1472 - bool res; 1473 1473 - struct page *page = get_dump_page(addr); 1474 1474 - if (page) { 1475 1475 - void *kaddr = kmap(page); 1476 1476 - res = dump_emit(cprm, kaddr, PAGE_SIZE); 1477 1477 - kunmap(page); 1478 1478 - put_page(page); 1479 1479 - } else { 1480 1480 - res = dump_skip(cprm, PAGE_SIZE); 1481 1481 - } 1482 1482 - if (!res) 1483 1483 - return false; 1484 1484 - } 1485 1485 - #else 1486 1486 - if (!dump_emit(cprm, (void *) vma->vm_start, 1487 1487 - vma->vm_end - vma->vm_start)) 1536 1536 + if (!dump_user_range(cprm, meta->start, meta->dump_size)) 1488 1537 return false; 1489 1489 - #endif 1490 1538 } 1491 1539 return true; 1492 1492 - } 1493 1493 - 1494 1494 - static size_t elf_core_vma_data_size(unsigned long mm_flags) 1495 1495 - { 1496 1496 - struct vm_area_struct *vma; 1497 1497 - size_t size = 0; 1498 1498 - 1499 1499 - for (vma = current->mm->mmap; vma; vma = vma->vm_next) 1500 1500 - if (maydump(vma, mm_flags)) 1501 1501 - size += vma->vm_end - vma->vm_start; 1502 1502 - return size; 1503 1540 } 1504 1541 1505 1542 /* ··· 1479 1582 static int elf_fdpic_core_dump(struct coredump_params *cprm) 1480 1583 { 1481 1584 int has_dumped = 0; 1482 1482 - int segs; 1585 1585 + int vma_count, segs; 1483 1586 int i; 1484 1484 - struct vm_area_struct *vma; 1485 1587 struct elfhdr *elf = NULL; 1486 1588 loff_t offset = 0, dataoff; 1487 1589 struct memelfnote psinfo_note, auxv_note; ··· 1494 1598 elf_addr_t e_shoff; 1495 1599 struct core_thread *ct; 1496 1600 struct elf_thread_status *tmp; 1497 1497 - 1498 1498 - /* 1499 1499 - * We no longer stop all VM operations. 1500 1500 - * 1501 1501 - * This is because those proceses that could possibly change map_count 1502 1502 - * or the mmap / vma pages are now blocked in do_exit on current 1503 1503 - * finishing this core dump. 1504 1504 - * 1505 1505 - * Only ptrace can touch these memory addresses, but it doesn't change 1506 1506 - * the map_count or the pages allocated. So no possibility of crashing 1507 1507 - * exists while dumping the mm->vm_next areas to the core file. 1508 1508 - */ 1601 1601 + struct core_vma_metadata *vma_meta = NULL; 1602 1602 + size_t vma_data_size; 1509 1603 1510 1604 /* alloc memory for large data structures: too large to be on stack */ 1511 1605 elf = kmalloc(sizeof(*elf), GFP_KERNEL); ··· 1503 1617 goto end_coredump; 1504 1618 psinfo = kmalloc(sizeof(*psinfo), GFP_KERNEL); 1505 1619 if (!psinfo) 1620 1620 + goto end_coredump; 1621 1621 + 1622 1622 + if (dump_vma_snapshot(cprm, &vma_count, &vma_meta, &vma_data_size)) 1506 1623 goto end_coredump; 1507 1624 1508 1625 for (ct = current->mm->core_state->dumper.next; ··· 1527 1638 tmp->next = thread_list; 1528 1639 thread_list = tmp; 1529 1640 1530 1530 - segs = current->mm->map_count; 1531 1531 - segs += elf_core_extra_phdrs(); 1641 1641 + segs = vma_count + elf_core_extra_phdrs(); 1532 1642 1533 1643 /* for notes section */ 1534 1644 segs++; ··· 1572 1684 /* Page-align dumped data */ 1573 1685 dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE); 1574 1686 1575 1575 - offset += elf_core_vma_data_size(cprm->mm_flags); 1687 1687 + offset += vma_data_size; 1576 1688 offset += elf_core_extra_data_size(); 1577 1689 e_shoff = offset; 1578 1690 ··· 1592 1704 goto end_coredump; 1593 1705 1594 1706 /* write program headers for segments dump */ 1595 1595 - for (vma = current->mm->mmap; vma; vma = vma->vm_next) { 1707 1707 + for (i = 0; i < vma_count; i++) { 1708 1708 + struct core_vma_metadata *meta = vma_meta + i; 1596 1709 struct elf_phdr phdr; 1597 1710 size_t sz; 1598 1711 1599 1599 - sz = vma->vm_end - vma->vm_start; 1712 1712 + sz = meta->end - meta->start; 1600 1713 1601 1714 phdr.p_type = PT_LOAD; 1602 1715 phdr.p_offset = offset; 1603 1603 - phdr.p_vaddr = vma->vm_start; 1716 1716 + phdr.p_vaddr = meta->start; 1604 1717 phdr.p_paddr = 0; 1605 1605 - phdr.p_filesz = maydump(vma, cprm->mm_flags) ? sz : 0; 1718 1718 + phdr.p_filesz = meta->dump_size; 1606 1719 phdr.p_memsz = sz; 1607 1720 offset += phdr.p_filesz; 1608 1608 - phdr.p_flags = vma->vm_flags & VM_READ ? PF_R : 0; 1609 1609 - if (vma->vm_flags & VM_WRITE) 1721 1721 + phdr.p_flags = 0; 1722 1722 + if (meta->flags & VM_READ) 1723 1723 + phdr.p_flags |= PF_R; 1724 1724 + if (meta->flags & VM_WRITE) 1610 1725 phdr.p_flags |= PF_W; 1611 1611 - if (vma->vm_flags & VM_EXEC) 1726 1726 + if (meta->flags & VM_EXEC) 1612 1727 phdr.p_flags |= PF_X; 1613 1728 phdr.p_align = ELF_EXEC_PAGESIZE; 1614 1729 ··· 1643 1752 if (!dump_skip(cprm, dataoff - cprm->pos)) 1644 1753 goto end_coredump; 1645 1754 1646 1646 - if (!elf_fdpic_dump_segments(cprm)) 1755 1755 + if (!elf_fdpic_dump_segments(cprm, vma_meta, vma_count)) 1647 1756 goto end_coredump; 1648 1757 1649 1758 if (!elf_core_write_extra_data(cprm)) ··· 1667 1776 thread_list = thread_list->next; 1668 1777 kfree(tmp); 1669 1778 } 1779 1779 + kvfree(vma_meta); 1670 1780 kfree(phdr4note); 1671 1781 kfree(elf); 1672 1782 kfree(psinfo);

+1 -1

fs/configfs/dir.c

reviewed

··· 1168 1168 1169 1169 /* 1170 1170 * Release the dependent linkage. This is much simpler than 1171 1171 - * configfs_depend_item() because we know that that the client driver is 1171 1171 + * configfs_depend_item() because we know that the client driver is 1172 1172 * pinned, thus the subsystem is pinned, and therefore configfs is pinned. 1173 1173 */ 1174 1174 void configfs_undepend_item(struct config_item *target)

+1 -1

fs/configfs/file.c

reviewed

··· 267 267 * There is no easy way for us to know if userspace is only doing a partial 268 268 * write, so we don't support them. We expect the entire buffer to come 269 269 * on the first write. 270 270 - * Hint: if you're writing a value, first read the file, modify only the 270 270 + * Hint: if you're writing a value, first read the file, modify only 271 271 * the value you're changing, then write entire buffer back. 272 272 */ 273 273

+225 -11

fs/coredump.c

reviewed

··· 840 840 ssize_t n; 841 841 if (cprm->written + nr > cprm->limit) 842 842 return 0; 843 843 - while (nr) { 844 844 - if (dump_interrupted()) 845 845 - return 0; 846 846 - n = __kernel_write(file, addr, nr, &pos); 847 847 - if (n <= 0) 848 848 - return 0; 849 849 - file->f_pos = pos; 850 850 - cprm->written += n; 851 851 - cprm->pos += n; 852 852 - nr -= n; 853 853 - } 843 843 + 844 844 + 845 845 + if (dump_interrupted()) 846 846 + return 0; 847 847 + n = __kernel_write(file, addr, nr, &pos); 848 848 + if (n != nr) 849 849 + return 0; 850 850 + file->f_pos = pos; 851 851 + cprm->written += n; 852 852 + cprm->pos += n; 853 853 + 854 854 return 1; 855 855 } 856 856 EXPORT_SYMBOL(dump_emit); ··· 875 875 } 876 876 } 877 877 EXPORT_SYMBOL(dump_skip); 878 878 + 879 879 + #ifdef CONFIG_ELF_CORE 880 880 + int dump_user_range(struct coredump_params *cprm, unsigned long start, 881 881 + unsigned long len) 882 882 + { 883 883 + unsigned long addr; 884 884 + 885 885 + for (addr = start; addr < start + len; addr += PAGE_SIZE) { 886 886 + struct page *page; 887 887 + int stop; 888 888 + 889 889 + /* 890 890 + * To avoid having to allocate page tables for virtual address 891 891 + * ranges that have never been used yet, and also to make it 892 892 + * easy to generate sparse core files, use a helper that returns 893 893 + * NULL when encountering an empty page table entry that would 894 894 + * otherwise have been filled with the zero page. 895 895 + */ 896 896 + page = get_dump_page(addr); 897 897 + if (page) { 898 898 + void *kaddr = kmap(page); 899 899 + 900 900 + stop = !dump_emit(cprm, kaddr, PAGE_SIZE); 901 901 + kunmap(page); 902 902 + put_page(page); 903 903 + } else { 904 904 + stop = !dump_skip(cprm, PAGE_SIZE); 905 905 + } 906 906 + if (stop) 907 907 + return 0; 908 908 + } 909 909 + return 1; 910 910 + } 911 911 + #endif 878 912 879 913 int dump_align(struct coredump_params *cprm, int align) 880 914 { ··· 936 902 } 937 903 } 938 904 EXPORT_SYMBOL(dump_truncate); 905 905 + 906 906 + /* 907 907 + * The purpose of always_dump_vma() is to make sure that special kernel mappings 908 908 + * that are useful for post-mortem analysis are included in every core dump. 909 909 + * In that way we ensure that the core dump is fully interpretable later 910 910 + * without matching up the same kernel and hardware config to see what PC values 911 911 + * meant. These special mappings include - vDSO, vsyscall, and other 912 912 + * architecture specific mappings 913 913 + */ 914 914 + static bool always_dump_vma(struct vm_area_struct *vma) 915 915 + { 916 916 + /* Any vsyscall mappings? */ 917 917 + if (vma == get_gate_vma(vma->vm_mm)) 918 918 + return true; 919 919 + 920 920 + /* 921 921 + * Assume that all vmas with a .name op should always be dumped. 922 922 + * If this changes, a new vm_ops field can easily be added. 923 923 + */ 924 924 + if (vma->vm_ops && vma->vm_ops->name && vma->vm_ops->name(vma)) 925 925 + return true; 926 926 + 927 927 + /* 928 928 + * arch_vma_name() returns non-NULL for special architecture mappings, 929 929 + * such as vDSO sections. 930 930 + */ 931 931 + if (arch_vma_name(vma)) 932 932 + return true; 933 933 + 934 934 + return false; 935 935 + } 936 936 + 937 937 + /* 938 938 + * Decide how much of @vma's contents should be included in a core dump. 939 939 + */ 940 940 + static unsigned long vma_dump_size(struct vm_area_struct *vma, 941 941 + unsigned long mm_flags) 942 942 + { 943 943 + #define FILTER(type) (mm_flags & (1UL << MMF_DUMP_##type)) 944 944 + 945 945 + /* always dump the vdso and vsyscall sections */ 946 946 + if (always_dump_vma(vma)) 947 947 + goto whole; 948 948 + 949 949 + if (vma->vm_flags & VM_DONTDUMP) 950 950 + return 0; 951 951 + 952 952 + /* support for DAX */ 953 953 + if (vma_is_dax(vma)) { 954 954 + if ((vma->vm_flags & VM_SHARED) && FILTER(DAX_SHARED)) 955 955 + goto whole; 956 956 + if (!(vma->vm_flags & VM_SHARED) && FILTER(DAX_PRIVATE)) 957 957 + goto whole; 958 958 + return 0; 959 959 + } 960 960 + 961 961 + /* Hugetlb memory check */ 962 962 + if (is_vm_hugetlb_page(vma)) { 963 963 + if ((vma->vm_flags & VM_SHARED) && FILTER(HUGETLB_SHARED)) 964 964 + goto whole; 965 965 + if (!(vma->vm_flags & VM_SHARED) && FILTER(HUGETLB_PRIVATE)) 966 966 + goto whole; 967 967 + return 0; 968 968 + } 969 969 + 970 970 + /* Do not dump I/O mapped devices or special mappings */ 971 971 + if (vma->vm_flags & VM_IO) 972 972 + return 0; 973 973 + 974 974 + /* By default, dump shared memory if mapped from an anonymous file. */ 975 975 + if (vma->vm_flags & VM_SHARED) { 976 976 + if (file_inode(vma->vm_file)->i_nlink == 0 ? 977 977 + FILTER(ANON_SHARED) : FILTER(MAPPED_SHARED)) 978 978 + goto whole; 979 979 + return 0; 980 980 + } 981 981 + 982 982 + /* Dump segments that have been written to. */ 983 983 + if ((!IS_ENABLED(CONFIG_MMU) || vma->anon_vma) && FILTER(ANON_PRIVATE)) 984 984 + goto whole; 985 985 + if (vma->vm_file == NULL) 986 986 + return 0; 987 987 + 988 988 + if (FILTER(MAPPED_PRIVATE)) 989 989 + goto whole; 990 990 + 991 991 + /* 992 992 + * If this is the beginning of an executable file mapping, 993 993 + * dump the first page to aid in determining what was mapped here. 994 994 + */ 995 995 + if (FILTER(ELF_HEADERS) && 996 996 + vma->vm_pgoff == 0 && (vma->vm_flags & VM_READ) && 997 997 + (READ_ONCE(file_inode(vma->vm_file)->i_mode) & 0111) != 0) 998 998 + return PAGE_SIZE; 999 999 + 1000 1000 + #undef FILTER 1001 1001 + 1002 1002 + return 0; 1003 1003 + 1004 1004 + whole: 1005 1005 + return vma->vm_end - vma->vm_start; 1006 1006 + } 1007 1007 + 1008 1008 + static struct vm_area_struct *first_vma(struct task_struct *tsk, 1009 1009 + struct vm_area_struct *gate_vma) 1010 1010 + { 1011 1011 + struct vm_area_struct *ret = tsk->mm->mmap; 1012 1012 + 1013 1013 + if (ret) 1014 1014 + return ret; 1015 1015 + return gate_vma; 1016 1016 + } 1017 1017 + 1018 1018 + /* 1019 1019 + * Helper function for iterating across a vma list. It ensures that the caller 1020 1020 + * will visit `gate_vma' prior to terminating the search. 1021 1021 + */ 1022 1022 + static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma, 1023 1023 + struct vm_area_struct *gate_vma) 1024 1024 + { 1025 1025 + struct vm_area_struct *ret; 1026 1026 + 1027 1027 + ret = this_vma->vm_next; 1028 1028 + if (ret) 1029 1029 + return ret; 1030 1030 + if (this_vma == gate_vma) 1031 1031 + return NULL; 1032 1032 + return gate_vma; 1033 1033 + } 1034 1034 + 1035 1035 + /* 1036 1036 + * Under the mmap_lock, take a snapshot of relevant information about the task's 1037 1037 + * VMAs. 1038 1038 + */ 1039 1039 + int dump_vma_snapshot(struct coredump_params *cprm, int *vma_count, 1040 1040 + struct core_vma_metadata **vma_meta, 1041 1041 + size_t *vma_data_size_ptr) 1042 1042 + { 1043 1043 + struct vm_area_struct *vma, *gate_vma; 1044 1044 + struct mm_struct *mm = current->mm; 1045 1045 + int i; 1046 1046 + size_t vma_data_size = 0; 1047 1047 + 1048 1048 + /* 1049 1049 + * Once the stack expansion code is fixed to not change VMA bounds 1050 1050 + * under mmap_lock in read mode, this can be changed to take the 1051 1051 + * mmap_lock in read mode. 1052 1052 + */ 1053 1053 + if (mmap_write_lock_killable(mm)) 1054 1054 + return -EINTR; 1055 1055 + 1056 1056 + gate_vma = get_gate_vma(mm); 1057 1057 + *vma_count = mm->map_count + (gate_vma ? 1 : 0); 1058 1058 + 1059 1059 + *vma_meta = kvmalloc_array(*vma_count, sizeof(**vma_meta), GFP_KERNEL); 1060 1060 + if (!*vma_meta) { 1061 1061 + mmap_write_unlock(mm); 1062 1062 + return -ENOMEM; 1063 1063 + } 1064 1064 + 1065 1065 + for (i = 0, vma = first_vma(current, gate_vma); vma != NULL; 1066 1066 + vma = next_vma(vma, gate_vma), i++) { 1067 1067 + struct core_vma_metadata *m = (*vma_meta) + i; 1068 1068 + 1069 1069 + m->start = vma->vm_start; 1070 1070 + m->end = vma->vm_end; 1071 1071 + m->flags = vma->vm_flags; 1072 1072 + m->dump_size = vma_dump_size(vma, cprm->mm_flags); 1073 1073 + 1074 1074 + vma_data_size += m->dump_size; 1075 1075 + } 1076 1076 + 1077 1077 + mmap_write_unlock(mm); 1078 1078 + 1079 1079 + if (WARN_ON(i != *vma_count)) 1080 1080 + return -EFAULT; 1081 1081 + 1082 1082 + *vma_data_size_ptr = vma_data_size; 1083 1083 + return 0; 1084 1084 + }

+2 -2

fs/ext4/verity.c

reviewed

··· 349 349 pgoff_t index, 350 350 unsigned long num_ra_pages) 351 351 { 352 352 + DEFINE_READAHEAD(ractl, NULL, inode->i_mapping, index); 352 353 struct page *page; 353 354 354 355 index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT; ··· 359 358 if (page) 360 359 put_page(page); 361 360 else if (num_ra_pages > 1) 362 362 - page_cache_readahead_unbounded(inode->i_mapping, NULL, 363 363 - index, num_ra_pages, 0); 361 361 + page_cache_ra_unbounded(&ractl, num_ra_pages, 0); 364 362 page = read_mapping_page(inode->i_mapping, index, NULL); 365 363 } 366 364 return page;

+2 -2

fs/f2fs/verity.c

reviewed

··· 228 228 pgoff_t index, 229 229 unsigned long num_ra_pages) 230 230 { 231 231 + DEFINE_READAHEAD(ractl, NULL, inode->i_mapping, index); 231 232 struct page *page; 232 233 233 234 index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT; ··· 238 237 if (page) 239 238 put_page(page); 240 239 else if (num_ra_pages > 1) 241 241 - page_cache_readahead_unbounded(inode->i_mapping, NULL, 242 242 - index, num_ra_pages, 0); 240 240 + page_cache_ra_unbounded(&ractl, num_ra_pages, 0); 243 241 page = read_mapping_page(inode->i_mapping, index, NULL); 244 242 } 245 243 return page;

+2

fs/inode.c

reviewed

··· 181 181 mapping->a_ops = &empty_aops; 182 182 mapping->host = inode; 183 183 mapping->flags = 0; 184 184 + if (sb->s_type->fs_flags & FS_THP_SUPPORT) 185 185 + __set_bit(AS_THP_SUPPORT, &mapping->flags); 184 186 mapping->wb_err = 0; 185 187 atomic_set(&mapping->i_mmap_writable, 0); 186 188 #ifdef CONFIG_READ_ONLY_THP_FOR_FS

+1 -1

fs/nilfs2/bmap.c

reviewed

··· 355 355 /** 356 356 * nilfs_bmap_assign - assign a new block number to a block 357 357 * @bmap: bmap 358 358 - * @bhp: pointer to buffer head 358 358 + * @bh: pointer to buffer head 359 359 * @blocknr: block number 360 360 * @binfo: block information 361 361 *

+3 -3

fs/nilfs2/cpfile.c

reviewed

··· 889 889 * nilfs_cpfile_change_cpmode - change checkpoint mode 890 890 * @cpfile: inode of checkpoint file 891 891 * @cno: checkpoint number 892 892 - * @status: mode of checkpoint 892 892 + * @mode: mode of checkpoint 893 893 * 894 894 * Description: nilfs_change_cpmode() changes the mode of the checkpoint 895 895 * specified by @cno. The mode @mode is NILFS_CHECKPOINT or NILFS_SNAPSHOT. ··· 930 930 /** 931 931 * nilfs_cpfile_get_stat - get checkpoint statistics 932 932 * @cpfile: inode of checkpoint file 933 933 - * @stat: pointer to a structure of checkpoint statistics 933 933 + * @cpstat: pointer to a structure of checkpoint statistics 934 934 * 935 935 * Description: nilfs_cpfile_get_stat() returns information about checkpoints. 936 936 * 937 937 * Return Value: On success, 0 is returned, and checkpoints information is 938 938 - * stored in the place pointed by @stat. On error, one of the following 938 938 + * stored in the place pointed by @cpstat. On error, one of the following 939 939 * negative error codes is returned. 940 940 * 941 941 * %-EIO - I/O error.

-1

fs/nilfs2/page.c

reviewed

··· 69 69 70 70 /** 71 71 * nilfs_forget_buffer - discard dirty state 72 72 - * @inode: owner inode of the buffer 73 72 * @bh: buffer head of the buffer to be discarded 74 73 */ 75 74 void nilfs_forget_buffer(struct buffer_head *bh)

+2 -2

fs/nilfs2/sufile.c

reviewed

··· 546 546 /** 547 547 * nilfs_sufile_get_stat - get segment usage statistics 548 548 * @sufile: inode of segment usage file 549 549 - * @stat: pointer to a structure of segment usage statistics 549 549 + * @sustat: pointer to a structure of segment usage statistics 550 550 * 551 551 * Description: nilfs_sufile_get_stat() returns information about segment 552 552 * usage. 553 553 * 554 554 * Return Value: On success, 0 is returned, and segment usage information is 555 555 - * stored in the place pointed by @stat. On error, one of the following 555 555 + * stored in the place pointed by @sustat. On error, one of the following 556 556 * negative error codes is returned. 557 557 * 558 558 * %-EIO - I/O error.

-18

fs/proc/task_mmu.c

reviewed

··· 1244 1244 count = -EINTR; 1245 1245 goto out_mm; 1246 1246 } 1247 1247 - /* 1248 1248 - * Avoid to modify vma->vm_flags 1249 1249 - * without locked ops while the 1250 1250 - * coredump reads the vm_flags. 1251 1251 - */ 1252 1252 - if (!mmget_still_valid(mm)) { 1253 1253 - /* 1254 1254 - * Silently return "count" 1255 1255 - * like if get_task_mm() 1256 1256 - * failed. FIXME: should this 1257 1257 - * function have returned 1258 1258 - * -ESRCH if get_task_mm() 1259 1259 - * failed like if 1260 1260 - * get_proc_task() fails? 1261 1261 - */ 1262 1262 - mmap_write_unlock(mm); 1263 1263 - goto out_mm; 1264 1264 - } 1265 1247 for (vma = mm->mmap; vma; vma = vma->vm_next) { 1266 1248 vma->vm_flags &= ~VM_SOFTDIRTY; 1267 1249 vma_set_page_prot(vma);

+1 -1

fs/ramfs/file-nommu.c

reviewed

··· 224 224 if (!pages) 225 225 goto out_free; 226 226 227 227 - nr = find_get_pages(inode->i_mapping, &pgoff, lpages, pages); 227 227 + nr = find_get_pages_contig(inode->i_mapping, pgoff, lpages, pages); 228 228 if (nr != lpages) 229 229 goto out_free_pages; /* leave if some pages were missing */ 230 230

+1

fs/romfs/super.c

reviewed

··· 356 356 } 357 357 358 358 i->i_mode = mode; 359 359 + i->i_blocks = (i->i_size + 511) >> 9; 359 360 360 361 unlock_new_inode(i); 361 362 return i;

+9 -19

fs/userfaultfd.c

reviewed

··· 601 601 602 602 /* the various vma->vm_userfaultfd_ctx still points to it */ 603 603 mmap_write_lock(mm); 604 604 - /* no task can run (and in turn coredump) yet */ 605 605 - VM_WARN_ON(!mmget_still_valid(mm)); 606 604 for (vma = mm->mmap; vma; vma = vma->vm_next) 607 605 if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { 608 606 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; ··· 840 842 /* len == 0 means wake all */ 841 843 struct userfaultfd_wake_range range = { .len = 0, }; 842 844 unsigned long new_flags; 843 843 - bool still_valid; 844 845 845 846 WRITE_ONCE(ctx->released, true); 846 847 ··· 855 858 * taking the mmap_lock for writing. 856 859 */ 857 860 mmap_write_lock(mm); 858 858 - still_valid = mmget_still_valid(mm); 859 861 prev = NULL; 860 862 for (vma = mm->mmap; vma; vma = vma->vm_next) { 861 863 cond_resched(); ··· 865 869 continue; 866 870 } 867 871 new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP); 868 868 - if (still_valid) { 869 869 - prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end, 870 870 - new_flags, vma->anon_vma, 871 871 - vma->vm_file, vma->vm_pgoff, 872 872 - vma_policy(vma), 873 873 - NULL_VM_UFFD_CTX); 874 874 - if (prev) 875 875 - vma = prev; 876 876 - else 877 877 - prev = vma; 878 878 - } 872 872 + prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end, 873 873 + new_flags, vma->anon_vma, 874 874 + vma->vm_file, vma->vm_pgoff, 875 875 + vma_policy(vma), 876 876 + NULL_VM_UFFD_CTX); 877 877 + if (prev) 878 878 + vma = prev; 879 879 + else 880 880 + prev = vma; 879 881 vma->vm_flags = new_flags; 880 882 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; 881 883 } ··· 1303 1309 goto out; 1304 1310 1305 1311 mmap_write_lock(mm); 1306 1306 - if (!mmget_still_valid(mm)) 1307 1307 - goto out_unlock; 1308 1312 vma = find_vma_prev(mm, start, &prev); 1309 1313 if (!vma) 1310 1314 goto out_unlock; ··· 1503 1511 goto out; 1504 1512 1505 1513 mmap_write_lock(mm); 1506 1506 - if (!mmget_still_valid(mm)) 1507 1507 - goto out_unlock; 1508 1514 vma = find_vma_prev(mm, start, &prev); 1509 1515 if (!vma) 1510 1516 goto out_unlock;

+4 -9

include/linux/bitops.h

reviewed

··· 188 188 189 189 static inline int get_count_order(unsigned int count) 190 190 { 191 191 - int order; 191 191 + if (count == 0) 192 192 + return -1; 192 193 193 193 - order = fls(count) - 1; 194 194 - if (count & (count - 1)) 195 195 - order++; 196 196 - return order; 194 194 + return fls(--count); 197 195 } 198 196 199 197 /** ··· 204 206 { 205 207 if (l == 0UL) 206 208 return -1; 207 207 - else if (l & (l - 1UL)) 208 208 - return (int)fls_long(l); 209 209 - else 210 210 - return (int)fls_long(l) - 1; 209 209 + return (int)fls_long(--l); 211 210 } 212 211 213 212 /**

+1

include/linux/blkdev.h

reviewed

··· 8 8 #include <linux/genhd.h> 9 9 #include <linux/list.h> 10 10 #include <linux/llist.h> 11 11 + #include <linux/minmax.h> 11 12 #include <linux/timer.h> 12 13 #include <linux/workqueue.h> 13 14 #include <linux/pagemap.h>

+5 -1

include/linux/bvec.h

reviewed

··· 7 7 #ifndef __LINUX_BVEC_ITER_H 8 8 #define __LINUX_BVEC_ITER_H 9 9 10 10 - #include <linux/kernel.h> 11 10 #include <linux/bug.h> 12 11 #include <linux/errno.h> 12 12 + #include <linux/limits.h> 13 13 + #include <linux/minmax.h> 13 14 #include <linux/mm.h> 15 15 + #include <linux/types.h> 16 16 + 17 17 + struct page; 14 18 15 19 /** 16 20 * struct bio_vec - a contiguous range of physical memory addresses

+11

include/linux/coredump.h

reviewed

··· 7 7 #include <linux/fs.h> 8 8 #include <asm/siginfo.h> 9 9 10 10 + struct core_vma_metadata { 11 11 + unsigned long start, end; 12 12 + unsigned long flags; 13 13 + unsigned long dump_size; 14 14 + }; 15 15 + 10 16 /* 11 17 * These are the only things you should do on a core-file: use only these 12 18 * functions to write out all the necessary info. ··· 22 16 extern int dump_emit(struct coredump_params *cprm, const void *addr, int nr); 23 17 extern int dump_align(struct coredump_params *cprm, int align); 24 18 extern void dump_truncate(struct coredump_params *cprm); 19 19 + int dump_user_range(struct coredump_params *cprm, unsigned long start, 20 20 + unsigned long len); 21 21 + int dump_vma_snapshot(struct coredump_params *cprm, int *vma_count, 22 22 + struct core_vma_metadata **vma_meta, 23 23 + size_t *vma_data_size_ptr); 25 24 #ifdef CONFIG_COREDUMP 26 25 extern void do_coredump(const kernel_siginfo_t *siginfo); 27 26 #else

+22

include/linux/fault-inject-usercopy.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 + #ifndef __LINUX_FAULT_INJECT_USERCOPY_H__ 3 3 + #define __LINUX_FAULT_INJECT_USERCOPY_H__ 4 4 + 5 5 + /* 6 6 + * This header provides a wrapper for injecting failures to user space memory 7 7 + * access functions. 8 8 + */ 9 9 + 10 10 + #include <linux/types.h> 11 11 + 12 12 + #ifdef CONFIG_FAULT_INJECTION_USERCOPY 13 13 + 14 14 + bool should_fail_usercopy(void); 15 15 + 16 16 + #else 17 17 + 18 18 + static inline bool should_fail_usercopy(void) { return false; } 19 19 + 20 20 + #endif /* CONFIG_FAULT_INJECTION_USERCOPY */ 21 21 + 22 22 + #endif /* __LINUX_FAULT_INJECT_USERCOPY_H__ */

+1 -27

include/linux/fs.h

reviewed

··· 2209 2209 #define FS_HAS_SUBTYPE 4 2210 2210 #define FS_USERNS_MOUNT 8 /* Can be mounted by userns root */ 2211 2211 #define FS_DISALLOW_NOTIFY_PERM 16 /* Disable fanotify permission events */ 2212 2212 + #define FS_THP_SUPPORT 8192 /* Remove once all fs converted */ 2212 2213 #define FS_RENAME_DOES_D_MOVE 32768 /* FS will handle d_move() during rename() internally. */ 2213 2214 int (*init_fs_context)(struct fs_context *); 2214 2215 const struct fs_parameter_spec *parameters; ··· 2695 2694 static inline errseq_t file_sample_sb_err(struct file *file) 2696 2695 { 2697 2696 return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err); 2698 2698 - } 2699 2699 - 2700 2700 - static inline int filemap_nr_thps(struct address_space *mapping) 2701 2701 - { 2702 2702 - #ifdef CONFIG_READ_ONLY_THP_FOR_FS 2703 2703 - return atomic_read(&mapping->nr_thps); 2704 2704 - #else 2705 2705 - return 0; 2706 2706 - #endif 2707 2707 - } 2708 2708 - 2709 2709 - static inline void filemap_nr_thps_inc(struct address_space *mapping) 2710 2710 - { 2711 2711 - #ifdef CONFIG_READ_ONLY_THP_FOR_FS 2712 2712 - atomic_inc(&mapping->nr_thps); 2713 2713 - #else 2714 2714 - WARN_ON_ONCE(1); 2715 2715 - #endif 2716 2716 - } 2717 2717 - 2718 2718 - static inline void filemap_nr_thps_dec(struct address_space *mapping) 2719 2719 - { 2720 2720 - #ifdef CONFIG_READ_ONLY_THP_FOR_FS 2721 2721 - atomic_dec(&mapping->nr_thps); 2722 2722 - #else 2723 2723 - WARN_ON_ONCE(1); 2724 2724 - #endif 2725 2697 } 2726 2698 2727 2699 extern int vfs_fsync_range(struct file *file, loff_t start, loff_t end,

+10 -3

include/linux/idr.h

reviewed

··· 263 263 * 264 264 * Allocate an ID between 0 and %INT_MAX, inclusive. 265 265 * 266 266 - * Context: Any context. 266 266 + * Context: Any context. It is safe to call this function without 267 267 + * locking in your code. 267 268 * Return: The allocated ID, or %-ENOMEM if memory could not be allocated, 268 269 * or %-ENOSPC if there are no free IDs. 269 270 */ ··· 281 280 * 282 281 * Allocate an ID between @min and %INT_MAX, inclusive. 283 282 * 284 284 - * Context: Any context. 283 283 + * Context: Any context. It is safe to call this function without 284 284 + * locking in your code. 285 285 * Return: The allocated ID, or %-ENOMEM if memory could not be allocated, 286 286 * or %-ENOSPC if there are no free IDs. 287 287 */ ··· 299 297 * 300 298 * Allocate an ID between 0 and @max, inclusive. 301 299 * 302 302 - * Context: Any context. 300 300 + * Context: Any context. It is safe to call this function without 301 301 + * locking in your code. 303 302 * Return: The allocated ID, or %-ENOMEM if memory could not be allocated, 304 303 * or %-ENOSPC if there are no free IDs. 305 304 */ ··· 314 311 xa_init_flags(&ida->xa, IDA_INIT_FLAGS); 315 312 } 316 313 314 314 + /* 315 315 + * ida_simple_get() and ida_simple_remove() are deprecated. Use 316 316 + * ida_alloc() and ida_free() instead respectively. 317 317 + */ 317 318 #define ida_simple_get(ida, start, end, gfp) \ 318 319 ida_alloc_range(ida, start, (end) - 1, gfp) 319 320 #define ida_simple_remove(ida, id) ida_free(ida, id)

+8 -3

include/linux/ioport.h

reviewed

··· 58 58 #define IORESOURCE_EXT_TYPE_BITS 0x01000000 /* Resource extended types */ 59 59 #define IORESOURCE_SYSRAM 0x01000000 /* System RAM (modifier) */ 60 60 61 61 + /* IORESOURCE_SYSRAM specific bits. */ 62 62 + #define IORESOURCE_SYSRAM_DRIVER_MANAGED 0x02000000 /* Always detected via a driver. */ 63 63 + #define IORESOURCE_SYSRAM_MERGEABLE 0x04000000 /* Resource can be merged. */ 64 64 + 61 65 #define IORESOURCE_EXCLUSIVE 0x08000000 /* Userland may not map this resource */ 62 66 63 67 #define IORESOURCE_DISABLED 0x10000000 ··· 107 103 #define IORESOURCE_MEM_32BIT (3<<3) 108 104 #define IORESOURCE_MEM_SHADOWABLE (1<<5) /* dup: IORESOURCE_SHADOWABLE */ 109 105 #define IORESOURCE_MEM_EXPANSIONROM (1<<6) 110 110 - #define IORESOURCE_MEM_DRIVER_MANAGED (1<<7) 111 106 112 107 /* PnP I/O specific bits (IORESOURCE_BITS) */ 113 108 #define IORESOURCE_IO_16BIT_ADDR (1<<0) ··· 251 248 extern void __release_region(struct resource *, resource_size_t, 252 249 resource_size_t); 253 250 #ifdef CONFIG_MEMORY_HOTREMOVE 254 254 - extern int release_mem_region_adjustable(struct resource *, resource_size_t, 255 255 - resource_size_t); 251 251 + extern void release_mem_region_adjustable(resource_size_t, resource_size_t); 252 252 + #endif 253 253 + #ifdef CONFIG_MEMORY_HOTPLUG 254 254 + extern void merge_system_ram_resource(struct resource *res); 256 255 #endif 257 256 258 257 /* Wrappers for managed devices */

+2 -1

include/linux/jiffies.h

reviewed

··· 3 3 #define _LINUX_JIFFIES_H 4 4 5 5 #include <linux/cache.h> 6 6 + #include <linux/limits.h> 6 7 #include <linux/math64.h> 7 7 - #include <linux/kernel.h> 8 8 + #include <linux/minmax.h> 8 9 #include <linux/types.h> 9 10 #include <linux/time.h> 10 11 #include <linux/timex.h>

+1 -149

include/linux/kernel.h

reviewed

··· 11 11 #include <linux/compiler.h> 12 12 #include <linux/bitops.h> 13 13 #include <linux/log2.h> 14 14 + #include <linux/minmax.h> 14 15 #include <linux/typecheck.h> 15 16 #include <linux/printk.h> 16 17 #include <linux/build_bug.h> ··· 833 832 } 834 833 static inline void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) { } 835 834 #endif /* CONFIG_TRACING */ 836 836 - 837 837 - /* 838 838 - * min()/max()/clamp() macros must accomplish three things: 839 839 - * 840 840 - * - avoid multiple evaluations of the arguments (so side-effects like 841 841 - * "x++" happen only once) when non-constant. 842 842 - * - perform strict type-checking (to generate warnings instead of 843 843 - * nasty runtime surprises). See the "unnecessary" pointer comparison 844 844 - * in __typecheck(). 845 845 - * - retain result as a constant expressions when called with only 846 846 - * constant expressions (to avoid tripping VLA warnings in stack 847 847 - * allocation usage). 848 848 - */ 849 849 - #define __typecheck(x, y) \ 850 850 - (!!(sizeof((typeof(x) *)1 == (typeof(y) *)1))) 851 851 - 852 852 - /* 853 853 - * This returns a constant expression while determining if an argument is 854 854 - * a constant expression, most importantly without evaluating the argument. 855 855 - * Glory to Martin Uecker <Martin.Uecker@med.uni-goettingen.de> 856 856 - */ 857 857 - #define __is_constexpr(x) \ 858 858 - (sizeof(int) == sizeof(*(8 ? ((void *)((long)(x) * 0l)) : (int *)8))) 859 859 - 860 860 - #define __no_side_effects(x, y) \ 861 861 - (__is_constexpr(x) && __is_constexpr(y)) 862 862 - 863 863 - #define __safe_cmp(x, y) \ 864 864 - (__typecheck(x, y) && __no_side_effects(x, y)) 865 865 - 866 866 - #define __cmp(x, y, op) ((x) op (y) ? (x) : (y)) 867 867 - 868 868 - #define __cmp_once(x, y, unique_x, unique_y, op) ({ \ 869 869 - typeof(x) unique_x = (x); \ 870 870 - typeof(y) unique_y = (y); \ 871 871 - __cmp(unique_x, unique_y, op); }) 872 872 - 873 873 - #define __careful_cmp(x, y, op) \ 874 874 - __builtin_choose_expr(__safe_cmp(x, y), \ 875 875 - __cmp(x, y, op), \ 876 876 - __cmp_once(x, y, __UNIQUE_ID(__x), __UNIQUE_ID(__y), op)) 877 877 - 878 878 - /** 879 879 - * min - return minimum of two values of the same or compatible types 880 880 - * @x: first value 881 881 - * @y: second value 882 882 - */ 883 883 - #define min(x, y) __careful_cmp(x, y, <) 884 884 - 885 885 - /** 886 886 - * max - return maximum of two values of the same or compatible types 887 887 - * @x: first value 888 888 - * @y: second value 889 889 - */ 890 890 - #define max(x, y) __careful_cmp(x, y, >) 891 891 - 892 892 - /** 893 893 - * min3 - return minimum of three values 894 894 - * @x: first value 895 895 - * @y: second value 896 896 - * @z: third value 897 897 - */ 898 898 - #define min3(x, y, z) min((typeof(x))min(x, y), z) 899 899 - 900 900 - /** 901 901 - * max3 - return maximum of three values 902 902 - * @x: first value 903 903 - * @y: second value 904 904 - * @z: third value 905 905 - */ 906 906 - #define max3(x, y, z) max((typeof(x))max(x, y), z) 907 907 - 908 908 - /** 909 909 - * min_not_zero - return the minimum that is _not_ zero, unless both are zero 910 910 - * @x: value1 911 911 - * @y: value2 912 912 - */ 913 913 - #define min_not_zero(x, y) ({ \ 914 914 - typeof(x) __x = (x); \ 915 915 - typeof(y) __y = (y); \ 916 916 - __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); }) 917 917 - 918 918 - /** 919 919 - * clamp - return a value clamped to a given range with strict typechecking 920 920 - * @val: current value 921 921 - * @lo: lowest allowable value 922 922 - * @hi: highest allowable value 923 923 - * 924 924 - * This macro does strict typechecking of @lo/@hi to make sure they are of the 925 925 - * same type as @val. See the unnecessary pointer comparisons. 926 926 - */ 927 927 - #define clamp(val, lo, hi) min((typeof(val))max(val, lo), hi) 928 928 - 929 929 - /* 930 930 - * ..and if you can't take the strict 931 931 - * types, you can specify one yourself. 932 932 - * 933 933 - * Or not use min/max/clamp at all, of course. 934 934 - */ 935 935 - 936 936 - /** 937 937 - * min_t - return minimum of two values, using the specified type 938 938 - * @type: data type to use 939 939 - * @x: first value 940 940 - * @y: second value 941 941 - */ 942 942 - #define min_t(type, x, y) __careful_cmp((type)(x), (type)(y), <) 943 943 - 944 944 - /** 945 945 - * max_t - return maximum of two values, using the specified type 946 946 - * @type: data type to use 947 947 - * @x: first value 948 948 - * @y: second value 949 949 - */ 950 950 - #define max_t(type, x, y) __careful_cmp((type)(x), (type)(y), >) 951 951 - 952 952 - /** 953 953 - * clamp_t - return a value clamped to a given range using a given type 954 954 - * @type: the type of variable to use 955 955 - * @val: current value 956 956 - * @lo: minimum allowable value 957 957 - * @hi: maximum allowable value 958 958 - * 959 959 - * This macro does no typechecking and uses temporary variables of type 960 960 - * @type to make all the comparisons. 961 961 - */ 962 962 - #define clamp_t(type, val, lo, hi) min_t(type, max_t(type, val, lo), hi) 963 963 - 964 964 - /** 965 965 - * clamp_val - return a value clamped to a given range using val's type 966 966 - * @val: current value 967 967 - * @lo: minimum allowable value 968 968 - * @hi: maximum allowable value 969 969 - * 970 970 - * This macro does no typechecking and uses temporary variables of whatever 971 971 - * type the input argument @val is. This is useful when @val is an unsigned 972 972 - * type and @lo and @hi are literals that will otherwise be assigned a signed 973 973 - * integer type. 974 974 - */ 975 975 - #define clamp_val(val, lo, hi) clamp_t(typeof(val), val, lo, hi) 976 976 - 977 977 - 978 978 - /** 979 979 - * swap - swap values of @a and @b 980 980 - * @a: first value 981 981 - * @b: second value 982 982 - */ 983 983 - #define swap(a, b) \ 984 984 - do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0) 985 835 986 836 /* This counts to 12. Any more, it will return 13th argument. */ 987 837 #define __COUNT_ARGS(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _n, X...) _n

+19 -10

include/linux/list.h

reviewed

··· 610 610 pos = n, n = pos->prev) 611 611 612 612 /** 613 613 + * list_entry_is_head - test if the entry points to the head of the list 614 614 + * @pos: the type * to cursor 615 615 + * @head: the head for your list. 616 616 + * @member: the name of the list_head within the struct. 617 617 + */ 618 618 + #define list_entry_is_head(pos, head, member) \ 619 619 + (&pos->member == (head)) 620 620 + 621 621 + /** 613 622 * list_for_each_entry - iterate over list of given type 614 623 * @pos: the type * to use as a loop cursor. 615 624 * @head: the head for your list. ··· 626 617 */ 627 618 #define list_for_each_entry(pos, head, member) \ 628 619 for (pos = list_first_entry(head, typeof(*pos), member); \ 629 629 - &pos->member != (head); \ 620 620 + !list_entry_is_head(pos, head, member); \ 630 621 pos = list_next_entry(pos, member)) 631 622 632 623 /** ··· 637 628 */ 638 629 #define list_for_each_entry_reverse(pos, head, member) \ 639 630 for (pos = list_last_entry(head, typeof(*pos), member); \ 640 640 - &pos->member != (head); \ 631 631 + !list_entry_is_head(pos, head, member); \ 641 632 pos = list_prev_entry(pos, member)) 642 633 643 634 /** ··· 662 653 */ 663 654 #define list_for_each_entry_continue(pos, head, member) \ 664 655 for (pos = list_next_entry(pos, member); \ 665 665 - &pos->member != (head); \ 656 656 + !list_entry_is_head(pos, head, member); \ 666 657 pos = list_next_entry(pos, member)) 667 658 668 659 /** ··· 676 667 */ 677 668 #define list_for_each_entry_continue_reverse(pos, head, member) \ 678 669 for (pos = list_prev_entry(pos, member); \ 679 679 - &pos->member != (head); \ 670 670 + !list_entry_is_head(pos, head, member); \ 680 671 pos = list_prev_entry(pos, member)) 681 672 682 673 /** ··· 688 679 * Iterate over list of given type, continuing from current position. 689 680 */ 690 681 #define list_for_each_entry_from(pos, head, member) \ 691 691 - for (; &pos->member != (head); \ 682 682 + for (; !list_entry_is_head(pos, head, member); \ 692 683 pos = list_next_entry(pos, member)) 693 684 694 685 /** ··· 701 692 * Iterate backwards over list of given type, continuing from current position. 702 693 */ 703 694 #define list_for_each_entry_from_reverse(pos, head, member) \ 704 704 - for (; &pos->member != (head); \ 695 695 + for (; !list_entry_is_head(pos, head, member); \ 705 696 pos = list_prev_entry(pos, member)) 706 697 707 698 /** ··· 714 705 #define list_for_each_entry_safe(pos, n, head, member) \ 715 706 for (pos = list_first_entry(head, typeof(*pos), member), \ 716 707 n = list_next_entry(pos, member); \ 717 717 - &pos->member != (head); \ 708 708 + !list_entry_is_head(pos, head, member); \ 718 709 pos = n, n = list_next_entry(n, member)) 719 710 720 711 /** ··· 730 721 #define list_for_each_entry_safe_continue(pos, n, head, member) \ 731 722 for (pos = list_next_entry(pos, member), \ 732 723 n = list_next_entry(pos, member); \ 733 733 - &pos->member != (head); \ 724 724 + !list_entry_is_head(pos, head, member); \ 734 725 pos = n, n = list_next_entry(n, member)) 735 726 736 727 /** ··· 745 736 */ 746 737 #define list_for_each_entry_safe_from(pos, n, head, member) \ 747 738 for (n = list_next_entry(pos, member); \ 748 748 - &pos->member != (head); \ 739 739 + !list_entry_is_head(pos, head, member); \ 749 740 pos = n, n = list_next_entry(n, member)) 750 741 751 742 /** ··· 761 752 #define list_for_each_entry_safe_reverse(pos, n, head, member) \ 762 753 for (pos = list_last_entry(head, typeof(*pos), member), \ 763 754 n = list_prev_entry(pos, member); \ 764 764 - &pos->member != (head); \ 755 755 + !list_entry_is_head(pos, head, member); \ 765 756 pos = n, n = list_prev_entry(n, member)) 766 757 767 758 /**

+26 -16

include/linux/memory_hotplug.h

reviewed

··· 57 57 MMOP_ONLINE_MOVABLE, 58 58 }; 59 59 60 60 + /* Flags for add_memory() and friends to specify memory hotplug details. */ 61 61 + typedef int __bitwise mhp_t; 62 62 + 63 63 + /* No special request */ 64 64 + #define MHP_NONE ((__force mhp_t)0) 65 65 + /* 66 66 + * Allow merging of the added System RAM resource with adjacent, 67 67 + * mergeable resources. After a successful call to add_memory_resource() 68 68 + * with this flag set, the resource pointer must no longer be used as it 69 69 + * might be stale, or the resource might have changed. 70 70 + */ 71 71 + #define MEMHP_MERGE_RESOURCE ((__force mhp_t)BIT(0)) 72 72 + 60 73 /* 61 74 * Extended parameters for memory hotplug: 62 75 * altmap: alternative allocator for memmap array (optional) ··· 116 103 int online_type, int nid); 117 104 extern struct zone *test_pages_in_a_zone(unsigned long start_pfn, 118 105 unsigned long end_pfn); 119 119 - extern unsigned long __offline_isolated_pages(unsigned long start_pfn, 120 120 - unsigned long end_pfn); 106 106 + extern void __offline_isolated_pages(unsigned long start_pfn, 107 107 + unsigned long end_pfn); 121 108 122 109 typedef void (*online_page_callback_t)(struct page *page, unsigned int order); 123 110 ··· 260 247 static inline void zone_span_writeunlock(struct zone *zone) {} 261 248 static inline void zone_seqlock_init(struct zone *zone) {} 262 249 263 263 - static inline int mhp_notimplemented(const char *func) 264 264 - { 265 265 - printk(KERN_WARNING "%s() called, with CONFIG_MEMORY_HOTPLUG disabled\n", func); 266 266 - dump_stack(); 267 267 - return -ENOSYS; 268 268 - } 269 269 - 270 250 static inline void register_page_bootmem_info_node(struct pglist_data *pgdat) 271 251 { 272 252 } ··· 350 344 extern void set_zone_contiguous(struct zone *zone); 351 345 extern void clear_zone_contiguous(struct zone *zone); 352 346 347 347 + #ifdef CONFIG_MEMORY_HOTPLUG 353 348 extern void __ref free_area_init_core_hotplug(int nid); 354 354 - extern int __add_memory(int nid, u64 start, u64 size); 355 355 - extern int add_memory(int nid, u64 start, u64 size); 356 356 - extern int add_memory_resource(int nid, struct resource *resource); 349 349 + extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); 350 350 + extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); 351 351 + extern int add_memory_resource(int nid, struct resource *resource, 352 352 + mhp_t mhp_flags); 357 353 extern int add_memory_driver_managed(int nid, u64 start, u64 size, 358 358 - const char *resource_name); 354 354 + const char *resource_name, 355 355 + mhp_t mhp_flags); 359 356 extern void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, 360 360 - unsigned long nr_pages, struct vmem_altmap *altmap); 357 357 + unsigned long nr_pages, 358 358 + struct vmem_altmap *altmap, int migratetype); 361 359 extern void remove_pfn_range_from_zone(struct zone *zone, 362 360 unsigned long start_pfn, 363 361 unsigned long nr_pages); ··· 373 363 unsigned long map_offset, struct vmem_altmap *altmap); 374 364 extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map, 375 365 unsigned long pnum); 376 376 - extern bool allow_online_pfn_range(int nid, unsigned long pfn, unsigned long nr_pages, 377 377 - int online_type); 378 366 extern struct zone *zone_for_pfn_range(int online_type, int nid, unsigned start_pfn, 379 367 unsigned long nr_pages); 368 368 + #endif /* CONFIG_MEMORY_HOTPLUG */ 369 369 + 380 370 #endif /* __LINUX_MEMORY_HOTPLUG_H */

+153

include/linux/minmax.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 + #ifndef _LINUX_MINMAX_H 3 3 + #define _LINUX_MINMAX_H 4 4 + 5 5 + /* 6 6 + * min()/max()/clamp() macros must accomplish three things: 7 7 + * 8 8 + * - avoid multiple evaluations of the arguments (so side-effects like 9 9 + * "x++" happen only once) when non-constant. 10 10 + * - perform strict type-checking (to generate warnings instead of 11 11 + * nasty runtime surprises). See the "unnecessary" pointer comparison 12 12 + * in __typecheck(). 13 13 + * - retain result as a constant expressions when called with only 14 14 + * constant expressions (to avoid tripping VLA warnings in stack 15 15 + * allocation usage). 16 16 + */ 17 17 + #define __typecheck(x, y) \ 18 18 + (!!(sizeof((typeof(x) *)1 == (typeof(y) *)1))) 19 19 + 20 20 + /* 21 21 + * This returns a constant expression while determining if an argument is 22 22 + * a constant expression, most importantly without evaluating the argument. 23 23 + * Glory to Martin Uecker <Martin.Uecker@med.uni-goettingen.de> 24 24 + */ 25 25 + #define __is_constexpr(x) \ 26 26 + (sizeof(int) == sizeof(*(8 ? ((void *)((long)(x) * 0l)) : (int *)8))) 27 27 + 28 28 + #define __no_side_effects(x, y) \ 29 29 + (__is_constexpr(x) && __is_constexpr(y)) 30 30 + 31 31 + #define __safe_cmp(x, y) \ 32 32 + (__typecheck(x, y) && __no_side_effects(x, y)) 33 33 + 34 34 + #define __cmp(x, y, op) ((x) op (y) ? (x) : (y)) 35 35 + 36 36 + #define __cmp_once(x, y, unique_x, unique_y, op) ({ \ 37 37 + typeof(x) unique_x = (x); \ 38 38 + typeof(y) unique_y = (y); \ 39 39 + __cmp(unique_x, unique_y, op); }) 40 40 + 41 41 + #define __careful_cmp(x, y, op) \ 42 42 + __builtin_choose_expr(__safe_cmp(x, y), \ 43 43 + __cmp(x, y, op), \ 44 44 + __cmp_once(x, y, __UNIQUE_ID(__x), __UNIQUE_ID(__y), op)) 45 45 + 46 46 + /** 47 47 + * min - return minimum of two values of the same or compatible types 48 48 + * @x: first value 49 49 + * @y: second value 50 50 + */ 51 51 + #define min(x, y) __careful_cmp(x, y, <) 52 52 + 53 53 + /** 54 54 + * max - return maximum of two values of the same or compatible types 55 55 + * @x: first value 56 56 + * @y: second value 57 57 + */ 58 58 + #define max(x, y) __careful_cmp(x, y, >) 59 59 + 60 60 + /** 61 61 + * min3 - return minimum of three values 62 62 + * @x: first value 63 63 + * @y: second value 64 64 + * @z: third value 65 65 + */ 66 66 + #define min3(x, y, z) min((typeof(x))min(x, y), z) 67 67 + 68 68 + /** 69 69 + * max3 - return maximum of three values 70 70 + * @x: first value 71 71 + * @y: second value 72 72 + * @z: third value 73 73 + */ 74 74 + #define max3(x, y, z) max((typeof(x))max(x, y), z) 75 75 + 76 76 + /** 77 77 + * min_not_zero - return the minimum that is _not_ zero, unless both are zero 78 78 + * @x: value1 79 79 + * @y: value2 80 80 + */ 81 81 + #define min_not_zero(x, y) ({ \ 82 82 + typeof(x) __x = (x); \ 83 83 + typeof(y) __y = (y); \ 84 84 + __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); }) 85 85 + 86 86 + /** 87 87 + * clamp - return a value clamped to a given range with strict typechecking 88 88 + * @val: current value 89 89 + * @lo: lowest allowable value 90 90 + * @hi: highest allowable value 91 91 + * 92 92 + * This macro does strict typechecking of @lo/@hi to make sure they are of the 93 93 + * same type as @val. See the unnecessary pointer comparisons. 94 94 + */ 95 95 + #define clamp(val, lo, hi) min((typeof(val))max(val, lo), hi) 96 96 + 97 97 + /* 98 98 + * ..and if you can't take the strict 99 99 + * types, you can specify one yourself. 100 100 + * 101 101 + * Or not use min/max/clamp at all, of course. 102 102 + */ 103 103 + 104 104 + /** 105 105 + * min_t - return minimum of two values, using the specified type 106 106 + * @type: data type to use 107 107 + * @x: first value 108 108 + * @y: second value 109 109 + */ 110 110 + #define min_t(type, x, y) __careful_cmp((type)(x), (type)(y), <) 111 111 + 112 112 + /** 113 113 + * max_t - return maximum of two values, using the specified type 114 114 + * @type: data type to use 115 115 + * @x: first value 116 116 + * @y: second value 117 117 + */ 118 118 + #define max_t(type, x, y) __careful_cmp((type)(x), (type)(y), >) 119 119 + 120 120 + /** 121 121 + * clamp_t - return a value clamped to a given range using a given type 122 122 + * @type: the type of variable to use 123 123 + * @val: current value 124 124 + * @lo: minimum allowable value 125 125 + * @hi: maximum allowable value 126 126 + * 127 127 + * This macro does no typechecking and uses temporary variables of type 128 128 + * @type to make all the comparisons. 129 129 + */ 130 130 + #define clamp_t(type, val, lo, hi) min_t(type, max_t(type, val, lo), hi) 131 131 + 132 132 + /** 133 133 + * clamp_val - return a value clamped to a given range using val's type 134 134 + * @val: current value 135 135 + * @lo: minimum allowable value 136 136 + * @hi: maximum allowable value 137 137 + * 138 138 + * This macro does no typechecking and uses temporary variables of whatever 139 139 + * type the input argument @val is. This is useful when @val is an unsigned 140 140 + * type and @lo and @hi are literals that will otherwise be assigned a signed 141 141 + * integer type. 142 142 + */ 143 143 + #define clamp_val(val, lo, hi) clamp_t(typeof(val), val, lo, hi) 144 144 + 145 145 + /** 146 146 + * swap - swap values of @a and @b 147 147 + * @a: first value 148 148 + * @b: second value 149 149 + */ 150 150 + #define swap(a, b) \ 151 151 + do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0) 152 152 + 153 153 + #endif /* _LINUX_MINMAX_H */

+2 -3

include/linux/mm.h

reviewed

··· 2440 2440 2441 2441 extern void set_dma_reserve(unsigned long new_dma_reserve); 2442 2442 extern void memmap_init_zone(unsigned long, int, unsigned long, unsigned long, 2443 2443 - enum meminit_context, struct vmem_altmap *); 2443 2443 + enum meminit_context, struct vmem_altmap *, int migratetype); 2444 2444 extern void setup_per_zone_wmarks(void); 2445 2445 extern int __meminit init_per_zone_wmark_min(void); 2446 2446 extern void mem_init(void); ··· 3025 3025 extern void memory_failure_queue(unsigned long pfn, int flags); 3026 3026 extern void memory_failure_queue_kick(int cpu); 3027 3027 extern int unpoison_memory(unsigned long pfn); 3028 3028 - extern int get_hwpoison_page(struct page *page); 3029 3029 - #define put_hwpoison_page(page) put_page(page) 3030 3028 extern int sysctl_memory_failure_early_kill; 3031 3029 extern int sysctl_memory_failure_recovery; 3032 3030 extern void shake_page(struct page *p, int access); ··· 3064 3066 MF_MSG_BUDDY, 3065 3067 MF_MSG_BUDDY_2ND, 3066 3068 MF_MSG_DAX, 3069 3069 + MF_MSG_UNSPLIT_THP, 3067 3070 MF_MSG_UNKNOWN, 3068 3071 }; 3069 3072

+8 -9

include/linux/mmzone.h

reviewed

··· 266 266 return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE); 267 267 } 268 268 269 269 + #define ANON_AND_FILE 2 270 270 + 269 271 enum lruvec_flags { 270 272 LRUVEC_CONGESTED, /* lruvec has many dirty pages 271 273 * backed by a congested BDI ··· 285 283 unsigned long file_cost; 286 284 /* Non-resident age, driven by LRU movement */ 287 285 atomic_long_t nonresident_age; 288 288 - /* Refaults at the time of last reclaim cycle, anon=0, file=1 */ 289 289 - unsigned long refaults[2]; 286 286 + /* Refaults at the time of last reclaim cycle */ 287 287 + unsigned long refaults[ANON_AND_FILE]; 290 288 /* Various lruvec state flags (enum lruvec_flags) */ 291 289 unsigned long flags; 292 290 #ifdef CONFIG_MEMCG ··· 443 441 444 442 #ifndef __GENERATING_BOUNDS_H 445 443 444 444 + #define ASYNC_AND_SYNC 2 445 445 + 446 446 struct zone { 447 447 /* Read-mostly fields */ 448 448 ··· 564 560 #if defined CONFIG_COMPACTION || defined CONFIG_CMA 565 561 /* pfn where compaction free scanner should start */ 566 562 unsigned long compact_cached_free_pfn; 567 567 - /* pfn where async and sync compaction migration scanner should start */ 568 568 - unsigned long compact_cached_migrate_pfn[2]; 563 563 + /* pfn where compaction migration scanner should start */ 564 564 + unsigned long compact_cached_migrate_pfn[ASYNC_AND_SYNC]; 569 565 unsigned long compact_init_migrate_pfn; 570 566 unsigned long compact_init_free_pfn; 571 567 #endif ··· 1420 1416 #define pfn_to_nid(pfn) (0) 1421 1417 #endif 1422 1418 1423 1423 - #define early_pfn_valid(pfn) pfn_valid(pfn) 1424 1419 void sparse_init(void); 1425 1420 #else 1426 1421 #define sparse_init() do {} while (0) ··· 1438 1435 unsigned long last_end; 1439 1436 int last_nid; 1440 1437 }; 1441 1441 - 1442 1442 - #ifndef early_pfn_valid 1443 1443 - #define early_pfn_valid(pfn) (1) 1444 1444 - #endif 1445 1438 1446 1439 /* 1447 1440 * If it is possible to have holes within a MAX_ORDER_NR_PAGES, then we

+7 -9

include/linux/node.h

reviewed

··· 99 99 typedef void (*node_registration_func_t)(struct node *); 100 100 101 101 #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) && defined(CONFIG_NUMA) 102 102 - int link_mem_sections(int nid, unsigned long start_pfn, 103 103 - unsigned long end_pfn, 104 104 - enum meminit_context context); 102 102 + void link_mem_sections(int nid, unsigned long start_pfn, 103 103 + unsigned long end_pfn, 104 104 + enum meminit_context context); 105 105 #else 106 106 - static inline int link_mem_sections(int nid, unsigned long start_pfn, 107 107 - unsigned long end_pfn, 108 108 - enum meminit_context context) 106 106 + static inline void link_mem_sections(int nid, unsigned long start_pfn, 107 107 + unsigned long end_pfn, 108 108 + enum meminit_context context) 109 109 { 110 110 - return 0; 111 110 } 112 111 #endif 113 112 ··· 129 130 if (error) 130 131 return error; 131 132 /* link memory sections under this node */ 132 132 - error = link_mem_sections(nid, start_pfn, end_pfn, 133 133 - MEMINIT_EARLY); 133 133 + link_mem_sections(nid, start_pfn, end_pfn, MEMINIT_EARLY); 134 134 } 135 135 136 136 return error;

+1 -1

include/linux/nodemask.h

reviewed

··· 90 90 * for such situations. See below and CPUMASK_ALLOC also. 91 91 */ 92 92 93 93 - #include <linux/kernel.h> 94 93 #include <linux/threads.h> 95 94 #include <linux/bitmap.h> 95 95 + #include <linux/minmax.h> 96 96 #include <linux/numa.h> 97 97 98 98 typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t;

+1 -5

include/linux/page-flags.h

reviewed

··· 431 431 PAGEFLAG(HWPoison, hwpoison, PF_ANY) 432 432 TESTSCFLAG(HWPoison, hwpoison, PF_ANY) 433 433 #define __PG_HWPOISON (1UL << PG_hwpoison) 434 434 - extern bool set_hwpoison_free_buddy_page(struct page *page); 434 434 + extern bool take_page_off_buddy(struct page *page); 435 435 #else 436 436 PAGEFLAG_FALSE(HWPoison) 437 437 - static inline bool set_hwpoison_free_buddy_page(struct page *page) 438 438 - { 439 439 - return 0; 440 440 - } 441 437 #define __PG_HWPOISON 0 442 438 #endif 443 439

+3 -3

include/linux/page_owner.h

reviewed

··· 11 11 extern void __reset_page_owner(struct page *page, unsigned int order); 12 12 extern void __set_page_owner(struct page *page, 13 13 unsigned int order, gfp_t gfp_mask); 14 14 - extern void __split_page_owner(struct page *page, unsigned int order); 14 14 + extern void __split_page_owner(struct page *page, unsigned int nr); 15 15 extern void __copy_page_owner(struct page *oldpage, struct page *newpage); 16 16 extern void __set_page_owner_migrate_reason(struct page *page, int reason); 17 17 extern void __dump_page_owner(struct page *page); ··· 31 31 __set_page_owner(page, order, gfp_mask); 32 32 } 33 33 34 34 - static inline void split_page_owner(struct page *page, unsigned int order) 34 34 + static inline void split_page_owner(struct page *page, unsigned int nr) 35 35 { 36 36 if (static_branch_unlikely(&page_owner_inited)) 37 37 - __split_page_owner(page, order); 37 37 + __split_page_owner(page, nr); 38 38 } 39 39 static inline void copy_page_owner(struct page *oldpage, struct page *newpage) 40 40 {

+96 -11

include/linux/pagemap.h

reviewed

··· 29 29 AS_EXITING = 4, /* final truncate in progress */ 30 30 /* writeback related tags are not used */ 31 31 AS_NO_WRITEBACK_TAGS = 5, 32 32 + AS_THP_SUPPORT = 6, /* THPs supported */ 32 33 }; 33 34 34 35 /** ··· 119 118 static inline void mapping_set_gfp_mask(struct address_space *m, gfp_t mask) 120 119 { 121 120 m->gfp_mask = mask; 121 121 + } 122 122 + 123 123 + static inline bool mapping_thp_support(struct address_space *mapping) 124 124 + { 125 125 + return test_bit(AS_THP_SUPPORT, &mapping->flags); 126 126 + } 127 127 + 128 128 + static inline int filemap_nr_thps(struct address_space *mapping) 129 129 + { 130 130 + #ifdef CONFIG_READ_ONLY_THP_FOR_FS 131 131 + return atomic_read(&mapping->nr_thps); 132 132 + #else 133 133 + return 0; 134 134 + #endif 135 135 + } 136 136 + 137 137 + static inline void filemap_nr_thps_inc(struct address_space *mapping) 138 138 + { 139 139 + #ifdef CONFIG_READ_ONLY_THP_FOR_FS 140 140 + if (!mapping_thp_support(mapping)) 141 141 + atomic_inc(&mapping->nr_thps); 142 142 + #else 143 143 + WARN_ON_ONCE(1); 144 144 + #endif 145 145 + } 146 146 + 147 147 + static inline void filemap_nr_thps_dec(struct address_space *mapping) 148 148 + { 149 149 + #ifdef CONFIG_READ_ONLY_THP_FOR_FS 150 150 + if (!mapping_thp_support(mapping)) 151 151 + atomic_dec(&mapping->nr_thps); 152 152 + #else 153 153 + WARN_ON_ONCE(1); 154 154 + #endif 122 155 } 123 156 124 157 void release_pages(struct page **pages, int nr); ··· 761 726 void delete_from_page_cache_batch(struct address_space *mapping, 762 727 struct pagevec *pvec); 763 728 764 764 - #define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) 765 765 - 766 766 - void page_cache_sync_readahead(struct address_space *, struct file_ra_state *, 767 767 - struct file *, pgoff_t index, unsigned long req_count); 768 768 - void page_cache_async_readahead(struct address_space *, struct file_ra_state *, 769 769 - struct file *, struct page *, pgoff_t index, 770 770 - unsigned long req_count); 771 771 - void page_cache_readahead_unbounded(struct address_space *, struct file *, 772 772 - pgoff_t index, unsigned long nr_to_read, 773 773 - unsigned long lookahead_count); 774 774 - 775 729 /* 776 730 * Like add_to_page_cache_locked, but used to add newly allocated pages: 777 731 * the page is new, so we can just run __SetPageLocked() against it. ··· 800 776 unsigned int _nr_pages; 801 777 unsigned int _batch_count; 802 778 }; 779 779 + 780 780 + #define DEFINE_READAHEAD(rac, f, m, i) \ 781 781 + struct readahead_control rac = { \ 782 782 + .file = f, \ 783 783 + .mapping = m, \ 784 784 + ._index = i, \ 785 785 + } 786 786 + 787 787 + #define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) 788 788 + 789 789 + void page_cache_ra_unbounded(struct readahead_control *, 790 790 + unsigned long nr_to_read, unsigned long lookahead_count); 791 791 + void page_cache_sync_ra(struct readahead_control *, struct file_ra_state *, 792 792 + unsigned long req_count); 793 793 + void page_cache_async_ra(struct readahead_control *, struct file_ra_state *, 794 794 + struct page *, unsigned long req_count); 795 795 + 796 796 + /** 797 797 + * page_cache_sync_readahead - generic file readahead 798 798 + * @mapping: address_space which holds the pagecache and I/O vectors 799 799 + * @ra: file_ra_state which holds the readahead state 800 800 + * @file: Used by the filesystem for authentication. 801 801 + * @index: Index of first page to be read. 802 802 + * @req_count: Total number of pages being read by the caller. 803 803 + * 804 804 + * page_cache_sync_readahead() should be called when a cache miss happened: 805 805 + * it will submit the read. The readahead logic may decide to piggyback more 806 806 + * pages onto the read request if access patterns suggest it will improve 807 807 + * performance. 808 808 + */ 809 809 + static inline 810 810 + void page_cache_sync_readahead(struct address_space *mapping, 811 811 + struct file_ra_state *ra, struct file *file, pgoff_t index, 812 812 + unsigned long req_count) 813 813 + { 814 814 + DEFINE_READAHEAD(ractl, file, mapping, index); 815 815 + page_cache_sync_ra(&ractl, ra, req_count); 816 816 + } 817 817 + 818 818 + /** 819 819 + * page_cache_async_readahead - file readahead for marked pages 820 820 + * @mapping: address_space which holds the pagecache and I/O vectors 821 821 + * @ra: file_ra_state which holds the readahead state 822 822 + * @file: Used by the filesystem for authentication. 823 823 + * @page: The page at @index which triggered the readahead call. 824 824 + * @index: Index of first page to be read. 825 825 + * @req_count: Total number of pages being read by the caller. 826 826 + * 827 827 + * page_cache_async_readahead() should be called when a page is used which 828 828 + * is marked as PageReadahead; this is a marker to suggest that the application 829 829 + * has used up enough of the readahead window that we should start pulling in 830 830 + * more pages. 831 831 + */ 832 832 + static inline 833 833 + void page_cache_async_readahead(struct address_space *mapping, 834 834 + struct file_ra_state *ra, struct file *file, 835 835 + struct page *page, pgoff_t index, unsigned long req_count) 836 836 + { 837 837 + DEFINE_READAHEAD(ractl, file, mapping, index); 838 838 + page_cache_async_ra(&ractl, ra, page, req_count); 839 839 + } 803 840 804 841 /** 805 842 * readahead_page - Get the next page to read.

+1 -1

include/linux/sched.h

reviewed

··· 1013 1013 struct held_lock held_locks[MAX_LOCK_DEPTH]; 1014 1014 #endif 1015 1015 1016 1016 - #ifdef CONFIG_UBSAN 1016 1016 + #if defined(CONFIG_UBSAN) && !defined(CONFIG_UBSAN_TRAP) 1017 1017 unsigned int in_ubsan; 1018 1018 #endif 1019 1019

-25

include/linux/sched/mm.h

reviewed

··· 49 49 __mmdrop(mm); 50 50 } 51 51 52 52 - /* 53 53 - * This has to be called after a get_task_mm()/mmget_not_zero() 54 54 - * followed by taking the mmap_lock for writing before modifying the 55 55 - * vmas or anything the coredump pretends not to change from under it. 56 56 - * 57 57 - * It also has to be called when mmgrab() is used in the context of 58 58 - * the process, but then the mm_count refcount is transferred outside 59 59 - * the context of the process to run down_write() on that pinned mm. 60 60 - * 61 61 - * NOTE: find_extend_vma() called from GUP context is the only place 62 62 - * that can modify the "mm" (notably the vm_start/end) under mmap_lock 63 63 - * for reading and outside the context of the process, so it is also 64 64 - * the only case that holds the mmap_lock for reading that must call 65 65 - * this function. Generally if the mmap_lock is hold for reading 66 66 - * there's no need of this check after get_task_mm()/mmget_not_zero(). 67 67 - * 68 68 - * This function can be obsoleted and the check can be removed, after 69 69 - * the coredump code will hold the mmap_lock for writing before 70 70 - * invoking the ->core_dump methods. 71 71 - */ 72 72 - static inline bool mmget_still_valid(struct mm_struct *mm) 73 73 - { 74 74 - return likely(!mm->core_state); 75 75 - } 76 76 - 77 52 /** 78 53 * mmget() - Pin the address space associated with a &struct mm_struct. 79 54 * @mm: The address space to pin.

+11 -1

include/linux/uaccess.h

reviewed

··· 2 2 #ifndef __LINUX_UACCESS_H__ 3 3 #define __LINUX_UACCESS_H__ 4 4 5 5 + #include <linux/fault-inject-usercopy.h> 5 6 #include <linux/instrumented.h> 7 7 + #include <linux/minmax.h> 6 8 #include <linux/sched.h> 7 9 #include <linux/thread_info.h> 8 10 ··· 85 83 __copy_from_user(void *to, const void __user *from, unsigned long n) 86 84 { 87 85 might_fault(); 86 86 + if (should_fail_usercopy()) 87 87 + return n; 88 88 instrument_copy_from_user(to, from, n); 89 89 check_object_size(to, n, false); 90 90 return raw_copy_from_user(to, from, n); ··· 108 104 static __always_inline __must_check unsigned long 109 105 __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n) 110 106 { 107 107 + if (should_fail_usercopy()) 108 108 + return n; 111 109 instrument_copy_to_user(to, from, n); 112 110 check_object_size(from, n, true); 113 111 return raw_copy_to_user(to, from, n); ··· 119 113 __copy_to_user(void __user *to, const void *from, unsigned long n) 120 114 { 121 115 might_fault(); 116 116 + if (should_fail_usercopy()) 117 117 + return n; 122 118 instrument_copy_to_user(to, from, n); 123 119 check_object_size(from, n, true); 124 120 return raw_copy_to_user(to, from, n); ··· 132 124 { 133 125 unsigned long res = n; 134 126 might_fault(); 135 135 - if (likely(access_ok(from, n))) { 127 127 + if (!should_fail_usercopy() && likely(access_ok(from, n))) { 136 128 instrument_copy_from_user(to, from, n); 137 129 res = raw_copy_from_user(to, from, n); 138 130 } ··· 150 142 _copy_to_user(void __user *to, const void *from, unsigned long n) 151 143 { 152 144 might_fault(); 145 145 + if (should_fail_usercopy()) 146 146 + return n; 153 147 if (access_ok(to, n)) { 154 148 instrument_copy_to_user(to, from, n); 155 149 n = raw_copy_to_user(to, from, n);

+1 -1

include/linux/vmstat.h

reviewed

··· 28 28 unsigned nr_writeback; 29 29 unsigned nr_immediate; 30 30 unsigned nr_pageout; 31 31 - unsigned nr_activate[2]; 31 31 + unsigned nr_activate[ANON_AND_FILE]; 32 32 unsigned nr_ref_keep; 33 33 unsigned nr_unmap_fail; 34 34 unsigned nr_lazyfree_fail;

+22

include/linux/xarray.h

reviewed

··· 1505 1505 1506 1506 void xas_create_range(struct xa_state *); 1507 1507 1508 1508 + #ifdef CONFIG_XARRAY_MULTI 1509 1509 + int xa_get_order(struct xarray *, unsigned long index); 1510 1510 + void xas_split(struct xa_state *, void *entry, unsigned int order); 1511 1511 + void xas_split_alloc(struct xa_state *, void *entry, unsigned int order, gfp_t); 1512 1512 + #else 1513 1513 + static inline int xa_get_order(struct xarray *xa, unsigned long index) 1514 1514 + { 1515 1515 + return 0; 1516 1516 + } 1517 1517 + 1518 1518 + static inline void xas_split(struct xa_state *xas, void *entry, 1519 1519 + unsigned int order) 1520 1520 + { 1521 1521 + xas_store(xas, entry); 1522 1522 + } 1523 1523 + 1524 1524 + static inline void xas_split_alloc(struct xa_state *xas, void *entry, 1525 1525 + unsigned int order, gfp_t gfp) 1526 1526 + { 1527 1527 + } 1528 1528 + #endif 1529 1529 + 1508 1530 /** 1509 1531 * xas_reload() - Refetch an entry from the xarray. 1510 1532 * @xas: XArray operation state.

+3

include/ras/ras_event.h

reviewed

··· 361 361 EM ( MF_MSG_POISONED_HUGE, "huge page already hardware poisoned" ) \ 362 362 EM ( MF_MSG_HUGE, "huge page" ) \ 363 363 EM ( MF_MSG_FREE_HUGE, "free huge page" ) \ 364 364 + EM ( MF_MSG_NON_PMD_HUGE, "non-pmd-sized huge page" ) \ 364 365 EM ( MF_MSG_UNMAP_FAILED, "unmapping failed page" ) \ 365 366 EM ( MF_MSG_DIRTY_SWAPCACHE, "dirty swapcache page" ) \ 366 367 EM ( MF_MSG_CLEAN_SWAPCACHE, "clean swapcache page" ) \ ··· 374 373 EM ( MF_MSG_TRUNCATED_LRU, "already truncated LRU page" ) \ 375 374 EM ( MF_MSG_BUDDY, "free buddy page" ) \ 376 375 EM ( MF_MSG_BUDDY_2ND, "free buddy page (2nd try)" ) \ 376 376 + EM ( MF_MSG_DAX, "dax page" ) \ 377 377 + EM ( MF_MSG_UNSPLIT_THP, "unsplit thp" ) \ 377 378 EMe ( MF_MSG_UNKNOWN, "unknown page" ) 378 379 379 380 /*

+4 -6

kernel/acct.c

reviewed

··· 25 25 * Now we silently close acct_file on attempt to reopen. Cleaned sys_acct(). 26 26 * XTerms and EMACS are manifestations of pure evil. 21/10/98, AV. 27 27 * 28 28 - * Fixed a nasty interaction with with sys_umount(). If the accointing 28 28 + * Fixed a nasty interaction with sys_umount(). If the accounting 29 29 * was suspeneded we failed to stop it on umount(). Messy. 30 30 * Another one: remount to readonly didn't stop accounting. 31 31 * Question: what should we do if we have CAP_SYS_ADMIN but not ··· 263 263 * sys_acct - enable/disable process accounting 264 264 * @name: file name for accounting records or NULL to shutdown accounting 265 265 * 266 266 - * Returns 0 for success or negative errno values for failure. 267 267 - * 268 266 * sys_acct() is the only system call needed to implement process 269 267 * accounting. It takes the name of the file where accounting records 270 268 * should be written. If the filename is NULL, accounting will be 271 269 * shutdown. 270 270 + * 271 271 + * Returns: 0 for success or negative errno values for failure. 272 272 */ 273 273 SYSCALL_DEFINE1(acct, const char __user *, name) 274 274 { ··· 586 586 } 587 587 588 588 /** 589 589 - * acct_process 590 590 - * 591 591 - * handles process accounting for an exiting task 589 589 + * acct_process - handles process accounting for an exiting task 592 590 */ 593 591 void acct_process(void) 594 592 {

+1 -1

kernel/cgroup/cpuset.c

reviewed

··· 390 390 * The top cpuset doesn't have any online cpu as a 391 391 * consequence of a race between cpuset_hotplug_work 392 392 * and cpu hotplug notifier. But we know the top 393 393 - * cpuset's effective_cpus is on its way to to be 393 393 + * cpuset's effective_cpus is on its way to be 394 394 * identical to cpu_online_mask. 395 395 */ 396 396 cpumask_copy(pmask, cpu_online_mask);

+1 -1

kernel/dma/direct.c

reviewed

··· 16 16 #include "direct.h" 17 17 18 18 /* 19 19 - * Most architectures use ZONE_DMA for the first 16 Megabytes, but some use it 19 19 + * Most architectures use ZONE_DMA for the first 16 Megabytes, but some use 20 20 * it for entirely different regions. In that case the arch code needs to 21 21 * override the variable below for dma-direct to work properly. 22 22 */

+2 -2

kernel/fork.c

reviewed

··· 556 556 557 557 get_file(file); 558 558 if (tmp->vm_flags & VM_DENYWRITE) 559 559 - atomic_dec(&inode->i_writecount); 559 559 + put_write_access(inode); 560 560 i_mmap_lock_write(mapping); 561 561 if (tmp->vm_flags & VM_SHARED) 562 562 mapping_allow_writable(mapping); ··· 2189 2189 2190 2190 /* 2191 2191 * Ensure that the cgroup subsystem policies allow the new process to be 2192 2192 - * forked. It should be noted the the new process's css_set can be changed 2192 2192 + * forked. It should be noted that the new process's css_set can be changed 2193 2193 * between here and cgroup_post_fork() if an organisation operation is in 2194 2194 * progress. 2195 2195 */

+1 -1

kernel/futex.c

reviewed

··· 916 916 * [10] Found | Found | task | !=taskTID | 0/1 | Invalid 917 917 * 918 918 * [1] Indicates that the kernel can acquire the futex atomically. We 919 919 - * came came here due to a stale FUTEX_WAITERS/FUTEX_OWNER_DIED bit. 919 919 + * came here due to a stale FUTEX_WAITERS/FUTEX_OWNER_DIED bit. 920 920 * 921 921 * [2] Valid, if TID does not belong to a kernel thread. If no matching 922 922 * thread is found then it indicates that the owner TID has died.

+1 -1

kernel/irq/timings.c

reviewed

··· 604 604 605 605 /* 606 606 * Some platforms can have the same private interrupt per cpu, 607 607 - * so this function may be be called several times with the 607 607 + * so this function may be called several times with the 608 608 * same interrupt number. Just bail out in case the per cpu 609 609 * stat structure is already allocated. 610 610 */

+1 -1

kernel/jump_label.c

reviewed

··· 19 19 #include <linux/cpu.h> 20 20 #include <asm/sections.h> 21 21 22 22 - /* mutex to protect coming/going of the the jump_label table */ 22 22 + /* mutex to protect coming/going of the jump_label table */ 23 23 static DEFINE_MUTEX(jump_label_mutex); 24 24 25 25 void jump_label_lock(void)

+1 -1

kernel/kcsan/encoding.h

reviewed

··· 32 32 * 1. different addresses but with the same encoded address race; 33 33 * 2. and both map onto the same watchpoint slots; 34 34 * 35 35 - * Both these are assumed to be very unlikely. However, in case it still happens 35 35 + * Both these are assumed to be very unlikely. However, in case it still 36 36 * happens, the report logic will filter out the false positive (see report.c). 37 37 */ 38 38 #define WATCHPOINT_ADDR_BITS (BITS_PER_LONG-1 - WATCHPOINT_SIZE_BITS)

+1 -1

kernel/kexec_core.c

reviewed

··· 109 109 * defined more restrictively in <asm/kexec.h>. 110 110 * 111 111 * The code for the transition from the current kernel to the 112 112 - * the new kernel is placed in the control_code_buffer, whose size 112 112 + * new kernel is placed in the control_code_buffer, whose size 113 113 * is given by KEXEC_CONTROL_PAGE_SIZE. In the best case only a single 114 114 * page of memory is necessary, but some architectures require more. 115 115 * Because this memory must be identity mapped in the transition from

+1 -1

kernel/kexec_file.c

reviewed

··· 521 521 /* Returning 0 will take to next memory range */ 522 522 523 523 /* Don't use memory that will be detected and handled by a driver. */ 524 524 - if (res->flags & IORESOURCE_MEM_DRIVER_MANAGED) 524 524 + if (res->flags & IORESOURCE_SYSRAM_DRIVER_MANAGED) 525 525 return 0; 526 526 527 527 if (sz < kbuf->memsz)

+1 -1

kernel/kthread.c

reviewed

··· 775 775 776 776 /** 777 777 * kthread_create_worker_on_cpu - create a kthread worker and bind it 778 778 - * it to a given CPU and the associated NUMA node. 778 778 + * to a given CPU and the associated NUMA node. 779 779 * @cpu: CPU number 780 780 * @flags: flags modifying the default behavior of the worker 781 781 * @namefmt: printf-style name for the kthread worker (task).

+1 -1

kernel/livepatch/state.c

reviewed

··· 55 55 * 56 56 * The function can be called only during transition when a new 57 57 * livepatch is being enabled or when such a transition is reverted. 58 58 - * It is typically called only from from pre/post (un)patch 58 58 + * It is typically called only from pre/post (un)patch 59 59 * callbacks. 60 60 * 61 61 * Return: pointer to the latest struct klp_state from already

+6 -6

kernel/panic.c

reviewed

··· 589 589 if (args) 590 590 vprintk(args->fmt, args->args); 591 591 592 592 + print_modules(); 593 593 + 594 594 + if (regs) 595 595 + show_regs(regs); 596 596 + 592 597 if (panic_on_warn) { 593 598 /* 594 599 * This thread may hit another WARN() in the panic path. ··· 605 600 panic("panic_on_warn set ...\n"); 606 601 } 607 602 608 608 - print_modules(); 609 609 - 610 610 - if (regs) 611 611 - show_regs(regs); 612 612 - else 613 613 - dump_stack(); 603 603 + dump_stack(); 614 604 615 605 print_irqtrace_events(current); 616 606

+1 -1

kernel/pid_namespace.c

reviewed

··· 233 233 * to pid_ns->child_reaper. Thus pidns->child_reaper needs to 234 234 * stay valid until they all go away. 235 235 * 236 236 - * The code relies on the the pid_ns->child_reaper ignoring 236 236 + * The code relies on the pid_ns->child_reaper ignoring 237 237 * SIGCHILD to cause those EXIT_ZOMBIE processes to be 238 238 * autoreaped if reparented. 239 239 *

+1 -1

kernel/power/snapshot.c

reviewed

··· 735 735 */ 736 736 737 737 /* 738 738 - * If the zone we wish to scan is the the current zone and the 738 738 + * If the zone we wish to scan is the current zone and the 739 739 * pfn falls into the current node then we do not need to walk 740 740 * the tree. 741 741 */

+2 -1

kernel/range.c

reviewed

··· 2 2 /* 3 3 * Range add and subtract 4 4 */ 5 5 - #include <linux/kernel.h> 6 5 #include <linux/init.h> 6 6 + #include <linux/minmax.h> 7 7 + #include <linux/printk.h> 7 8 #include <linux/sort.h> 8 9 #include <linux/string.h> 9 10 #include <linux/range.h>

+1 -1

kernel/relay.c

reviewed

··· 1002 1002 size_t subbuf_size = buf->chan->subbuf_size; 1003 1003 size_t n_subbufs = buf->chan->n_subbufs; 1004 1004 size_t produced = buf->subbufs_produced; 1005 1005 - size_t consumed = buf->subbufs_consumed; 1005 1005 + size_t consumed; 1006 1006 1007 1007 relay_file_read_consume(buf, 0, 0); 1008 1008

+88 -22

kernel/resource.c

reviewed

··· 1240 1240 #ifdef CONFIG_MEMORY_HOTREMOVE 1241 1241 /** 1242 1242 * release_mem_region_adjustable - release a previously reserved memory region 1243 1243 - * @parent: parent resource descriptor 1244 1243 * @start: resource start address 1245 1244 * @size: resource region size 1246 1245 * ··· 1257 1258 * assumes that all children remain in the lower address entry for 1258 1259 * simplicity. Enhance this logic when necessary. 1259 1260 */ 1260 1260 - int release_mem_region_adjustable(struct resource *parent, 1261 1261 - resource_size_t start, resource_size_t size) 1261 1261 + void release_mem_region_adjustable(resource_size_t start, resource_size_t size) 1262 1262 { 1263 1263 + struct resource *parent = &iomem_resource; 1264 1264 + struct resource *new_res = NULL; 1265 1265 + bool alloc_nofail = false; 1263 1266 struct resource **p; 1264 1267 struct resource *res; 1265 1265 - struct resource *new_res; 1266 1268 resource_size_t end; 1267 1267 - int ret = -EINVAL; 1268 1269 1269 1270 end = start + size - 1; 1270 1270 - if ((start < parent->start) || (end > parent->end)) 1271 1271 - return ret; 1271 1271 + if (WARN_ON_ONCE((start < parent->start) || (end > parent->end))) 1272 1272 + return; 1272 1273 1273 1273 - /* The alloc_resource() result gets checked later */ 1274 1274 - new_res = alloc_resource(GFP_KERNEL); 1274 1274 + /* 1275 1275 + * We free up quite a lot of memory on memory hotunplug (esp., memap), 1276 1276 + * just before releasing the region. This is highly unlikely to 1277 1277 + * fail - let's play save and make it never fail as the caller cannot 1278 1278 + * perform any error handling (e.g., trying to re-add memory will fail 1279 1279 + * similarly). 1280 1280 + */ 1281 1281 + retry: 1282 1282 + new_res = alloc_resource(GFP_KERNEL | (alloc_nofail ? __GFP_NOFAIL : 0)); 1275 1283 1276 1284 p = &parent->child; 1277 1285 write_lock(&resource_lock); ··· 1304 1298 * so if we are dealing with them, let us just back off here. 1305 1299 */ 1306 1300 if (!(res->flags & IORESOURCE_SYSRAM)) { 1307 1307 - ret = 0; 1308 1301 break; 1309 1302 } 1310 1303 ··· 1320 1315 /* free the whole entry */ 1321 1316 *p = res->sibling; 1322 1317 free_resource(res); 1323 1323 - ret = 0; 1324 1318 } else if (res->start == start && res->end != end) { 1325 1319 /* adjust the start */ 1326 1326 - ret = __adjust_resource(res, end + 1, 1327 1327 - res->end - end); 1320 1320 + WARN_ON_ONCE(__adjust_resource(res, end + 1, 1321 1321 + res->end - end)); 1328 1322 } else if (res->start != start && res->end == end) { 1329 1323 /* adjust the end */ 1330 1330 - ret = __adjust_resource(res, res->start, 1331 1331 - start - res->start); 1324 1324 + WARN_ON_ONCE(__adjust_resource(res, res->start, 1325 1325 + start - res->start)); 1332 1326 } else { 1333 1333 - /* split into two entries */ 1327 1327 + /* split into two entries - we need a new resource */ 1334 1328 if (!new_res) { 1335 1335 - ret = -ENOMEM; 1336 1336 - break; 1329 1329 + new_res = alloc_resource(GFP_ATOMIC); 1330 1330 + if (!new_res) { 1331 1331 + alloc_nofail = true; 1332 1332 + write_unlock(&resource_lock); 1333 1333 + goto retry; 1334 1334 + } 1337 1335 } 1338 1336 new_res->name = res->name; 1339 1337 new_res->start = end + 1; ··· 1347 1339 new_res->sibling = res->sibling; 1348 1340 new_res->child = NULL; 1349 1341 1350 1350 - ret = __adjust_resource(res, res->start, 1351 1351 - start - res->start); 1352 1352 - if (ret) 1342 1342 + if (WARN_ON_ONCE(__adjust_resource(res, res->start, 1343 1343 + start - res->start))) 1353 1344 break; 1354 1345 res->sibling = new_res; 1355 1346 new_res = NULL; ··· 1359 1352 1360 1353 write_unlock(&resource_lock); 1361 1354 free_resource(new_res); 1362 1362 - return ret; 1363 1355 } 1364 1356 #endif /* CONFIG_MEMORY_HOTREMOVE */ 1357 1357 + 1358 1358 + #ifdef CONFIG_MEMORY_HOTPLUG 1359 1359 + static bool system_ram_resources_mergeable(struct resource *r1, 1360 1360 + struct resource *r2) 1361 1361 + { 1362 1362 + /* We assume either r1 or r2 is IORESOURCE_SYSRAM_MERGEABLE. */ 1363 1363 + return r1->flags == r2->flags && r1->end + 1 == r2->start && 1364 1364 + r1->name == r2->name && r1->desc == r2->desc && 1365 1365 + !r1->child && !r2->child; 1366 1366 + } 1367 1367 + 1368 1368 + /* 1369 1369 + * merge_system_ram_resource - mark the System RAM resource mergeable and try to 1370 1370 + * merge it with adjacent, mergeable resources 1371 1371 + * @res: resource descriptor 1372 1372 + * 1373 1373 + * This interface is intended for memory hotplug, whereby lots of contiguous 1374 1374 + * system ram resources are added (e.g., via add_memory*()) by a driver, and 1375 1375 + * the actual resource boundaries are not of interest (e.g., it might be 1376 1376 + * relevant for DIMMs). Only resources that are marked mergeable, that have the 1377 1377 + * same parent, and that don't have any children are considered. All mergeable 1378 1378 + * resources must be immutable during the request. 1379 1379 + * 1380 1380 + * Note: 1381 1381 + * - The caller has to make sure that no pointers to resources that are 1382 1382 + * marked mergeable are used anymore after this call - the resource might 1383 1383 + * be freed and the pointer might be stale! 1384 1384 + * - release_mem_region_adjustable() will split on demand on memory hotunplug 1385 1385 + */ 1386 1386 + void merge_system_ram_resource(struct resource *res) 1387 1387 + { 1388 1388 + const unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; 1389 1389 + struct resource *cur; 1390 1390 + 1391 1391 + if (WARN_ON_ONCE((res->flags & flags) != flags)) 1392 1392 + return; 1393 1393 + 1394 1394 + write_lock(&resource_lock); 1395 1395 + res->flags |= IORESOURCE_SYSRAM_MERGEABLE; 1396 1396 + 1397 1397 + /* Try to merge with next item in the list. */ 1398 1398 + cur = res->sibling; 1399 1399 + if (cur && system_ram_resources_mergeable(res, cur)) { 1400 1400 + res->end = cur->end; 1401 1401 + res->sibling = cur->sibling; 1402 1402 + free_resource(cur); 1403 1403 + } 1404 1404 + 1405 1405 + /* Try to merge with previous item in the list. */ 1406 1406 + cur = res->parent->child; 1407 1407 + while (cur && cur->sibling != res) 1408 1408 + cur = cur->sibling; 1409 1409 + if (cur && system_ram_resources_mergeable(cur, res)) { 1410 1410 + cur->end = res->end; 1411 1411 + cur->sibling = res->sibling; 1412 1412 + free_resource(res); 1413 1413 + } 1414 1414 + write_unlock(&resource_lock); 1415 1415 + } 1416 1416 + #endif /* CONFIG_MEMORY_HOTPLUG */ 1365 1417 1366 1418 /* 1367 1419 * Managed region resource

+1 -1

kernel/smp.c

reviewed

··· 741 741 * for all the required CPUs to finish. This may include the local 742 742 * processor. 743 743 * @cond_func: A callback function that is passed a cpu id and 744 744 - * the the info parameter. The function is called 744 744 + * the info parameter. The function is called 745 745 * with preemption disabled. The function should 746 746 * return a blooean value indicating whether to IPI 747 747 * the specified CPU.

+1 -1

kernel/sys.c

reviewed

··· 2034 2034 * VMAs already unmapped and kernel uses these members for statistics 2035 2035 * output in procfs mostly, except 2036 2036 * 2037 2037 - * - @start_brk/@brk which are used in do_brk but kernel lookups 2037 2037 + * - @start_brk/@brk which are used in do_brk_flags but kernel lookups 2038 2038 * for VMAs when updating these memvers so anything wrong written 2039 2039 * here cause kernel to swear at userspace program but won't lead 2040 2040 * to any problem in kernel itself

+1 -1

kernel/user_namespace.c

reviewed

··· 515 515 * 516 516 * When there is no mapping defined for the user-namespace projid 517 517 * pair INVALID_PROJID is returned. Callers are expected to test 518 518 - * for and handle handle INVALID_PROJID being returned. INVALID_PROJID 518 518 + * for and handle INVALID_PROJID being returned. INVALID_PROJID 519 519 * may be tested for using projid_valid(). 520 520 */ 521 521 kprojid_t make_kprojid(struct user_namespace *ns, projid_t projid)

+7

lib/Kconfig.debug

reviewed

··· 1768 1768 help 1769 1769 Provide fault-injection capability for alloc_pages(). 1770 1770 1771 1771 + config FAULT_INJECTION_USERCOPY 1772 1772 + bool "Fault injection capability for usercopy functions" 1773 1773 + depends on FAULT_INJECTION 1774 1774 + help 1775 1775 + Provides fault-injection capability to inject failures 1776 1776 + in usercopy functions (copy_from_user(), get_user(), ...). 1777 1777 + 1771 1778 config FAIL_MAKE_REQUEST 1772 1779 bool "Fault-injection capability for disk IO" 1773 1780 depends on FAULT_INJECTION && BLOCK

+14

lib/Kconfig.ubsan

reviewed

··· 47 47 to the {str,mem}*cpy() family of functions (that is addressed 48 48 by CONFIG_FORTIFY_SOURCE). 49 49 50 50 + config UBSAN_LOCAL_BOUNDS 51 51 + bool "Perform array local bounds checking" 52 52 + depends on UBSAN_TRAP 53 53 + depends on CC_IS_CLANG 54 54 + depends on !UBSAN_KCOV_BROKEN 55 55 + help 56 56 + This option enables -fsanitize=local-bounds which traps when an 57 57 + exception/error is detected. Therefore, it should be enabled only 58 58 + if trapping is expected. 59 59 + Enabling this option detects errors due to accesses through a 60 60 + pointer that is derived from an object of a statically-known size, 61 61 + where an added offset (which may not be known statically) is 62 62 + out-of-bounds. 63 63 + 50 64 config UBSAN_MISC 51 65 bool "Enable all other Undefined Behavior sanity checks" 52 66 default UBSAN

+1

lib/Makefile

reviewed

··· 210 210 211 211 obj-$(CONFIG_IOMMU_HELPER) += iommu-helper.o 212 212 obj-$(CONFIG_FAULT_INJECTION) += fault-inject.o 213 213 + obj-$(CONFIG_FAULT_INJECTION_USERCOPY) += fault-inject-usercopy.o 213 214 obj-$(CONFIG_NOTIFIER_ERROR_INJECTION) += notifier-error-inject.o 214 215 obj-$(CONFIG_PM_NOTIFIER_ERROR_INJECT) += pm-notifier-error-inject.o 215 216 obj-$(CONFIG_NETDEV_NOTIFIER_ERROR_INJECT) += netdev-notifier-error-inject.o

+1 -1

lib/bitmap.c

reviewed

··· 23 23 /** 24 24 * DOC: bitmap introduction 25 25 * 26 26 - * bitmaps provide an array of bits, implemented using an an 26 26 + * bitmaps provide an array of bits, implemented using an 27 27 * array of unsigned longs. The number of valid bits in a 28 28 * given bitmap does _not_ need to be an exact multiple of 29 29 * BITS_PER_LONG.

+1 -1

lib/crc32.c

reviewed

··· 331 331 return crc; 332 332 } 333 333 334 334 - #if CRC_LE_BITS == 1 334 334 + #if CRC_BE_BITS == 1 335 335 u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len) 336 336 { 337 337 return crc32_be_generic(crc, p, len, NULL, CRC32_POLY_BE);

+1 -1

lib/decompress_bunzip2.c

reviewed

··· 390 390 j = (bd->inbufBits >> bd->inbufBitCount)& 391 391 ((1 << hufGroup->maxLen)-1); 392 392 got_huff_bits: 393 393 - /* Figure how how many bits are in next symbol and 393 393 + /* Figure how many bits are in next symbol and 394 394 * unget extras */ 395 395 i = hufGroup->minLen; 396 396 while (j > limit[i])

+2 -2

lib/dynamic_queue_limits.c

reviewed

··· 60 60 * A decrease is only considered if the queue has been busy in 61 61 * the whole interval (the check above). 62 62 * 63 63 - * If there is slack, the amount of execess data queued above 64 64 - * the the amount needed to prevent starvation, the queue limit 63 63 + * If there is slack, the amount of excess data queued above 64 64 + * the amount needed to prevent starvation, the queue limit 65 65 * can be decreased. To avoid hysteresis we consider the 66 66 * minimum amount of slack found over several iterations of the 67 67 * completion routine.

+1 -1

lib/earlycpio.c

reviewed

··· 42 42 /** 43 43 * cpio_data find_cpio_data - Search for files in an uncompressed cpio 44 44 * @path: The directory to search for, including a slash at the end 45 45 - * @data: Pointer to the the cpio archive or a header inside 45 45 + * @data: Pointer to the cpio archive or a header inside 46 46 * @len: Remaining length of the cpio based on data pointer 47 47 * @nextoff: When a matching file is found, this is the offset from the 48 48 * beginning of the cpio to the beginning of the next file, not the

+39

lib/fault-inject-usercopy.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + #include <linux/fault-inject.h> 3 3 + #include <linux/fault-inject-usercopy.h> 4 4 + 5 5 + static struct { 6 6 + struct fault_attr attr; 7 7 + } fail_usercopy = { 8 8 + .attr = FAULT_ATTR_INITIALIZER, 9 9 + }; 10 10 + 11 11 + static int __init setup_fail_usercopy(char *str) 12 12 + { 13 13 + return setup_fault_attr(&fail_usercopy.attr, str); 14 14 + } 15 15 + __setup("fail_usercopy=", setup_fail_usercopy); 16 16 + 17 17 + #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 18 18 + 19 19 + static int __init fail_usercopy_debugfs(void) 20 20 + { 21 21 + struct dentry *dir; 22 22 + 23 23 + dir = fault_create_debugfs_attr("fail_usercopy", NULL, 24 24 + &fail_usercopy.attr); 25 25 + if (IS_ERR(dir)) 26 26 + return PTR_ERR(dir); 27 27 + 28 28 + return 0; 29 29 + } 30 30 + 31 31 + late_initcall(fail_usercopy_debugfs); 32 32 + 33 33 + #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 34 34 + 35 35 + bool should_fail_usercopy(void) 36 36 + { 37 37 + return should_fail(&fail_usercopy.attr, 1); 38 38 + } 39 39 + EXPORT_SYMBOL_GPL(should_fail_usercopy);

+1

lib/find_bit.c

reviewed

··· 16 16 #include <linux/bitmap.h> 17 17 #include <linux/export.h> 18 18 #include <linux/kernel.h> 19 19 + #include <linux/minmax.h> 19 20 20 21 #if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ 21 22 !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) || \

+1

lib/hexdump.c

reviewed

··· 7 7 #include <linux/ctype.h> 8 8 #include <linux/errno.h> 9 9 #include <linux/kernel.h> 10 10 + #include <linux/minmax.h> 10 11 #include <linux/export.h> 11 12 #include <asm/unaligned.h> 12 13

+6 -3

lib/idr.c

reviewed

··· 372 372 * Allocate an ID between @min and @max, inclusive. The allocated ID will 373 373 * not exceed %INT_MAX, even if @max is larger. 374 374 * 375 375 - * Context: Any context. 375 375 + * Context: Any context. It is safe to call this function without 376 376 + * locking in your code. 376 377 * Return: The allocated ID, or %-ENOMEM if memory could not be allocated, 377 378 * or %-ENOSPC if there are no free IDs. 378 379 */ ··· 480 479 * @ida: IDA handle. 481 480 * @id: Previously allocated ID. 482 481 * 483 483 - * Context: Any context. 482 482 + * Context: Any context. It is safe to call this function without 483 483 + * locking in your code. 484 484 */ 485 485 void ida_free(struct ida *ida, unsigned int id) 486 486 { ··· 533 531 * or freed. If the IDA is already empty, there is no need to call this 534 532 * function. 535 533 * 536 536 - * Context: Any context. 534 534 + * Context: Any context. It is safe to call this function without 535 535 + * locking in your code. 537 536 */ 538 537 void ida_destroy(struct ida *ida) 539 538 {

+5

lib/iov_iter.c

reviewed

··· 2 2 #include <crypto/hash.h> 3 3 #include <linux/export.h> 4 4 #include <linux/bvec.h> 5 5 + #include <linux/fault-inject-usercopy.h> 5 6 #include <linux/uio.h> 6 7 #include <linux/pagemap.h> 7 8 #include <linux/slab.h> ··· 141 140 142 141 static int copyout(void __user *to, const void *from, size_t n) 143 142 { 143 143 + if (should_fail_usercopy()) 144 144 + return n; 144 145 if (access_ok(to, n)) { 145 146 instrument_copy_to_user(to, from, n); 146 147 n = raw_copy_to_user(to, from, n); ··· 152 149 153 150 static int copyin(void *to, const void __user *from, size_t n) 154 151 { 152 152 + if (should_fail_usercopy()) 153 153 + return n; 155 154 if (access_ok(from, n)) { 156 155 instrument_copy_from_user(to, from, n); 157 156 n = raw_copy_from_user(to, from, n);

+1 -1

lib/libcrc32c.c

reviewed

··· 12 12 * pages = {}, 13 13 * month = {June}, 14 14 *} 15 15 - * Used by the iSCSI driver, possibly others, and derived from the 15 15 + * Used by the iSCSI driver, possibly others, and derived from 16 16 * the iscsi-crc.c module of the linux-iscsi driver at 17 17 * http://linux-iscsi.sourceforge.net. 18 18 *

+1 -1

lib/math/rational.c

reviewed

··· 11 11 #include <linux/rational.h> 12 12 #include <linux/compiler.h> 13 13 #include <linux/export.h> 14 14 - #include <linux/kernel.h> 14 14 + #include <linux/minmax.h> 15 15 16 16 /* 17 17 * calculate best rational approximation for a given fraction

+1

lib/math/reciprocal_div.c

reviewed

··· 4 4 #include <asm/div64.h> 5 5 #include <linux/reciprocal_div.h> 6 6 #include <linux/export.h> 7 7 + #include <linux/minmax.h> 7 8 8 9 /* 9 10 * For a description of the algorithm please have a look at

+1 -1

lib/mpi/mpi-bit.c

reviewed

··· 1 1 - /* mpi-bit.c - MPI bit level fucntions 1 1 + /* mpi-bit.c - MPI bit level functions 2 2 * Copyright (C) 1998, 1999 Free Software Foundation, Inc. 3 3 * 4 4 * This file is part of GnuPG.

+1 -1

lib/percpu_counter.c

reviewed

··· 85 85 86 86 preempt_disable(); 87 87 count = __this_cpu_read(*fbc->counters) + amount; 88 88 - if (count >= batch || count <= -batch) { 88 88 + if (abs(count) >= batch) { 89 89 unsigned long flags; 90 90 raw_spin_lock_irqsave(&fbc->lock, flags); 91 91 fbc->count += count;

+1 -1

lib/radix-tree.c

reviewed

··· 325 325 int ret = -ENOMEM; 326 326 327 327 /* 328 328 - * Nodes preloaded by one cgroup can be be used by another cgroup, so 328 328 + * Nodes preloaded by one cgroup can be used by another cgroup, so 329 329 * they should never be accounted to any particular memory cgroup. 330 330 */ 331 331 gfp_mask &= ~__GFP_ACCOUNT;

+1 -1

lib/scatterlist.c

reviewed

··· 504 504 nalloc++; 505 505 } 506 506 sgl = kmalloc_array(nalloc, sizeof(struct scatterlist), 507 507 - (gfp & ~GFP_DMA) | __GFP_ZERO); 507 507 + gfp & ~GFP_DMA); 508 508 if (!sgl) 509 509 return NULL; 510 510

+3

lib/strncpy_from_user.c

reviewed

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 #include <linux/compiler.h> 3 3 #include <linux/export.h> 4 4 + #include <linux/fault-inject-usercopy.h> 4 5 #include <linux/kasan-checks.h> 5 6 #include <linux/thread_info.h> 6 7 #include <linux/uaccess.h> ··· 100 99 unsigned long max_addr, src_addr; 101 100 102 101 might_fault(); 102 102 + if (should_fail_usercopy()) 103 103 + return -EFAULT; 103 104 if (unlikely(count <= 0)) 104 105 return 0; 105 106

+1 -1

lib/syscall.c

reviewed

··· 44 44 * .data.instruction_pointer - filled with user PC 45 45 * 46 46 * If @target is blocked in a system call, returns zero with @info.data.nr 47 47 - * set to the the call's number and @info.data.args filled in with its 47 47 + * set to the call's number and @info.data.args filled in with its 48 48 * arguments. Registers not used for system call arguments may not be available 49 49 * and it is not kosher to use &struct user_regset calls while the system 50 50 * call is still in progress. Note we may get this result if @target

+1 -1

lib/test_hmm.c

reviewed

··· 461 461 462 462 devmem = kzalloc(sizeof(*devmem), GFP_KERNEL); 463 463 if (!devmem) 464 464 - return -ENOMEM; 464 464 + return false; 465 465 466 466 res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, 467 467 "hmm_dmirror");

+1 -1

lib/test_sysctl.c

reviewed

··· 16 16 */ 17 17 18 18 /* 19 19 - * This module provides an interface to the the proc sysctl interfaces. This 19 19 + * This module provides an interface to the proc sysctl interfaces. This 20 20 * driver requires CONFIG_PROC_SYSCTL. It will not normally be loaded by the 21 21 * system unless explicitly requested by name. You can also build this driver 22 22 * into your kernel.

+65

lib/test_xarray.c

reviewed

··· 1503 1503 } 1504 1504 } 1505 1505 1506 1506 + #ifdef CONFIG_XARRAY_MULTI 1507 1507 + static void check_split_1(struct xarray *xa, unsigned long index, 1508 1508 + unsigned int order) 1509 1509 + { 1510 1510 + XA_STATE(xas, xa, index); 1511 1511 + void *entry; 1512 1512 + unsigned int i = 0; 1513 1513 + 1514 1514 + xa_store_order(xa, index, order, xa, GFP_KERNEL); 1515 1515 + 1516 1516 + xas_split_alloc(&xas, xa, order, GFP_KERNEL); 1517 1517 + xas_lock(&xas); 1518 1518 + xas_split(&xas, xa, order); 1519 1519 + xas_unlock(&xas); 1520 1520 + 1521 1521 + xa_for_each(xa, index, entry) { 1522 1522 + XA_BUG_ON(xa, entry != xa); 1523 1523 + i++; 1524 1524 + } 1525 1525 + XA_BUG_ON(xa, i != 1 << order); 1526 1526 + 1527 1527 + xa_set_mark(xa, index, XA_MARK_0); 1528 1528 + XA_BUG_ON(xa, !xa_get_mark(xa, index, XA_MARK_0)); 1529 1529 + 1530 1530 + xa_destroy(xa); 1531 1531 + } 1532 1532 + 1533 1533 + static noinline void check_split(struct xarray *xa) 1534 1534 + { 1535 1535 + unsigned int order; 1536 1536 + 1537 1537 + XA_BUG_ON(xa, !xa_empty(xa)); 1538 1538 + 1539 1539 + for (order = 1; order < 2 * XA_CHUNK_SHIFT; order++) { 1540 1540 + check_split_1(xa, 0, order); 1541 1541 + check_split_1(xa, 1UL << order, order); 1542 1542 + check_split_1(xa, 3UL << order, order); 1543 1543 + } 1544 1544 + } 1545 1545 + #else 1546 1546 + static void check_split(struct xarray *xa) { } 1547 1547 + #endif 1548 1548 + 1506 1549 static void check_align_1(struct xarray *xa, char *name) 1507 1550 { 1508 1551 int i; ··· 1692 1649 #endif 1693 1650 } 1694 1651 1652 1652 + static noinline void check_get_order(struct xarray *xa) 1653 1653 + { 1654 1654 + unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 20 : 1; 1655 1655 + unsigned int order; 1656 1656 + unsigned long i, j; 1657 1657 + 1658 1658 + for (i = 0; i < 3; i++) 1659 1659 + XA_BUG_ON(xa, xa_get_order(xa, i) != 0); 1660 1660 + 1661 1661 + for (order = 0; order < max_order; order++) { 1662 1662 + for (i = 0; i < 10; i++) { 1663 1663 + xa_store_order(xa, i << order, order, 1664 1664 + xa_mk_index(i << order), GFP_KERNEL); 1665 1665 + for (j = i << order; j < (i + 1) << order; j++) 1666 1666 + XA_BUG_ON(xa, xa_get_order(xa, j) != order); 1667 1667 + xa_erase(xa, i << order); 1668 1668 + } 1669 1669 + } 1670 1670 + } 1671 1671 + 1695 1672 static noinline void check_destroy(struct xarray *xa) 1696 1673 { 1697 1674 unsigned long index; ··· 1760 1697 check_reserve(&array); 1761 1698 check_reserve(&xa0); 1762 1699 check_multi_store(&array); 1700 1700 + check_get_order(&array); 1763 1701 check_xa_alloc(); 1764 1702 check_find(&array); 1765 1703 check_find_entry(&array); ··· 1772 1708 check_store_range(&array); 1773 1709 check_store_iter(&array); 1774 1710 check_align(&xa0); 1711 1711 + check_split(&array); 1775 1712 1776 1713 check_workingset(&array, 0); 1777 1714 check_workingset(&array, 64);

+4 -1

lib/usercopy.c

reviewed

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 #include <linux/bitops.h> 3 3 + #include <linux/fault-inject-usercopy.h> 3 4 #include <linux/instrumented.h> 4 5 #include <linux/uaccess.h> 5 6 ··· 11 10 { 12 11 unsigned long res = n; 13 12 might_fault(); 14 14 - if (likely(access_ok(from, n))) { 13 13 + if (!should_fail_usercopy() && likely(access_ok(from, n))) { 15 14 instrument_copy_from_user(to, from, n); 16 15 res = raw_copy_from_user(to, from, n); 17 16 } ··· 26 25 unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) 27 26 { 28 27 might_fault(); 28 28 + if (should_fail_usercopy()) 29 29 + return n; 29 30 if (likely(access_ok(to, n))) { 30 31 instrument_copy_to_user(to, from, n); 31 32 n = raw_copy_to_user(to, from, n);

+199 -9

lib/xarray.c

reviewed

··· 266 266 */ 267 267 static void xas_destroy(struct xa_state *xas) 268 268 { 269 269 - struct xa_node *node = xas->xa_alloc; 269 269 + struct xa_node *next, *node = xas->xa_alloc; 270 270 271 271 - if (!node) 272 272 - return; 273 273 - XA_NODE_BUG_ON(node, !list_empty(&node->private_list)); 274 274 - kmem_cache_free(radix_tree_node_cachep, node); 275 275 - xas->xa_alloc = NULL; 271 271 + while (node) { 272 272 + XA_NODE_BUG_ON(node, !list_empty(&node->private_list)); 273 273 + next = rcu_dereference_raw(node->parent); 274 274 + radix_tree_node_rcu_free(&node->rcu_head); 275 275 + xas->xa_alloc = node = next; 276 276 + } 276 277 } 277 278 278 279 /** ··· 305 304 xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep, gfp); 306 305 if (!xas->xa_alloc) 307 306 return false; 307 307 + xas->xa_alloc->parent = NULL; 308 308 XA_NODE_BUG_ON(xas->xa_alloc, !list_empty(&xas->xa_alloc->private_list)); 309 309 xas->xa_node = XAS_RESTART; 310 310 return true; ··· 341 339 } 342 340 if (!xas->xa_alloc) 343 341 return false; 342 342 + xas->xa_alloc->parent = NULL; 344 343 XA_NODE_BUG_ON(xas->xa_alloc, !list_empty(&xas->xa_alloc->private_list)); 345 344 xas->xa_node = XAS_RESTART; 346 345 return true; ··· 406 403 /* 407 404 * Use this to calculate the maximum index that will need to be created 408 405 * in order to add the entry described by @xas. Because we cannot store a 409 409 - * multiple-index entry at index 0, the calculation is a little more complex 406 406 + * multi-index entry at index 0, the calculation is a little more complex 410 407 * than you might expect. 411 408 */ 412 409 static unsigned long xas_max(struct xa_state *xas) ··· 949 946 } 950 947 EXPORT_SYMBOL_GPL(xas_init_marks); 951 948 949 949 + #ifdef CONFIG_XARRAY_MULTI 950 950 + static unsigned int node_get_marks(struct xa_node *node, unsigned int offset) 951 951 + { 952 952 + unsigned int marks = 0; 953 953 + xa_mark_t mark = XA_MARK_0; 954 954 + 955 955 + for (;;) { 956 956 + if (node_get_mark(node, offset, mark)) 957 957 + marks |= 1 << (__force unsigned int)mark; 958 958 + if (mark == XA_MARK_MAX) 959 959 + break; 960 960 + mark_inc(mark); 961 961 + } 962 962 + 963 963 + return marks; 964 964 + } 965 965 + 966 966 + static void node_set_marks(struct xa_node *node, unsigned int offset, 967 967 + struct xa_node *child, unsigned int marks) 968 968 + { 969 969 + xa_mark_t mark = XA_MARK_0; 970 970 + 971 971 + for (;;) { 972 972 + if (marks & (1 << (__force unsigned int)mark)) { 973 973 + node_set_mark(node, offset, mark); 974 974 + if (child) 975 975 + node_mark_all(child, mark); 976 976 + } 977 977 + if (mark == XA_MARK_MAX) 978 978 + break; 979 979 + mark_inc(mark); 980 980 + } 981 981 + } 982 982 + 983 983 + /** 984 984 + * xas_split_alloc() - Allocate memory for splitting an entry. 985 985 + * @xas: XArray operation state. 986 986 + * @entry: New entry which will be stored in the array. 987 987 + * @order: New entry order. 988 988 + * @gfp: Memory allocation flags. 989 989 + * 990 990 + * This function should be called before calling xas_split(). 991 991 + * If necessary, it will allocate new nodes (and fill them with @entry) 992 992 + * to prepare for the upcoming split of an entry of @order size into 993 993 + * entries of the order stored in the @xas. 994 994 + * 995 995 + * Context: May sleep if @gfp flags permit. 996 996 + */ 997 997 + void xas_split_alloc(struct xa_state *xas, void *entry, unsigned int order, 998 998 + gfp_t gfp) 999 999 + { 1000 1000 + unsigned int sibs = (1 << (order % XA_CHUNK_SHIFT)) - 1; 1001 1001 + unsigned int mask = xas->xa_sibs; 1002 1002 + 1003 1003 + /* XXX: no support for splitting really large entries yet */ 1004 1004 + if (WARN_ON(xas->xa_shift + 2 * XA_CHUNK_SHIFT < order)) 1005 1005 + goto nomem; 1006 1006 + if (xas->xa_shift + XA_CHUNK_SHIFT > order) 1007 1007 + return; 1008 1008 + 1009 1009 + do { 1010 1010 + unsigned int i; 1011 1011 + void *sibling; 1012 1012 + struct xa_node *node; 1013 1013 + 1014 1014 + node = kmem_cache_alloc(radix_tree_node_cachep, gfp); 1015 1015 + if (!node) 1016 1016 + goto nomem; 1017 1017 + node->array = xas->xa; 1018 1018 + for (i = 0; i < XA_CHUNK_SIZE; i++) { 1019 1019 + if ((i & mask) == 0) { 1020 1020 + RCU_INIT_POINTER(node->slots[i], entry); 1021 1021 + sibling = xa_mk_sibling(0); 1022 1022 + } else { 1023 1023 + RCU_INIT_POINTER(node->slots[i], sibling); 1024 1024 + } 1025 1025 + } 1026 1026 + RCU_INIT_POINTER(node->parent, xas->xa_alloc); 1027 1027 + xas->xa_alloc = node; 1028 1028 + } while (sibs-- > 0); 1029 1029 + 1030 1030 + return; 1031 1031 + nomem: 1032 1032 + xas_destroy(xas); 1033 1033 + xas_set_err(xas, -ENOMEM); 1034 1034 + } 1035 1035 + EXPORT_SYMBOL_GPL(xas_split_alloc); 1036 1036 + 1037 1037 + /** 1038 1038 + * xas_split() - Split a multi-index entry into smaller entries. 1039 1039 + * @xas: XArray operation state. 1040 1040 + * @entry: New entry to store in the array. 1041 1041 + * @order: New entry order. 1042 1042 + * 1043 1043 + * The value in the entry is copied to all the replacement entries. 1044 1044 + * 1045 1045 + * Context: Any context. The caller should hold the xa_lock. 1046 1046 + */ 1047 1047 + void xas_split(struct xa_state *xas, void *entry, unsigned int order) 1048 1048 + { 1049 1049 + unsigned int sibs = (1 << (order % XA_CHUNK_SHIFT)) - 1; 1050 1050 + unsigned int offset, marks; 1051 1051 + struct xa_node *node; 1052 1052 + void *curr = xas_load(xas); 1053 1053 + int values = 0; 1054 1054 + 1055 1055 + node = xas->xa_node; 1056 1056 + if (xas_top(node)) 1057 1057 + return; 1058 1058 + 1059 1059 + marks = node_get_marks(node, xas->xa_offset); 1060 1060 + 1061 1061 + offset = xas->xa_offset + sibs; 1062 1062 + do { 1063 1063 + if (xas->xa_shift < node->shift) { 1064 1064 + struct xa_node *child = xas->xa_alloc; 1065 1065 + 1066 1066 + xas->xa_alloc = rcu_dereference_raw(child->parent); 1067 1067 + child->shift = node->shift - XA_CHUNK_SHIFT; 1068 1068 + child->offset = offset; 1069 1069 + child->count = XA_CHUNK_SIZE; 1070 1070 + child->nr_values = xa_is_value(entry) ? 1071 1071 + XA_CHUNK_SIZE : 0; 1072 1072 + RCU_INIT_POINTER(child->parent, node); 1073 1073 + node_set_marks(node, offset, child, marks); 1074 1074 + rcu_assign_pointer(node->slots[offset], 1075 1075 + xa_mk_node(child)); 1076 1076 + if (xa_is_value(curr)) 1077 1077 + values--; 1078 1078 + } else { 1079 1079 + unsigned int canon = offset - xas->xa_sibs; 1080 1080 + 1081 1081 + node_set_marks(node, canon, NULL, marks); 1082 1082 + rcu_assign_pointer(node->slots[canon], entry); 1083 1083 + while (offset > canon) 1084 1084 + rcu_assign_pointer(node->slots[offset--], 1085 1085 + xa_mk_sibling(canon)); 1086 1086 + values += (xa_is_value(entry) - xa_is_value(curr)) * 1087 1087 + (xas->xa_sibs + 1); 1088 1088 + } 1089 1089 + } while (offset-- > xas->xa_offset); 1090 1090 + 1091 1091 + node->nr_values += values; 1092 1092 + } 1093 1093 + EXPORT_SYMBOL_GPL(xas_split); 1094 1094 + #endif 1095 1095 + 952 1096 /** 953 1097 * xas_pause() - Pause a walk to drop a lock. 954 1098 * @xas: XArray operation state. ··· 1557 1407 * @gfp: Memory allocation flags. 1558 1408 * 1559 1409 * After this function returns, loads from this index will return @entry. 1560 1560 - * Storing into an existing multislot entry updates the entry of every index. 1410 1410 + * Storing into an existing multi-index entry updates the entry of every index. 1561 1411 * The marks associated with @index are unaffected unless @entry is %NULL. 1562 1412 * 1563 1413 * Context: Any context. Takes and releases the xa_lock. ··· 1699 1549 * 1700 1550 * After this function returns, loads from any index between @first and @last, 1701 1551 * inclusive will return @entry. 1702 1702 - * Storing into an existing multislot entry updates the entry of every index. 1552 1552 + * Storing into an existing multi-index entry updates the entry of every index. 1703 1553 * The marks associated with @index are unaffected unless @entry is %NULL. 1704 1554 * 1705 1555 * Context: Process context. Takes and releases the xa_lock. May sleep ··· 1742 1592 return xas_result(&xas, NULL); 1743 1593 } 1744 1594 EXPORT_SYMBOL(xa_store_range); 1595 1595 + 1596 1596 + /** 1597 1597 + * xa_get_order() - Get the order of an entry. 1598 1598 + * @xa: XArray. 1599 1599 + * @index: Index of the entry. 1600 1600 + * 1601 1601 + * Return: A number between 0 and 63 indicating the order of the entry. 1602 1602 + */ 1603 1603 + int xa_get_order(struct xarray *xa, unsigned long index) 1604 1604 + { 1605 1605 + XA_STATE(xas, xa, index); 1606 1606 + void *entry; 1607 1607 + int order = 0; 1608 1608 + 1609 1609 + rcu_read_lock(); 1610 1610 + entry = xas_load(&xas); 1611 1611 + 1612 1612 + if (!entry) 1613 1613 + goto unlock; 1614 1614 + 1615 1615 + if (!xas.xa_node) 1616 1616 + goto unlock; 1617 1617 + 1618 1618 + for (;;) { 1619 1619 + unsigned int slot = xas.xa_offset + (1 << order); 1620 1620 + 1621 1621 + if (slot >= XA_CHUNK_SIZE) 1622 1622 + break; 1623 1623 + if (!xa_is_sibling(xas.xa_node->slots[slot])) 1624 1624 + break; 1625 1625 + order++; 1626 1626 + } 1627 1627 + 1628 1628 + order += xas.xa_node->shift; 1629 1629 + unlock: 1630 1630 + rcu_read_unlock(); 1631 1631 + 1632 1632 + return order; 1633 1633 + } 1634 1634 + EXPORT_SYMBOL(xa_get_order); 1745 1635 #endif /* CONFIG_XARRAY_MULTI */ 1746 1636 1747 1637 /**

+1 -1

mm/Kconfig

reviewed

··· 152 152 # eventually, we can have this option just 'select SPARSEMEM' 153 153 config MEMORY_HOTPLUG 154 154 bool "Allow for memory hot-add" 155 155 + select MEMORY_ISOLATION 155 156 depends on SPARSEMEM || X86_64_ACPI_NUMA 156 157 depends on ARCH_ENABLE_MEMORY_HOTPLUG 157 158 depends on 64BIT || BROKEN ··· 179 178 180 179 config MEMORY_HOTREMOVE 181 180 bool "Allow for memory hot remove" 182 182 - select MEMORY_ISOLATION 183 181 select HAVE_BOOTMEM_INFO_NODE if (X86_64 || PPC64) 184 182 depends on MEMORY_HOTPLUG && ARCH_ENABLE_MEMORY_HOTREMOVE 185 183 depends on MIGRATION

+3 -3

mm/compaction.c

reviewed

··· 625 625 } 626 626 627 627 /* Found a free page, will break it into order-0 pages */ 628 628 - order = page_order(page); 628 628 + order = buddy_order(page); 629 629 isolated = __isolate_free_page(page, order); 630 630 if (!isolated) 631 631 break; ··· 898 898 * potential isolation targets. 899 899 */ 900 900 if (PageBuddy(page)) { 901 901 - unsigned long freepage_order = page_order_unsafe(page); 901 901 + unsigned long freepage_order = buddy_order_unsafe(page); 902 902 903 903 /* 904 904 * Without lock, we cannot be sure that what we got is ··· 1172 1172 * the only small danger is that we skip a potentially suitable 1173 1173 * pageblock, so it's not worth to check order for valid range. 1174 1174 */ 1175 1175 - if (page_order_unsafe(page) >= pageblock_order) 1175 1175 + if (buddy_order_unsafe(page) >= pageblock_order) 1176 1176 return false; 1177 1177 } 1178 1178

+101 -106

mm/debug_vm_pgtable.c

reviewed

··· 28 28 #include <linux/swapops.h> 29 29 #include <linux/start_kernel.h> 30 30 #include <linux/sched/mm.h> 31 31 + #include <linux/io.h> 31 32 #include <asm/pgalloc.h> 32 33 #include <asm/tlbflush.h> 33 34 ··· 45 44 * entry type. But these bits might affect the ability to clear entries with 46 45 * pxx_clear() because of how dynamic page table folding works on s390. So 47 46 * while loading up the entries do not change the lower 4 bits. It does not 48 48 - * have affect any other platform. 47 47 + * have affect any other platform. Also avoid the 62nd bit on ppc64 that is 48 48 + * used to mark a pte entry. 49 49 */ 50 50 - #define S390_MASK_BITS 4 51 51 - #define RANDOM_ORVALUE GENMASK(BITS_PER_LONG - 1, S390_MASK_BITS) 50 50 + #define S390_SKIP_MASK GENMASK(3, 0) 51 51 + #if __BITS_PER_LONG == 64 52 52 + #define PPC64_SKIP_MASK GENMASK(62, 62) 53 53 + #else 54 54 + #define PPC64_SKIP_MASK 0x0 55 55 + #endif 56 56 + #define ARCH_SKIP_MASK (S390_SKIP_MASK | PPC64_SKIP_MASK) 57 57 + #define RANDOM_ORVALUE (GENMASK(BITS_PER_LONG - 1, 0) & ~ARCH_SKIP_MASK) 52 58 #define RANDOM_NZVALUE GENMASK(7, 0) 53 59 54 60 static void __init pte_basic_tests(unsigned long pfn, pgprot_t prot) ··· 79 71 { 80 72 pte_t pte = pfn_pte(pfn, prot); 81 73 74 74 + /* 75 75 + * Architectures optimize set_pte_at by avoiding TLB flush. 76 76 + * This requires set_pte_at to be not used to update an 77 77 + * existing pte entry. Clear pte before we do set_pte_at 78 78 + */ 79 79 + 82 80 pr_debug("Validating PTE advanced\n"); 83 81 pte = pfn_pte(pfn, prot); 84 82 set_pte_at(mm, vaddr, ptep, pte); 85 83 ptep_set_wrprotect(mm, vaddr, ptep); 86 84 pte = ptep_get(ptep); 87 85 WARN_ON(pte_write(pte)); 88 88 - 89 89 - pte = pfn_pte(pfn, prot); 90 90 - set_pte_at(mm, vaddr, ptep, pte); 91 86 ptep_get_and_clear(mm, vaddr, ptep); 92 87 pte = ptep_get(ptep); 93 88 WARN_ON(!pte_none(pte)); ··· 104 93 ptep_set_access_flags(vma, vaddr, ptep, pte, 1); 105 94 pte = ptep_get(ptep); 106 95 WARN_ON(!(pte_write(pte) && pte_dirty(pte))); 107 107 - 108 108 - pte = pfn_pte(pfn, prot); 109 109 - set_pte_at(mm, vaddr, ptep, pte); 110 96 ptep_get_and_clear_full(mm, vaddr, ptep, 1); 111 97 pte = ptep_get(ptep); 112 98 WARN_ON(!pte_none(pte)); 113 99 100 100 + pte = pfn_pte(pfn, prot); 114 101 pte = pte_mkyoung(pte); 115 102 set_pte_at(mm, vaddr, ptep, pte); 116 103 ptep_test_and_clear_young(vma, vaddr, ptep); ··· 120 111 { 121 112 pte_t pte = pfn_pte(pfn, prot); 122 113 114 114 + if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) 115 115 + return; 116 116 + 123 117 pr_debug("Validating PTE saved write\n"); 124 118 WARN_ON(!pte_savedwrite(pte_mk_savedwrite(pte_clear_savedwrite(pte)))); 125 119 WARN_ON(pte_savedwrite(pte_clear_savedwrite(pte_mk_savedwrite(pte)))); 126 120 } 121 121 + 127 122 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 128 123 static void __init pmd_basic_tests(unsigned long pfn, pgprot_t prot) 129 124 { ··· 154 141 static void __init pmd_advanced_tests(struct mm_struct *mm, 155 142 struct vm_area_struct *vma, pmd_t *pmdp, 156 143 unsigned long pfn, unsigned long vaddr, 157 157 - pgprot_t prot) 144 144 + pgprot_t prot, pgtable_t pgtable) 158 145 { 159 146 pmd_t pmd = pfn_pmd(pfn, prot); 160 147 ··· 165 152 /* Align the address wrt HPAGE_PMD_SIZE */ 166 153 vaddr = (vaddr & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE; 167 154 155 155 + pgtable_trans_huge_deposit(mm, pmdp, pgtable); 156 156 + 168 157 pmd = pfn_pmd(pfn, prot); 169 158 set_pmd_at(mm, vaddr, pmdp, pmd); 170 159 pmdp_set_wrprotect(mm, vaddr, pmdp); 171 160 pmd = READ_ONCE(*pmdp); 172 161 WARN_ON(pmd_write(pmd)); 173 173 - 174 174 - pmd = pfn_pmd(pfn, prot); 175 175 - set_pmd_at(mm, vaddr, pmdp, pmd); 176 162 pmdp_huge_get_and_clear(mm, vaddr, pmdp); 177 163 pmd = READ_ONCE(*pmdp); 178 164 WARN_ON(!pmd_none(pmd)); ··· 185 173 pmdp_set_access_flags(vma, vaddr, pmdp, pmd, 1); 186 174 pmd = READ_ONCE(*pmdp); 187 175 WARN_ON(!(pmd_write(pmd) && pmd_dirty(pmd))); 188 188 - 189 189 - pmd = pmd_mkhuge(pfn_pmd(pfn, prot)); 190 190 - set_pmd_at(mm, vaddr, pmdp, pmd); 191 176 pmdp_huge_get_and_clear_full(vma, vaddr, pmdp, 1); 192 177 pmd = READ_ONCE(*pmdp); 193 178 WARN_ON(!pmd_none(pmd)); 194 179 180 180 + pmd = pmd_mkhuge(pfn_pmd(pfn, prot)); 195 181 pmd = pmd_mkyoung(pmd); 196 182 set_pmd_at(mm, vaddr, pmdp, pmd); 197 183 pmdp_test_and_clear_young(vma, vaddr, pmdp); 198 184 pmd = READ_ONCE(*pmdp); 199 185 WARN_ON(pmd_young(pmd)); 186 186 + 187 187 + /* Clear the pte entries */ 188 188 + pmdp_huge_get_and_clear(mm, vaddr, pmdp); 189 189 + pgtable = pgtable_trans_huge_withdraw(mm, pmdp); 200 190 } 201 191 202 192 static void __init pmd_leaf_tests(unsigned long pfn, pgprot_t prot) ··· 213 199 WARN_ON(!pmd_leaf(pmd)); 214 200 } 215 201 202 202 + #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 216 203 static void __init pmd_huge_tests(pmd_t *pmdp, unsigned long pfn, pgprot_t prot) 217 204 { 218 205 pmd_t pmd; 219 206 220 220 - if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP)) 207 207 + if (!arch_ioremap_pmd_supported()) 221 208 return; 222 209 223 210 pr_debug("Validating PMD huge\n"); ··· 232 217 pmd = READ_ONCE(*pmdp); 233 218 WARN_ON(!pmd_none(pmd)); 234 219 } 220 220 + #else /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 221 221 + static void __init pmd_huge_tests(pmd_t *pmdp, unsigned long pfn, pgprot_t prot) { } 222 222 + #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 235 223 236 224 static void __init pmd_savedwrite_tests(unsigned long pfn, pgprot_t prot) 237 225 { 238 226 pmd_t pmd = pfn_pmd(pfn, prot); 227 227 + 228 228 + if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) 229 229 + return; 239 230 240 231 pr_debug("Validating PMD saved write\n"); 241 232 WARN_ON(!pmd_savedwrite(pmd_mk_savedwrite(pmd_clear_savedwrite(pmd)))); ··· 293 272 WARN_ON(pud_write(pud)); 294 273 295 274 #ifndef __PAGETABLE_PMD_FOLDED 296 296 - pud = pfn_pud(pfn, prot); 297 297 - set_pud_at(mm, vaddr, pudp, pud); 298 275 pudp_huge_get_and_clear(mm, vaddr, pudp); 299 299 - pud = READ_ONCE(*pudp); 300 300 - WARN_ON(!pud_none(pud)); 301 301 - 302 302 - pud = pfn_pud(pfn, prot); 303 303 - set_pud_at(mm, vaddr, pudp, pud); 304 304 - pudp_huge_get_and_clear_full(mm, vaddr, pudp, 1); 305 276 pud = READ_ONCE(*pudp); 306 277 WARN_ON(!pud_none(pud)); 307 278 #endif /* __PAGETABLE_PMD_FOLDED */ ··· 307 294 pud = READ_ONCE(*pudp); 308 295 WARN_ON(!(pud_write(pud) && pud_dirty(pud))); 309 296 297 297 + #ifndef __PAGETABLE_PMD_FOLDED 298 298 + pudp_huge_get_and_clear_full(mm, vaddr, pudp, 1); 299 299 + pud = READ_ONCE(*pudp); 300 300 + WARN_ON(!pud_none(pud)); 301 301 + #endif /* __PAGETABLE_PMD_FOLDED */ 302 302 + 303 303 + pud = pfn_pud(pfn, prot); 310 304 pud = pud_mkyoung(pud); 311 305 set_pud_at(mm, vaddr, pudp, pud); 312 306 pudp_test_and_clear_young(vma, vaddr, pudp); 313 307 pud = READ_ONCE(*pudp); 314 308 WARN_ON(pud_young(pud)); 309 309 + 310 310 + pudp_huge_get_and_clear(mm, vaddr, pudp); 315 311 } 316 312 317 313 static void __init pud_leaf_tests(unsigned long pfn, pgprot_t prot) ··· 335 313 WARN_ON(!pud_leaf(pud)); 336 314 } 337 315 316 316 + #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 338 317 static void __init pud_huge_tests(pud_t *pudp, unsigned long pfn, pgprot_t prot) 339 318 { 340 319 pud_t pud; 341 320 342 342 - if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP)) 321 321 + if (!arch_ioremap_pud_supported()) 343 322 return; 344 323 345 324 pr_debug("Validating PUD huge\n"); ··· 354 331 pud = READ_ONCE(*pudp); 355 332 WARN_ON(!pud_none(pud)); 356 333 } 334 334 + #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ 335 335 + static void __init pud_huge_tests(pud_t *pudp, unsigned long pfn, pgprot_t prot) { } 336 336 + #endif /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ 337 337 + 357 338 #else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 358 339 static void __init pud_basic_tests(unsigned long pfn, pgprot_t prot) { } 359 340 static void __init pud_advanced_tests(struct mm_struct *mm, ··· 377 350 static void __init pmd_advanced_tests(struct mm_struct *mm, 378 351 struct vm_area_struct *vma, pmd_t *pmdp, 379 352 unsigned long pfn, unsigned long vaddr, 380 380 - pgprot_t prot) 353 353 + pgprot_t prot, pgtable_t pgtable) 381 354 { 382 355 } 383 356 static void __init pud_advanced_tests(struct mm_struct *mm, ··· 444 417 * This entry points to next level page table page. 445 418 * Hence this must not qualify as pud_bad(). 446 419 */ 447 447 - pmd_clear(pmdp); 448 448 - pud_clear(pudp); 449 420 pud_populate(mm, pudp, pmdp); 450 421 pud = READ_ONCE(*pudp); 451 422 WARN_ON(pud_bad(pud)); ··· 540 515 #endif /* PAGETABLE_P4D_FOLDED */ 541 516 542 517 static void __init pte_clear_tests(struct mm_struct *mm, pte_t *ptep, 543 543 - unsigned long vaddr) 518 518 + unsigned long pfn, unsigned long vaddr, 519 519 + pgprot_t prot) 544 520 { 545 545 - pte_t pte = ptep_get(ptep); 521 521 + pte_t pte = pfn_pte(pfn, prot); 546 522 547 523 pr_debug("Validating PTE clear\n"); 524 524 + #ifndef CONFIG_RISCV 548 525 pte = __pte(pte_val(pte) | RANDOM_ORVALUE); 526 526 + #endif 549 527 set_pte_at(mm, vaddr, ptep, pte); 550 528 barrier(); 551 529 pte_clear(mm, vaddr, ptep); ··· 578 550 * This entry points to next level page table page. 579 551 * Hence this must not qualify as pmd_bad(). 580 552 */ 581 581 - pmd_clear(pmdp); 582 553 pmd_populate(mm, pmdp, pgtable); 583 554 pmd = READ_ONCE(*pmdp); 584 555 WARN_ON(pmd_bad(pmd)); ··· 811 784 WARN_ON(!pte_huge(pte_mkhuge(pte))); 812 785 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 813 786 } 814 814 - 815 815 - static void __init hugetlb_advanced_tests(struct mm_struct *mm, 816 816 - struct vm_area_struct *vma, 817 817 - pte_t *ptep, unsigned long pfn, 818 818 - unsigned long vaddr, pgprot_t prot) 819 819 - { 820 820 - struct page *page = pfn_to_page(pfn); 821 821 - pte_t pte = ptep_get(ptep); 822 822 - unsigned long paddr = __pfn_to_phys(pfn) & PMD_MASK; 823 823 - 824 824 - pr_debug("Validating HugeTLB advanced\n"); 825 825 - pte = pte_mkhuge(mk_pte(pfn_to_page(PHYS_PFN(paddr)), prot)); 826 826 - set_huge_pte_at(mm, vaddr, ptep, pte); 827 827 - barrier(); 828 828 - WARN_ON(!pte_same(pte, huge_ptep_get(ptep))); 829 829 - huge_pte_clear(mm, vaddr, ptep, PMD_SIZE); 830 830 - pte = huge_ptep_get(ptep); 831 831 - WARN_ON(!huge_pte_none(pte)); 832 832 - 833 833 - pte = mk_huge_pte(page, prot); 834 834 - set_huge_pte_at(mm, vaddr, ptep, pte); 835 835 - barrier(); 836 836 - huge_ptep_set_wrprotect(mm, vaddr, ptep); 837 837 - pte = huge_ptep_get(ptep); 838 838 - WARN_ON(huge_pte_write(pte)); 839 839 - 840 840 - pte = mk_huge_pte(page, prot); 841 841 - set_huge_pte_at(mm, vaddr, ptep, pte); 842 842 - barrier(); 843 843 - huge_ptep_get_and_clear(mm, vaddr, ptep); 844 844 - pte = huge_ptep_get(ptep); 845 845 - WARN_ON(!huge_pte_none(pte)); 846 846 - 847 847 - pte = mk_huge_pte(page, prot); 848 848 - pte = huge_pte_wrprotect(pte); 849 849 - set_huge_pte_at(mm, vaddr, ptep, pte); 850 850 - barrier(); 851 851 - pte = huge_pte_mkwrite(pte); 852 852 - pte = huge_pte_mkdirty(pte); 853 853 - huge_ptep_set_access_flags(vma, vaddr, ptep, pte, 1); 854 854 - pte = huge_ptep_get(ptep); 855 855 - WARN_ON(!(huge_pte_write(pte) && huge_pte_dirty(pte))); 856 856 - } 857 787 #else /* !CONFIG_HUGETLB_PAGE */ 858 788 static void __init hugetlb_basic_tests(unsigned long pfn, pgprot_t prot) { } 859 859 - static void __init hugetlb_advanced_tests(struct mm_struct *mm, 860 860 - struct vm_area_struct *vma, 861 861 - pte_t *ptep, unsigned long pfn, 862 862 - unsigned long vaddr, pgprot_t prot) 863 863 - { 864 864 - } 865 789 #endif /* CONFIG_HUGETLB_PAGE */ 866 790 867 791 #ifdef CONFIG_TRANSPARENT_HUGEPAGE ··· 943 965 p4dp = p4d_alloc(mm, pgdp, vaddr); 944 966 pudp = pud_alloc(mm, p4dp, vaddr); 945 967 pmdp = pmd_alloc(mm, pudp, vaddr); 946 946 - ptep = pte_alloc_map_lock(mm, pmdp, vaddr, &ptl); 968 968 + /* 969 969 + * Allocate pgtable_t 970 970 + */ 971 971 + if (pte_alloc(mm, pmdp)) { 972 972 + pr_err("pgtable allocation failed\n"); 973 973 + return 1; 974 974 + } 947 975 948 976 /* 949 977 * Save all the page table page addresses as the page table ··· 969 985 p4d_basic_tests(p4d_aligned, prot); 970 986 pgd_basic_tests(pgd_aligned, prot); 971 987 972 972 - pte_clear_tests(mm, ptep, vaddr); 973 973 - pmd_clear_tests(mm, pmdp); 974 974 - pud_clear_tests(mm, pudp); 975 975 - p4d_clear_tests(mm, p4dp); 976 976 - pgd_clear_tests(mm, pgdp); 977 977 - 978 978 - pte_advanced_tests(mm, vma, ptep, pte_aligned, vaddr, prot); 979 979 - pmd_advanced_tests(mm, vma, pmdp, pmd_aligned, vaddr, prot); 980 980 - pud_advanced_tests(mm, vma, pudp, pud_aligned, vaddr, prot); 981 981 - hugetlb_advanced_tests(mm, vma, ptep, pte_aligned, vaddr, prot); 982 982 - 983 988 pmd_leaf_tests(pmd_aligned, prot); 984 989 pud_leaf_tests(pud_aligned, prot); 985 990 986 986 - pmd_huge_tests(pmdp, pmd_aligned, prot); 987 987 - pud_huge_tests(pudp, pud_aligned, prot); 988 988 - 989 989 - pte_savedwrite_tests(pte_aligned, prot); 990 990 - pmd_savedwrite_tests(pmd_aligned, prot); 991 991 - 992 992 - pte_unmap_unlock(ptep, ptl); 993 993 - 994 994 - pmd_populate_tests(mm, pmdp, saved_ptep); 995 995 - pud_populate_tests(mm, pudp, saved_pmdp); 996 996 - p4d_populate_tests(mm, p4dp, saved_pudp); 997 997 - pgd_populate_tests(mm, pgdp, saved_p4dp); 991 991 + pte_savedwrite_tests(pte_aligned, protnone); 992 992 + pmd_savedwrite_tests(pmd_aligned, protnone); 998 993 999 994 pte_special_tests(pte_aligned, prot); 1000 995 pte_protnone_tests(pte_aligned, protnone); ··· 992 1029 pmd_swap_tests(pmd_aligned, prot); 993 1030 994 1031 swap_migration_tests(); 995 995 - hugetlb_basic_tests(pte_aligned, prot); 996 1032 997 1033 pmd_thp_tests(pmd_aligned, prot); 998 1034 pud_thp_tests(pud_aligned, prot); 1035 1035 + 1036 1036 + hugetlb_basic_tests(pte_aligned, prot); 1037 1037 + 1038 1038 + /* 1039 1039 + * Page table modifying tests. They need to hold 1040 1040 + * proper page table lock. 1041 1041 + */ 1042 1042 + 1043 1043 + ptep = pte_offset_map_lock(mm, pmdp, vaddr, &ptl); 1044 1044 + pte_clear_tests(mm, ptep, pte_aligned, vaddr, prot); 1045 1045 + pte_advanced_tests(mm, vma, ptep, pte_aligned, vaddr, prot); 1046 1046 + pte_unmap_unlock(ptep, ptl); 1047 1047 + 1048 1048 + ptl = pmd_lock(mm, pmdp); 1049 1049 + pmd_clear_tests(mm, pmdp); 1050 1050 + pmd_advanced_tests(mm, vma, pmdp, pmd_aligned, vaddr, prot, saved_ptep); 1051 1051 + pmd_huge_tests(pmdp, pmd_aligned, prot); 1052 1052 + pmd_populate_tests(mm, pmdp, saved_ptep); 1053 1053 + spin_unlock(ptl); 1054 1054 + 1055 1055 + ptl = pud_lock(mm, pudp); 1056 1056 + pud_clear_tests(mm, pudp); 1057 1057 + pud_advanced_tests(mm, vma, pudp, pud_aligned, vaddr, prot); 1058 1058 + pud_huge_tests(pudp, pud_aligned, prot); 1059 1059 + pud_populate_tests(mm, pudp, saved_pmdp); 1060 1060 + spin_unlock(ptl); 1061 1061 + 1062 1062 + spin_lock(&mm->page_table_lock); 1063 1063 + p4d_clear_tests(mm, p4dp); 1064 1064 + pgd_clear_tests(mm, pgdp); 1065 1065 + p4d_populate_tests(mm, p4dp, saved_pudp); 1066 1066 + pgd_populate_tests(mm, pgdp, saved_p4dp); 1067 1067 + spin_unlock(&mm->page_table_lock); 999 1068 1000 1069 p4d_free(mm, saved_p4dp); 1001 1070 pud_free(mm, saved_pudp);

+39 -19

mm/filemap.c

reviewed

··· 249 249 freepage(page); 250 250 251 251 if (PageTransHuge(page) && !PageHuge(page)) { 252 252 - page_ref_sub(page, HPAGE_PMD_NR); 252 252 + page_ref_sub(page, thp_nr_pages(page)); 253 253 VM_BUG_ON_PAGE(page_count(page) <= 0, page); 254 254 } else { 255 255 put_page(page); ··· 829 829 830 830 noinline int __add_to_page_cache_locked(struct page *page, 831 831 struct address_space *mapping, 832 832 - pgoff_t offset, gfp_t gfp_mask, 832 832 + pgoff_t offset, gfp_t gfp, 833 833 void **shadowp) 834 834 { 835 835 XA_STATE(xas, &mapping->i_pages, offset); 836 836 int huge = PageHuge(page); 837 837 int error; 838 838 - void *old; 839 838 840 839 VM_BUG_ON_PAGE(!PageLocked(page), page); 841 840 VM_BUG_ON_PAGE(PageSwapBacked(page), page); ··· 845 846 page->index = offset; 846 847 847 848 if (!huge) { 848 848 - error = mem_cgroup_charge(page, current->mm, gfp_mask); 849 849 + error = mem_cgroup_charge(page, current->mm, gfp); 849 850 if (error) 850 851 goto error; 851 852 } 852 853 854 854 + gfp &= GFP_RECLAIM_MASK; 855 855 + 853 856 do { 857 857 + unsigned int order = xa_get_order(xas.xa, xas.xa_index); 858 858 + void *entry, *old = NULL; 859 859 + 860 860 + if (order > thp_order(page)) 861 861 + xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index), 862 862 + order, gfp); 854 863 xas_lock_irq(&xas); 855 855 - old = xas_load(&xas); 856 856 - if (old && !xa_is_value(old)) 857 857 - xas_set_err(&xas, -EEXIST); 864 864 + xas_for_each_conflict(&xas, entry) { 865 865 + old = entry; 866 866 + if (!xa_is_value(entry)) { 867 867 + xas_set_err(&xas, -EEXIST); 868 868 + goto unlock; 869 869 + } 870 870 + } 871 871 + 872 872 + if (old) { 873 873 + if (shadowp) 874 874 + *shadowp = old; 875 875 + /* entry may have been split before we acquired lock */ 876 876 + order = xa_get_order(xas.xa, xas.xa_index); 877 877 + if (order > thp_order(page)) { 878 878 + xas_split(&xas, old, order); 879 879 + xas_reset(&xas); 880 880 + } 881 881 + } 882 882 + 858 883 xas_store(&xas, page); 859 884 if (xas_error(&xas)) 860 885 goto unlock; 861 886 862 862 - if (xa_is_value(old)) { 887 887 + if (old) 863 888 mapping->nrexceptional--; 864 864 - if (shadowp) 865 865 - *shadowp = old; 866 866 - } 867 889 mapping->nrpages++; 868 890 869 891 /* hugetlb pages do not participate in page cache accounting */ ··· 892 872 __inc_lruvec_page_state(page, NR_FILE_PAGES); 893 873 unlock: 894 874 xas_unlock_irq(&xas); 895 895 - } while (xas_nomem(&xas, gfp_mask & GFP_RECLAIM_MASK)); 875 875 + } while (xas_nomem(&xas, gfp)); 896 876 897 877 if (xas_error(&xas)) { 898 878 error = xas_error(&xas); ··· 1445 1425 * unlock_page - unlock a locked page 1446 1426 * @page: the page 1447 1427 * 1448 1448 - * Unlocks the page and wakes up sleepers in ___wait_on_page_locked(). 1428 1428 + * Unlocks the page and wakes up sleepers in wait_on_page_locked(). 1449 1429 * Also wakes sleepers in wait_on_page_writeback() because the wakeup 1450 1430 * mechanism between PageLocked pages and PageWriteback pages is shared. 1451 1431 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep. ··· 2588 2568 struct file *file = vmf->vma->vm_file; 2589 2569 struct file_ra_state *ra = &file->f_ra; 2590 2570 struct address_space *mapping = file->f_mapping; 2571 2571 + DEFINE_READAHEAD(ractl, file, mapping, vmf->pgoff); 2591 2572 struct file *fpin = NULL; 2592 2592 - pgoff_t offset = vmf->pgoff; 2593 2573 unsigned int mmap_miss; 2594 2574 2595 2575 /* If we don't want any read-ahead, don't bother */ ··· 2600 2580 2601 2581 if (vmf->vma->vm_flags & VM_SEQ_READ) { 2602 2582 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 2603 2603 - page_cache_sync_readahead(mapping, ra, file, offset, 2604 2604 - ra->ra_pages); 2583 2583 + page_cache_sync_ra(&ractl, ra, ra->ra_pages); 2605 2584 return fpin; 2606 2585 } 2607 2586 ··· 2620 2601 * mmap read-around 2621 2602 */ 2622 2603 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 2623 2623 - ra->start = max_t(long, 0, offset - ra->ra_pages / 2); 2604 2604 + ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2); 2624 2605 ra->size = ra->ra_pages; 2625 2606 ra->async_size = ra->ra_pages / 4; 2626 2626 - ra_submit(ra, mapping, file); 2607 2607 + ractl._index = ra->start; 2608 2608 + do_page_cache_ra(&ractl, ra->size, ra->async_size); 2627 2609 return fpin; 2628 2610 } 2629 2611 ··· 3004 2984 goto out; 3005 2985 3006 2986 /* 3007 3007 - * Page is not up to date and may be locked due one of the following 2987 2987 + * Page is not up to date and may be locked due to one of the following 3008 2988 * case a: Page is being filled and the page lock is held 3009 2989 * case b: Read/write error clearing the page uptodate status 3010 2990 * case c: Truncation in progress (page locked)

+32 -29

mm/gup.c

reviewed

··· 1490 1490 mmap_read_unlock(mm); 1491 1491 return ret; /* 0 or negative error code */ 1492 1492 } 1493 1493 - 1494 1494 - /** 1495 1495 - * get_dump_page() - pin user page in memory while writing it to core dump 1496 1496 - * @addr: user address 1497 1497 - * 1498 1498 - * Returns struct page pointer of user page pinned for dump, 1499 1499 - * to be freed afterwards by put_page(). 1500 1500 - * 1501 1501 - * Returns NULL on any kind of failure - a hole must then be inserted into 1502 1502 - * the corefile, to preserve alignment with its headers; and also returns 1503 1503 - * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found - 1504 1504 - * allowing a hole to be left in the corefile to save diskspace. 1505 1505 - * 1506 1506 - * Called without mmap_lock, but after all other threads have been killed. 1507 1507 - */ 1508 1508 - #ifdef CONFIG_ELF_CORE 1509 1509 - struct page *get_dump_page(unsigned long addr) 1510 1510 - { 1511 1511 - struct vm_area_struct *vma; 1512 1512 - struct page *page; 1513 1513 - 1514 1514 - if (__get_user_pages(current->mm, addr, 1, 1515 1515 - FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma, 1516 1516 - NULL) < 1) 1517 1517 - return NULL; 1518 1518 - flush_cache_page(vma, addr, page_to_pfn(page)); 1519 1519 - return page; 1520 1520 - } 1521 1521 - #endif /* CONFIG_ELF_CORE */ 1522 1493 #else /* CONFIG_MMU */ 1523 1494 static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, 1524 1495 unsigned long nr_pages, struct page **pages, ··· 1534 1563 return i ? : -EFAULT; 1535 1564 } 1536 1565 #endif /* !CONFIG_MMU */ 1566 1566 + 1567 1567 + /** 1568 1568 + * get_dump_page() - pin user page in memory while writing it to core dump 1569 1569 + * @addr: user address 1570 1570 + * 1571 1571 + * Returns struct page pointer of user page pinned for dump, 1572 1572 + * to be freed afterwards by put_page(). 1573 1573 + * 1574 1574 + * Returns NULL on any kind of failure - a hole must then be inserted into 1575 1575 + * the corefile, to preserve alignment with its headers; and also returns 1576 1576 + * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found - 1577 1577 + * allowing a hole to be left in the corefile to save diskspace. 1578 1578 + * 1579 1579 + * Called without mmap_lock (takes and releases the mmap_lock by itself). 1580 1580 + */ 1581 1581 + #ifdef CONFIG_ELF_CORE 1582 1582 + struct page *get_dump_page(unsigned long addr) 1583 1583 + { 1584 1584 + struct mm_struct *mm = current->mm; 1585 1585 + struct page *page; 1586 1586 + int locked = 1; 1587 1587 + int ret; 1588 1588 + 1589 1589 + if (mmap_read_lock_killable(mm)) 1590 1590 + return NULL; 1591 1591 + ret = __get_user_pages_locked(mm, addr, 1, &page, NULL, &locked, 1592 1592 + FOLL_FORCE | FOLL_DUMP | FOLL_GET); 1593 1593 + if (locked) 1594 1594 + mmap_read_unlock(mm); 1595 1595 + return (ret == 1) ? page : NULL; 1596 1596 + } 1597 1597 + #endif /* CONFIG_ELF_CORE */ 1537 1598 1538 1599 #if defined(CONFIG_FS_DAX) || defined (CONFIG_CMA) 1539 1600 static bool check_dax_vmas(struct vm_area_struct **vmas, long nr_pages)

+2 -2

mm/highmem.c

reviewed

··· 369 369 } 370 370 371 371 EXPORT_SYMBOL(kunmap_high); 372 372 - #endif 372 372 + #endif /* CONFIG_HIGHMEM */ 373 373 374 374 #if defined(HASHED_PAGE_VIRTUAL) 375 375 ··· 481 481 } 482 482 } 483 483 484 484 - #endif /* defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) */ 484 484 + #endif /* defined(HASHED_PAGE_VIRTUAL) */

+24 -21

mm/huge_memory.c

reviewed

··· 2335 2335 VM_BUG_ON_PAGE(!unmap_success, page); 2336 2336 } 2337 2337 2338 2338 - static void remap_page(struct page *page) 2338 2338 + static void remap_page(struct page *page, unsigned int nr) 2339 2339 { 2340 2340 int i; 2341 2341 if (PageTransHuge(page)) { 2342 2342 remove_migration_ptes(page, page, true); 2343 2343 } else { 2344 2344 - for (i = 0; i < HPAGE_PMD_NR; i++) 2344 2344 + for (i = 0; i < nr; i++) 2345 2345 remove_migration_ptes(page + i, page + i, true); 2346 2346 } 2347 2347 } ··· 2419 2419 struct lruvec *lruvec; 2420 2420 struct address_space *swap_cache = NULL; 2421 2421 unsigned long offset = 0; 2422 2422 + unsigned int nr = thp_nr_pages(head); 2422 2423 int i; 2423 2424 2424 2425 lruvec = mem_cgroup_page_lruvec(head, pgdat); ··· 2435 2434 xa_lock(&swap_cache->i_pages); 2436 2435 } 2437 2436 2438 2438 - for (i = HPAGE_PMD_NR - 1; i >= 1; i--) { 2437 2437 + for (i = nr - 1; i >= 1; i--) { 2439 2438 __split_huge_page_tail(head, i, lruvec, list); 2440 2439 /* Some pages can be beyond i_size: drop them from page cache */ 2441 2440 if (head[i].index >= end) { ··· 2455 2454 2456 2455 ClearPageCompound(head); 2457 2456 2458 2458 - split_page_owner(head, HPAGE_PMD_ORDER); 2457 2457 + split_page_owner(head, nr); 2459 2458 2460 2459 /* See comment in __split_huge_page_tail() */ 2461 2460 if (PageAnon(head)) { ··· 2474 2473 2475 2474 spin_unlock_irqrestore(&pgdat->lru_lock, flags); 2476 2475 2477 2477 - remap_page(head); 2476 2476 + remap_page(head, nr); 2478 2477 2479 2479 - for (i = 0; i < HPAGE_PMD_NR; i++) { 2478 2478 + if (PageSwapCache(head)) { 2479 2479 + swp_entry_t entry = { .val = page_private(head) }; 2480 2480 + 2481 2481 + split_swap_cluster(entry); 2482 2482 + } 2483 2483 + 2484 2484 + for (i = 0; i < nr; i++) { 2480 2485 struct page *subpage = head + i; 2481 2486 if (subpage == page) 2482 2487 continue; ··· 2501 2494 2502 2495 int total_mapcount(struct page *page) 2503 2496 { 2504 2504 - int i, compound, ret; 2497 2497 + int i, compound, nr, ret; 2505 2498 2506 2499 VM_BUG_ON_PAGE(PageTail(page), page); 2507 2500 ··· 2509 2502 return atomic_read(&page->_mapcount) + 1; 2510 2503 2511 2504 compound = compound_mapcount(page); 2505 2505 + nr = compound_nr(page); 2512 2506 if (PageHuge(page)) 2513 2507 return compound; 2514 2508 ret = compound; 2515 2515 - for (i = 0; i < HPAGE_PMD_NR; i++) 2509 2509 + for (i = 0; i < nr; i++) 2516 2510 ret += atomic_read(&page[i]._mapcount) + 1; 2517 2511 /* File pages has compound_mapcount included in _mapcount */ 2518 2512 if (!PageAnon(page)) 2519 2519 - return ret - compound * HPAGE_PMD_NR; 2513 2513 + return ret - compound * nr; 2520 2514 if (PageDoubleMap(page)) 2521 2521 - ret -= HPAGE_PMD_NR; 2515 2515 + ret -= nr; 2522 2516 return ret; 2523 2517 } 2524 2518 ··· 2564 2556 page = compound_head(page); 2565 2557 2566 2558 _total_mapcount = ret = 0; 2567 2567 - for (i = 0; i < HPAGE_PMD_NR; i++) { 2559 2559 + for (i = 0; i < thp_nr_pages(page); i++) { 2568 2560 mapcount = atomic_read(&page[i]._mapcount) + 1; 2569 2561 ret = max(ret, mapcount); 2570 2562 _total_mapcount += mapcount; 2571 2563 } 2572 2564 if (PageDoubleMap(page)) { 2573 2565 ret -= 1; 2574 2574 - _total_mapcount -= HPAGE_PMD_NR; 2566 2566 + _total_mapcount -= thp_nr_pages(page); 2575 2567 } 2576 2568 mapcount = compound_mapcount(page); 2577 2569 ret += mapcount; ··· 2588 2580 2589 2581 /* Additional pins from page cache */ 2590 2582 if (PageAnon(page)) 2591 2591 - extra_pins = PageSwapCache(page) ? HPAGE_PMD_NR : 0; 2583 2583 + extra_pins = PageSwapCache(page) ? thp_nr_pages(page) : 0; 2592 2584 else 2593 2593 - extra_pins = HPAGE_PMD_NR; 2585 2585 + extra_pins = thp_nr_pages(page); 2594 2586 if (pextra_pins) 2595 2587 *pextra_pins = extra_pins; 2596 2588 return total_mapcount(page) == page_count(page) - extra_pins - 1; ··· 2717 2709 } 2718 2710 2719 2711 __split_huge_page(page, list, end, flags); 2720 2720 - if (PageSwapCache(head)) { 2721 2721 - swp_entry_t entry = { .val = page_private(head) }; 2722 2722 - 2723 2723 - ret = split_swap_cluster(entry); 2724 2724 - } else 2725 2725 - ret = 0; 2712 2712 + ret = 0; 2726 2713 } else { 2727 2714 if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) { 2728 2715 pr_alert("total_mapcount: %u, page_count(): %u\n", ··· 2731 2728 fail: if (mapping) 2732 2729 xa_unlock(&mapping->i_pages); 2733 2730 spin_unlock_irqrestore(&pgdata->lru_lock, flags); 2734 2734 - remap_page(head); 2731 2731 + remap_page(head, thp_nr_pages(head)); 2735 2732 ret = -EBUSY; 2736 2733 } 2737 2734

+5 -13

mm/hwpoison-inject.c

reviewed

··· 26 26 27 27 p = pfn_to_page(pfn); 28 28 hpage = compound_head(p); 29 29 - /* 30 30 - * This implies unable to support free buddy pages. 31 31 - */ 32 32 - if (!get_hwpoison_page(p)) 33 33 - return 0; 34 29 35 30 if (!hwpoison_filter_enable) 36 31 goto inject; ··· 35 40 * This implies unable to support non-LRU pages. 36 41 */ 37 42 if (!PageLRU(hpage) && !PageHuge(p)) 38 38 - goto put_out; 43 43 + return 0; 39 44 40 45 /* 41 41 - * do a racy check with elevated page count, to make sure PG_hwpoison 42 42 - * will only be set for the targeted owner (or on a free page). 46 46 + * do a racy check to make sure PG_hwpoison will only be set for 47 47 + * the targeted owner (or on a free page). 43 48 * memory_failure() will redo the check reliably inside page lock. 44 49 */ 45 50 err = hwpoison_filter(hpage); 46 51 if (err) 47 47 - goto put_out; 52 52 + return 0; 48 53 49 54 inject: 50 55 pr_info("Injecting memory failure at pfn %#lx\n", pfn); 51 51 - return memory_failure(pfn, MF_COUNT_INCREASED); 52 52 - put_out: 53 53 - put_hwpoison_page(p); 54 54 - return 0; 56 56 + return memory_failure(pfn, 0); 55 57 } 56 58 57 59 static int hwpoison_unpoison(void *data, u64 val)

+11 -16

mm/internal.h

reviewed

··· 49 49 unsigned long addr, unsigned long end, 50 50 struct zap_details *details); 51 51 52 52 - void force_page_cache_readahead(struct address_space *, struct file *, 53 53 - pgoff_t index, unsigned long nr_to_read); 54 54 - void __do_page_cache_readahead(struct address_space *, struct file *, 55 55 - pgoff_t index, unsigned long nr_to_read, 52 52 + void do_page_cache_ra(struct readahead_control *, unsigned long nr_to_read, 56 53 unsigned long lookahead_size); 57 57 - 58 58 - /* 59 59 - * Submit IO for the read-ahead request in file_ra_state. 60 60 - */ 61 61 - static inline void ra_submit(struct file_ra_state *ra, 62 62 - struct address_space *mapping, struct file *filp) 54 54 + void force_page_cache_ra(struct readahead_control *, struct file_ra_state *, 55 55 + unsigned long nr); 56 56 + static inline void force_page_cache_readahead(struct address_space *mapping, 57 57 + struct file *file, pgoff_t index, unsigned long nr_to_read) 63 58 { 64 64 - __do_page_cache_readahead(mapping, filp, 65 65 - ra->start, ra->size, ra->async_size); 59 59 + DEFINE_READAHEAD(ractl, file, mapping, index); 60 60 + force_page_cache_ra(&ractl, &file->f_ra, nr_to_read); 66 61 } 67 62 68 63 struct page *find_get_entry(struct address_space *mapping, pgoff_t index); ··· 270 275 * page from being allocated in parallel and returning garbage as the order. 271 276 * If a caller does not hold page_zone(page)->lock, it must guarantee that the 272 277 * page cannot be allocated or merged in parallel. Alternatively, it must 273 273 - * handle invalid values gracefully, and use page_order_unsafe() below. 278 278 + * handle invalid values gracefully, and use buddy_order_unsafe() below. 274 279 */ 275 275 - static inline unsigned int page_order(struct page *page) 280 280 + static inline unsigned int buddy_order(struct page *page) 276 281 { 277 282 /* PageBuddy() must be checked by the caller */ 278 283 return page_private(page); 279 284 } 280 285 281 286 /* 282 282 - * Like page_order(), but for callers who cannot afford to hold the zone lock. 287 287 + * Like buddy_order(), but for callers who cannot afford to hold the zone lock. 283 288 * PageBuddy() should be checked first by the caller to minimize race window, 284 289 * and invalid values must be handled gracefully. 285 290 * ··· 289 294 * times, potentially observing different values in the tests and the actual 290 295 * use of the result. 291 296 */ 292 292 - #define page_order_unsafe(page) READ_ONCE(page_private(page)) 297 297 + #define buddy_order_unsafe(page) READ_ONCE(page_private(page)) 293 298 294 299 static inline bool is_cow_mapping(vm_flags_t flags) 295 300 {

+1 -1

mm/khugepaged.c

reviewed

··· 434 434 435 435 static inline int khugepaged_test_exit(struct mm_struct *mm) 436 436 { 437 437 - return atomic_read(&mm->mm_users) == 0 || !mmget_still_valid(mm); 437 437 + return atomic_read(&mm->mm_users) == 0; 438 438 } 439 439 440 440 static bool hugepage_vma_check(struct vm_area_struct *vma,

+13 -39

mm/madvise.c

reviewed

··· 872 872 static int madvise_inject_error(int behavior, 873 873 unsigned long start, unsigned long end) 874 874 { 875 875 - struct page *page; 876 875 struct zone *zone; 877 876 unsigned long size; 878 877 ··· 881 882 882 883 for (; start < end; start += size) { 883 884 unsigned long pfn; 885 885 + struct page *page; 884 886 int ret; 885 887 886 888 ret = get_user_pages_fast(start, 1, 0, &page); ··· 896 896 */ 897 897 size = page_size(compound_head(page)); 898 898 899 899 - if (PageHWPoison(page)) { 900 900 - put_page(page); 901 901 - continue; 902 902 - } 903 903 - 904 899 if (behavior == MADV_SOFT_OFFLINE) { 905 900 pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n", 906 906 - pfn, start); 907 907 - 901 901 + pfn, start); 908 902 ret = soft_offline_page(pfn, MF_COUNT_INCREASED); 909 909 - if (ret) 910 910 - return ret; 911 911 - continue; 903 903 + } else { 904 904 + pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n", 905 905 + pfn, start); 906 906 + /* 907 907 + * Drop the page reference taken by get_user_pages_fast(). In 908 908 + * the absence of MF_COUNT_INCREASED the memory_failure() 909 909 + * routine is responsible for pinning the page to prevent it 910 910 + * from being released back to the page allocator. 911 911 + */ 912 912 + put_page(page); 913 913 + ret = memory_failure(pfn, 0); 912 914 } 913 915 914 914 - pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n", 915 915 - pfn, start); 916 916 - 917 917 - /* 918 918 - * Drop the page reference taken by get_user_pages_fast(). In 919 919 - * the absence of MF_COUNT_INCREASED the memory_failure() 920 920 - * routine is responsible for pinning the page to prevent it 921 921 - * from being released back to the page allocator. 922 922 - */ 923 923 - put_page(page); 924 924 - ret = memory_failure(pfn, 0); 925 916 if (ret) 926 917 return ret; 927 918 } ··· 1085 1094 if (write) { 1086 1095 if (mmap_write_lock_killable(current->mm)) 1087 1096 return -EINTR; 1088 1088 - 1089 1089 - /* 1090 1090 - * We may have stolen the mm from another process 1091 1091 - * that is undergoing core dumping. 1092 1092 - * 1093 1093 - * Right now that's io_ring, in the future it may 1094 1094 - * be remote process management and not "current" 1095 1095 - * at all. 1096 1096 - * 1097 1097 - * We need to fix core dumping to not do this, 1098 1098 - * but for now we have the mmget_still_valid() 1099 1099 - * model. 1100 1100 - */ 1101 1101 - if (!mmget_still_valid(current->mm)) { 1102 1102 - mmap_write_unlock(current->mm); 1103 1103 - return -EINTR; 1104 1104 - } 1105 1097 } else { 1106 1098 mmap_read_lock(current->mm); 1107 1099 }

+145 -170

mm/memory-failure.c

reviewed

··· 65 65 66 66 atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); 67 67 68 68 + static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, bool release) 69 69 + { 70 70 + if (hugepage_or_freepage) { 71 71 + /* 72 72 + * Doing this check for free pages is also fine since dissolve_free_huge_page 73 73 + * returns 0 for non-hugetlb pages as well. 74 74 + */ 75 75 + if (dissolve_free_huge_page(page) || !take_page_off_buddy(page)) 76 76 + /* 77 77 + * We could fail to take off the target page from buddy 78 78 + * for example due to racy page allocaiton, but that's 79 79 + * acceptable because soft-offlined page is not broken 80 80 + * and if someone really want to use it, they should 81 81 + * take it. 82 82 + */ 83 83 + return false; 84 84 + } 85 85 + 86 86 + SetPageHWPoison(page); 87 87 + if (release) 88 88 + put_page(page); 89 89 + page_ref_inc(page); 90 90 + num_poisoned_pages_inc(); 91 91 + 92 92 + return true; 93 93 + } 94 94 + 68 95 #if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE) 69 96 70 97 u32 hwpoison_filter_enable = 0; ··· 582 555 [MF_MSG_BUDDY] = "free buddy page", 583 556 [MF_MSG_BUDDY_2ND] = "free buddy page (2nd try)", 584 557 [MF_MSG_DAX] = "dax page", 558 558 + [MF_MSG_UNSPLIT_THP] = "unsplit thp", 585 559 [MF_MSG_UNKNOWN] = "unknown page", 586 560 }; 587 561 ··· 952 924 * Return: return 0 if failed to grab the refcount, otherwise true (some 953 925 * non-zero value.) 954 926 */ 955 955 - int get_hwpoison_page(struct page *page) 927 927 + static int get_hwpoison_page(struct page *page) 956 928 { 957 929 struct page *head = compound_head(page); 958 930 ··· 981 953 982 954 return 0; 983 955 } 984 984 - EXPORT_SYMBOL_GPL(get_hwpoison_page); 985 956 986 957 /* 987 958 * Do all that is necessary to remove user space mappings. Unmap ··· 1130 1103 return page_action(ps, p, pfn); 1131 1104 } 1132 1105 1106 1106 + static int try_to_split_thp_page(struct page *page, const char *msg) 1107 1107 + { 1108 1108 + lock_page(page); 1109 1109 + if (!PageAnon(page) || unlikely(split_huge_page(page))) { 1110 1110 + unsigned long pfn = page_to_pfn(page); 1111 1111 + 1112 1112 + unlock_page(page); 1113 1113 + if (!PageAnon(page)) 1114 1114 + pr_info("%s: %#lx: non anonymous thp\n", msg, pfn); 1115 1115 + else 1116 1116 + pr_info("%s: %#lx: thp split failed\n", msg, pfn); 1117 1117 + put_page(page); 1118 1118 + return -EBUSY; 1119 1119 + } 1120 1120 + unlock_page(page); 1121 1121 + 1122 1122 + return 0; 1123 1123 + } 1124 1124 + 1133 1125 static int memory_failure_hugetlb(unsigned long pfn, int flags) 1134 1126 { 1135 1127 struct page *p = pfn_to_page(pfn); ··· 1190 1144 pr_err("Memory failure: %#lx: just unpoisoned\n", pfn); 1191 1145 num_poisoned_pages_dec(); 1192 1146 unlock_page(head); 1193 1193 - put_hwpoison_page(head); 1147 1147 + put_page(head); 1194 1148 return 0; 1195 1149 } 1196 1150 ··· 1371 1325 } 1372 1326 1373 1327 if (PageTransHuge(hpage)) { 1374 1374 - lock_page(p); 1375 1375 - if (!PageAnon(p) || unlikely(split_huge_page(p))) { 1376 1376 - unlock_page(p); 1377 1377 - if (!PageAnon(p)) 1378 1378 - pr_err("Memory failure: %#lx: non anonymous thp\n", 1379 1379 - pfn); 1380 1380 - else 1381 1381 - pr_err("Memory failure: %#lx: thp split failed\n", 1382 1382 - pfn); 1383 1383 - if (TestClearPageHWPoison(p)) 1384 1384 - num_poisoned_pages_dec(); 1385 1385 - put_hwpoison_page(p); 1328 1328 + if (try_to_split_thp_page(p, "Memory Failure") < 0) { 1329 1329 + action_result(pfn, MF_MSG_UNSPLIT_THP, MF_IGNORED); 1386 1330 return -EBUSY; 1387 1331 } 1388 1388 - unlock_page(p); 1389 1332 VM_BUG_ON_PAGE(!page_count(p), p); 1390 1390 - hpage = compound_head(p); 1391 1333 } 1392 1334 1393 1335 /* ··· 1415 1381 * page_remove_rmap() in try_to_unmap_one(). So to determine page status 1416 1382 * correctly, we save a copy of the page flags at this time. 1417 1383 */ 1418 1418 - if (PageHuge(p)) 1419 1419 - page_flags = hpage->flags; 1420 1420 - else 1421 1421 - page_flags = p->flags; 1384 1384 + page_flags = p->flags; 1422 1385 1423 1386 /* 1424 1387 * unpoison always clear PG_hwpoison inside page lock ··· 1424 1393 pr_err("Memory failure: %#lx: just unpoisoned\n", pfn); 1425 1394 num_poisoned_pages_dec(); 1426 1395 unlock_page(p); 1427 1427 - put_hwpoison_page(p); 1396 1396 + put_page(p); 1428 1397 return 0; 1429 1398 } 1430 1399 if (hwpoison_filter(p)) { 1431 1400 if (TestClearPageHWPoison(p)) 1432 1401 num_poisoned_pages_dec(); 1433 1402 unlock_page(p); 1434 1434 - put_hwpoison_page(p); 1403 1403 + put_page(p); 1435 1404 return 0; 1436 1405 } 1437 1406 ··· 1447 1416 /* 1448 1417 * Now take care of user space mappings. 1449 1418 * Abort on fail: __delete_from_page_cache() assumes unmapped page. 1450 1450 - * 1451 1451 - * When the raw error page is thp tail page, hpage points to the raw 1452 1452 - * page after thp split. 1453 1419 */ 1454 1454 - if (!hwpoison_user_mappings(p, pfn, flags, &hpage)) { 1420 1420 + if (!hwpoison_user_mappings(p, pfn, flags, &p)) { 1455 1421 action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED); 1456 1422 res = -EBUSY; 1457 1423 goto out; ··· 1665 1637 } 1666 1638 unlock_page(page); 1667 1639 1668 1668 - put_hwpoison_page(page); 1640 1640 + put_page(page); 1669 1641 if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1)) 1670 1670 - put_hwpoison_page(page); 1642 1642 + put_page(page); 1671 1643 1672 1644 return 0; 1673 1645 } ··· 1707 1679 } else if (is_free_buddy_page(p)) { 1708 1680 pr_info("%s: %#lx free buddy page\n", __func__, pfn); 1709 1681 ret = 0; 1682 1682 + } else if (page_count(p)) { 1683 1683 + /* raced with allocation */ 1684 1684 + ret = -EBUSY; 1710 1685 } else { 1711 1686 pr_info("%s: %#lx: unknown zero refcount page type %lx\n", 1712 1687 __func__, pfn, p->flags); ··· 1726 1695 { 1727 1696 int ret = __get_any_page(page, pfn, flags); 1728 1697 1698 1698 + if (ret == -EBUSY) 1699 1699 + ret = __get_any_page(page, pfn, flags); 1700 1700 + 1729 1701 if (ret == 1 && !PageHuge(page) && 1730 1702 !PageLRU(page) && !__PageMovable(page)) { 1731 1703 /* 1732 1704 * Try to free it. 1733 1705 */ 1734 1734 - put_hwpoison_page(page); 1706 1706 + put_page(page); 1735 1707 shake_page(page, 1); 1736 1708 1737 1709 /* ··· 1743 1709 ret = __get_any_page(page, pfn, 0); 1744 1710 if (ret == 1 && !PageLRU(page)) { 1745 1711 /* Drop page reference which is from __get_any_page() */ 1746 1746 - put_hwpoison_page(page); 1712 1712 + put_page(page); 1747 1713 pr_info("soft_offline: %#lx: unknown non LRU page type %lx (%pGp)\n", 1748 1714 pfn, page->flags, &page->flags); 1749 1715 return -EIO; ··· 1752 1718 return ret; 1753 1719 } 1754 1720 1755 1755 - static int soft_offline_huge_page(struct page *page, int flags) 1721 1721 + static bool isolate_page(struct page *page, struct list_head *pagelist) 1756 1722 { 1757 1757 - int ret; 1758 1758 - unsigned long pfn = page_to_pfn(page); 1759 1759 - struct page *hpage = compound_head(page); 1760 1760 - LIST_HEAD(pagelist); 1723 1723 + bool isolated = false; 1724 1724 + bool lru = PageLRU(page); 1761 1725 1762 1762 - /* 1763 1763 - * This double-check of PageHWPoison is to avoid the race with 1764 1764 - * memory_failure(). See also comment in __soft_offline_page(). 1765 1765 - */ 1766 1766 - lock_page(hpage); 1767 1767 - if (PageHWPoison(hpage)) { 1768 1768 - unlock_page(hpage); 1769 1769 - put_hwpoison_page(hpage); 1770 1770 - pr_info("soft offline: %#lx hugepage already poisoned\n", pfn); 1771 1771 - return -EBUSY; 1772 1772 - } 1773 1773 - unlock_page(hpage); 1774 1774 - 1775 1775 - ret = isolate_huge_page(hpage, &pagelist); 1776 1776 - /* 1777 1777 - * get_any_page() and isolate_huge_page() takes a refcount each, 1778 1778 - * so need to drop one here. 1779 1779 - */ 1780 1780 - put_hwpoison_page(hpage); 1781 1781 - if (!ret) { 1782 1782 - pr_info("soft offline: %#lx hugepage failed to isolate\n", pfn); 1783 1783 - return -EBUSY; 1784 1784 - } 1785 1785 - 1786 1786 - ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL, 1787 1787 - MIGRATE_SYNC, MR_MEMORY_FAILURE); 1788 1788 - if (ret) { 1789 1789 - pr_info("soft offline: %#lx: hugepage migration failed %d, type %lx (%pGp)\n", 1790 1790 - pfn, ret, page->flags, &page->flags); 1791 1791 - if (!list_empty(&pagelist)) 1792 1792 - putback_movable_pages(&pagelist); 1793 1793 - if (ret > 0) 1794 1794 - ret = -EIO; 1726 1726 + if (PageHuge(page)) { 1727 1727 + isolated = isolate_huge_page(page, pagelist); 1795 1728 } else { 1796 1796 - /* 1797 1797 - * We set PG_hwpoison only when the migration source hugepage 1798 1798 - * was successfully dissolved, because otherwise hwpoisoned 1799 1799 - * hugepage remains on free hugepage list, then userspace will 1800 1800 - * find it as SIGBUS by allocation failure. That's not expected 1801 1801 - * in soft-offlining. 1802 1802 - */ 1803 1803 - ret = dissolve_free_huge_page(page); 1804 1804 - if (!ret) { 1805 1805 - if (set_hwpoison_free_buddy_page(page)) 1806 1806 - num_poisoned_pages_inc(); 1807 1807 - else 1808 1808 - ret = -EBUSY; 1809 1809 - } 1729 1729 + if (lru) 1730 1730 + isolated = !isolate_lru_page(page); 1731 1731 + else 1732 1732 + isolated = !isolate_movable_page(page, ISOLATE_UNEVICTABLE); 1733 1733 + 1734 1734 + if (isolated) 1735 1735 + list_add(&page->lru, pagelist); 1810 1736 } 1811 1811 - return ret; 1737 1737 + 1738 1738 + if (isolated && lru) 1739 1739 + inc_node_page_state(page, NR_ISOLATED_ANON + 1740 1740 + page_is_file_lru(page)); 1741 1741 + 1742 1742 + /* 1743 1743 + * If we succeed to isolate the page, we grabbed another refcount on 1744 1744 + * the page, so we can safely drop the one we got from get_any_pages(). 1745 1745 + * If we failed to isolate the page, it means that we cannot go further 1746 1746 + * and we will return an error, so drop the reference we got from 1747 1747 + * get_any_pages() as well. 1748 1748 + */ 1749 1749 + put_page(page); 1750 1750 + return isolated; 1812 1751 } 1813 1752 1814 1814 - static int __soft_offline_page(struct page *page, int flags) 1753 1753 + /* 1754 1754 + * __soft_offline_page handles hugetlb-pages and non-hugetlb pages. 1755 1755 + * If the page is a non-dirty unmapped page-cache page, it simply invalidates. 1756 1756 + * If the page is mapped, it migrates the contents over. 1757 1757 + */ 1758 1758 + static int __soft_offline_page(struct page *page) 1815 1759 { 1816 1816 - int ret; 1760 1760 + int ret = 0; 1817 1761 unsigned long pfn = page_to_pfn(page); 1762 1762 + struct page *hpage = compound_head(page); 1763 1763 + char const *msg_page[] = {"page", "hugepage"}; 1764 1764 + bool huge = PageHuge(page); 1765 1765 + LIST_HEAD(pagelist); 1818 1766 1819 1767 /* 1820 1768 * Check PageHWPoison again inside page lock because PageHWPoison ··· 1805 1789 * so there's no race between soft_offline_page() and memory_failure(). 1806 1790 */ 1807 1791 lock_page(page); 1808 1808 - wait_on_page_writeback(page); 1792 1792 + if (!PageHuge(page)) 1793 1793 + wait_on_page_writeback(page); 1809 1794 if (PageHWPoison(page)) { 1810 1795 unlock_page(page); 1811 1811 - put_hwpoison_page(page); 1796 1796 + put_page(page); 1812 1797 pr_info("soft offline: %#lx page already poisoned\n", pfn); 1813 1813 - return -EBUSY; 1798 1798 + return 0; 1814 1799 } 1815 1815 - /* 1816 1816 - * Try to invalidate first. This should work for 1817 1817 - * non dirty unmapped page cache pages. 1818 1818 - */ 1819 1819 - ret = invalidate_inode_page(page); 1800 1800 + 1801 1801 + if (!PageHuge(page)) 1802 1802 + /* 1803 1803 + * Try to invalidate first. This should work for 1804 1804 + * non dirty unmapped page cache pages. 1805 1805 + */ 1806 1806 + ret = invalidate_inode_page(page); 1820 1807 unlock_page(page); 1808 1808 + 1821 1809 /* 1822 1810 * RED-PEN would be better to keep it isolated here, but we 1823 1811 * would need to fix isolation locking first. 1824 1812 */ 1825 1825 - if (ret == 1) { 1826 1826 - put_hwpoison_page(page); 1813 1813 + if (ret) { 1827 1814 pr_info("soft_offline: %#lx: invalidated\n", pfn); 1828 1828 - SetPageHWPoison(page); 1829 1829 - num_poisoned_pages_inc(); 1815 1815 + page_handle_poison(page, false, true); 1830 1816 return 0; 1831 1817 } 1832 1818 1833 1833 - /* 1834 1834 - * Simple invalidation didn't work. 1835 1835 - * Try to migrate to a new page instead. migrate.c 1836 1836 - * handles a large number of cases for us. 1837 1837 - */ 1838 1838 - if (PageLRU(page)) 1839 1839 - ret = isolate_lru_page(page); 1840 1840 - else 1841 1841 - ret = isolate_movable_page(page, ISOLATE_UNEVICTABLE); 1842 1842 - /* 1843 1843 - * Drop page reference which is came from get_any_page() 1844 1844 - * successful isolate_lru_page() already took another one. 1845 1845 - */ 1846 1846 - put_hwpoison_page(page); 1847 1847 - if (!ret) { 1848 1848 - LIST_HEAD(pagelist); 1849 1849 - /* 1850 1850 - * After isolated lru page, the PageLRU will be cleared, 1851 1851 - * so use !__PageMovable instead for LRU page's mapping 1852 1852 - * cannot have PAGE_MAPPING_MOVABLE. 1853 1853 - */ 1854 1854 - if (!__PageMovable(page)) 1855 1855 - inc_node_page_state(page, NR_ISOLATED_ANON + 1856 1856 - page_is_file_lru(page)); 1857 1857 - list_add(&page->lru, &pagelist); 1819 1819 + if (isolate_page(hpage, &pagelist)) { 1858 1820 ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL, 1859 1821 MIGRATE_SYNC, MR_MEMORY_FAILURE); 1860 1860 - if (ret) { 1822 1822 + if (!ret) { 1823 1823 + bool release = !huge; 1824 1824 + 1825 1825 + if (!page_handle_poison(page, huge, release)) 1826 1826 + ret = -EBUSY; 1827 1827 + } else { 1861 1828 if (!list_empty(&pagelist)) 1862 1829 putback_movable_pages(&pagelist); 1863 1830 1864 1864 - pr_info("soft offline: %#lx: migration failed %d, type %lx (%pGp)\n", 1865 1865 - pfn, ret, page->flags, &page->flags); 1831 1831 + pr_info("soft offline: %#lx: %s migration failed %d, type %lx (%pGp)\n", 1832 1832 + pfn, msg_page[huge], ret, page->flags, &page->flags); 1866 1833 if (ret > 0) 1867 1834 ret = -EIO; 1868 1835 } 1869 1836 } else { 1870 1870 - pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx (%pGp)\n", 1871 1871 - pfn, ret, page_count(page), page->flags, &page->flags); 1837 1837 + pr_info("soft offline: %#lx: %s isolation failed: %d, page count %d, type %lx (%pGp)\n", 1838 1838 + pfn, msg_page[huge], ret, page_count(page), page->flags, &page->flags); 1839 1839 + ret = -EBUSY; 1872 1840 } 1873 1841 return ret; 1874 1842 } 1875 1843 1876 1876 - static int soft_offline_in_use_page(struct page *page, int flags) 1844 1844 + static int soft_offline_in_use_page(struct page *page) 1877 1845 { 1878 1878 - int ret; 1879 1879 - int mt; 1880 1846 struct page *hpage = compound_head(page); 1881 1847 1882 1882 - if (!PageHuge(page) && PageTransHuge(hpage)) { 1883 1883 - lock_page(page); 1884 1884 - if (!PageAnon(page) || unlikely(split_huge_page(page))) { 1885 1885 - unlock_page(page); 1886 1886 - if (!PageAnon(page)) 1887 1887 - pr_info("soft offline: %#lx: non anonymous thp\n", page_to_pfn(page)); 1888 1888 - else 1889 1889 - pr_info("soft offline: %#lx: thp split failed\n", page_to_pfn(page)); 1890 1890 - put_hwpoison_page(page); 1848 1848 + if (!PageHuge(page) && PageTransHuge(hpage)) 1849 1849 + if (try_to_split_thp_page(page, "soft offline") < 0) 1891 1850 return -EBUSY; 1892 1892 - } 1893 1893 - unlock_page(page); 1894 1894 - } 1895 1895 - 1896 1896 - /* 1897 1897 - * Setting MIGRATE_ISOLATE here ensures that the page will be linked 1898 1898 - * to free list immediately (not via pcplist) when released after 1899 1899 - * successful page migration. Otherwise we can't guarantee that the 1900 1900 - * page is really free after put_page() returns, so 1901 1901 - * set_hwpoison_free_buddy_page() highly likely fails. 1902 1902 - */ 1903 1903 - mt = get_pageblock_migratetype(page); 1904 1904 - set_pageblock_migratetype(page, MIGRATE_ISOLATE); 1905 1905 - if (PageHuge(page)) 1906 1906 - ret = soft_offline_huge_page(page, flags); 1907 1907 - else 1908 1908 - ret = __soft_offline_page(page, flags); 1909 1909 - set_pageblock_migratetype(page, mt); 1910 1910 - return ret; 1851 1851 + return __soft_offline_page(page); 1911 1852 } 1912 1853 1913 1854 static int soft_offline_free_page(struct page *page) 1914 1855 { 1915 1915 - int rc = dissolve_free_huge_page(page); 1856 1856 + int rc = 0; 1916 1857 1917 1917 - if (!rc) { 1918 1918 - if (set_hwpoison_free_buddy_page(page)) 1919 1919 - num_poisoned_pages_inc(); 1920 1920 - else 1921 1921 - rc = -EBUSY; 1922 1922 - } 1858 1858 + if (!page_handle_poison(page, true, false)) 1859 1859 + rc = -EBUSY; 1860 1860 + 1923 1861 return rc; 1924 1862 } 1925 1863 ··· 1903 1933 { 1904 1934 int ret; 1905 1935 struct page *page; 1936 1936 + bool try_again = true; 1906 1937 1907 1938 if (!pfn_valid(pfn)) 1908 1939 return -ENXIO; ··· 1915 1944 if (PageHWPoison(page)) { 1916 1945 pr_info("soft offline: %#lx page already poisoned\n", pfn); 1917 1946 if (flags & MF_COUNT_INCREASED) 1918 1918 - put_hwpoison_page(page); 1919 1919 - return -EBUSY; 1947 1947 + put_page(page); 1948 1948 + return 0; 1920 1949 } 1921 1950 1951 1951 + retry: 1922 1952 get_online_mems(); 1923 1953 ret = get_any_page(page, pfn, flags); 1924 1954 put_online_mems(); 1925 1955 1926 1956 if (ret > 0) 1927 1927 - ret = soft_offline_in_use_page(page, flags); 1957 1957 + ret = soft_offline_in_use_page(page); 1928 1958 else if (ret == 0) 1929 1929 - ret = soft_offline_free_page(page); 1959 1959 + if (soft_offline_free_page(page) && try_again) { 1960 1960 + try_again = false; 1961 1961 + goto retry; 1962 1962 + } 1930 1963 1931 1964 return ret; 1932 1965 }

+4 -3

mm/memory.c

reviewed

··· 3709 3709 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 3710 3710 pmd_t entry; 3711 3711 int i; 3712 3712 - vm_fault_t ret; 3712 3712 + vm_fault_t ret = VM_FAULT_FALLBACK; 3713 3713 3714 3714 if (!transhuge_vma_suitable(vma, haddr)) 3715 3715 - return VM_FAULT_FALLBACK; 3715 3715 + return ret; 3716 3716 3717 3717 - ret = VM_FAULT_FALLBACK; 3718 3717 page = compound_head(page); 3718 3718 + if (compound_order(page) != HPAGE_PMD_ORDER) 3719 3719 + return ret; 3719 3720 3720 3721 /* 3721 3722 * Archs like ppc64 need additonal space to store information

+86 -125

mm/memory_hotplug.c

reviewed

··· 105 105 unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; 106 106 107 107 if (strcmp(resource_name, "System RAM")) 108 108 - flags |= IORESOURCE_MEM_DRIVER_MANAGED; 108 108 + flags |= IORESOURCE_SYSRAM_DRIVER_MANAGED; 109 109 110 110 /* 111 111 * Make sure value parsed from 'mem=' only restricts memory adding ··· 625 625 } 626 626 EXPORT_SYMBOL_GPL(generic_online_page); 627 627 628 628 - static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages, 629 629 - void *arg) 628 628 + static void online_pages_range(unsigned long start_pfn, unsigned long nr_pages) 630 629 { 631 630 const unsigned long end_pfn = start_pfn + nr_pages; 632 631 unsigned long pfn; 633 633 - int order; 634 632 635 633 /* 636 636 - * Online the pages. The callback might decide to keep some pages 637 637 - * PG_reserved (to add them to the buddy later), but we still account 638 638 - * them as being online/belonging to this zone ("present"). 634 634 + * Online the pages in MAX_ORDER - 1 aligned chunks. The callback might 635 635 + * decide to not expose all pages to the buddy (e.g., expose them 636 636 + * later). We account all pages as being online and belonging to this 637 637 + * zone ("present"). 639 638 */ 640 640 - for (pfn = start_pfn; pfn < end_pfn; pfn += 1ul << order) { 641 641 - order = min(MAX_ORDER - 1, get_order(PFN_PHYS(end_pfn - pfn))); 642 642 - /* __free_pages_core() wants pfns to be aligned to the order */ 643 643 - if (WARN_ON_ONCE(!IS_ALIGNED(pfn, 1ul << order))) 644 644 - order = 0; 645 645 - (*online_page_callback)(pfn_to_page(pfn), order); 646 646 - } 639 639 + for (pfn = start_pfn; pfn < end_pfn; pfn += MAX_ORDER_NR_PAGES) 640 640 + (*online_page_callback)(pfn_to_page(pfn), MAX_ORDER - 1); 647 641 648 642 /* mark all involved sections as online */ 649 643 online_mem_sections(start_pfn, end_pfn); 650 650 - 651 651 - *(unsigned long *)arg += nr_pages; 652 652 - return 0; 653 644 } 654 645 655 646 /* check which state of node_states will be changed when online memory */ ··· 701 710 * Associate the pfn range with the given zone, initializing the memmaps 702 711 * and resizing the pgdat/zone data to span the added pages. After this 703 712 * call, all affected pages are PG_reserved. 713 713 + * 714 714 + * All aligned pageblocks are initialized to the specified migratetype 715 715 + * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 716 716 + * zone stats (e.g., nr_isolate_pageblock) are touched. 704 717 */ 705 718 void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, 706 706 - unsigned long nr_pages, struct vmem_altmap *altmap) 719 719 + unsigned long nr_pages, 720 720 + struct vmem_altmap *altmap, int migratetype) 707 721 { 708 722 struct pglist_data *pgdat = zone->zone_pgdat; 709 723 int nid = pgdat->node_id; ··· 733 737 * are reserved so nobody should be touching them so we should be safe 734 738 */ 735 739 memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn, 736 736 - MEMINIT_HOTPLUG, altmap); 740 740 + MEMINIT_HOTPLUG, altmap, migratetype); 737 741 738 742 set_zone_contiguous(zone); 739 743 } ··· 799 803 int online_type, int nid) 800 804 { 801 805 unsigned long flags; 802 802 - unsigned long onlined_pages = 0; 803 806 struct zone *zone; 804 807 int need_zonelists_rebuild = 0; 805 808 int ret; 806 809 struct memory_notify arg; 807 810 811 811 + /* We can only online full sections (e.g., SECTION_IS_ONLINE) */ 812 812 + if (WARN_ON_ONCE(!nr_pages || 813 813 + !IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION))) 814 814 + return -EINVAL; 815 815 + 808 816 mem_hotplug_begin(); 809 817 810 818 /* associate pfn range with the zone */ 811 819 zone = zone_for_pfn_range(online_type, nid, pfn, nr_pages); 812 812 - move_pfn_range_to_zone(zone, pfn, nr_pages, NULL); 820 820 + move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_ISOLATE); 813 821 814 822 arg.start_pfn = pfn; 815 823 arg.nr_pages = nr_pages; ··· 825 825 goto failed_addition; 826 826 827 827 /* 828 828 + * Fixup the number of isolated pageblocks before marking the sections 829 829 + * onlining, such that undo_isolate_page_range() works correctly. 830 830 + */ 831 831 + spin_lock_irqsave(&zone->lock, flags); 832 832 + zone->nr_isolate_pageblock += nr_pages / pageblock_nr_pages; 833 833 + spin_unlock_irqrestore(&zone->lock, flags); 834 834 + 835 835 + /* 828 836 * If this zone is not populated, then it is not in zonelist. 829 837 * This means the page allocator ignores this zone. 830 838 * So, zonelist must be updated after online. ··· 842 834 setup_zone_pageset(zone); 843 835 } 844 836 845 845 - ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages, 846 846 - online_pages_range); 847 847 - if (ret) { 848 848 - /* not a single memory resource was applicable */ 849 849 - if (need_zonelists_rebuild) 850 850 - zone_pcp_reset(zone); 851 851 - goto failed_addition; 852 852 - } 853 853 - 854 854 - zone->present_pages += onlined_pages; 837 837 + online_pages_range(pfn, nr_pages); 838 838 + zone->present_pages += nr_pages; 855 839 856 840 pgdat_resize_lock(zone->zone_pgdat, &flags); 857 857 - zone->zone_pgdat->node_present_pages += onlined_pages; 841 841 + zone->zone_pgdat->node_present_pages += nr_pages; 858 842 pgdat_resize_unlock(zone->zone_pgdat, &flags); 859 859 - 860 860 - /* 861 861 - * When exposing larger, physically contiguous memory areas to the 862 862 - * buddy, shuffling in the buddy (when freeing onlined pages, putting 863 863 - * them either to the head or the tail of the freelist) is only helpful 864 864 - * for maintaining the shuffle, but not for creating the initial 865 865 - * shuffle. Shuffle the whole zone to make sure the just onlined pages 866 866 - * are properly distributed across the whole freelist. 867 867 - */ 868 868 - shuffle_zone(zone); 869 843 870 844 node_states_set_node(nid, &arg); 871 845 if (need_zonelists_rebuild) 872 846 build_all_zonelists(NULL); 873 847 zone_pcp_update(zone); 848 848 + 849 849 + /* Basic onlining is complete, allow allocation of onlined pages. */ 850 850 + undo_isolate_page_range(pfn, pfn + nr_pages, MIGRATE_MOVABLE); 851 851 + 852 852 + /* 853 853 + * Freshly onlined pages aren't shuffled (e.g., all pages are placed to 854 854 + * the tail of the freelist when undoing isolation). Shuffle the whole 855 855 + * zone to make sure the just onlined pages are properly distributed 856 856 + * across the whole freelist - to create an initial shuffle. 857 857 + */ 858 858 + shuffle_zone(zone); 874 859 875 860 init_per_zone_wmark_min(); 876 861 ··· 1036 1035 * 1037 1036 * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG 1038 1037 */ 1039 1039 - int __ref add_memory_resource(int nid, struct resource *res) 1038 1038 + int __ref add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) 1040 1039 { 1041 1040 struct mhp_params params = { .pgprot = PAGE_KERNEL }; 1042 1041 u64 start, size; ··· 1089 1088 } 1090 1089 1091 1090 /* link memory sections under this node.*/ 1092 1092 - ret = link_mem_sections(nid, PFN_DOWN(start), PFN_UP(start + size - 1), 1093 1093 - MEMINIT_HOTPLUG); 1094 1094 - BUG_ON(ret); 1091 1091 + link_mem_sections(nid, PFN_DOWN(start), PFN_UP(start + size - 1), 1092 1092 + MEMINIT_HOTPLUG); 1095 1093 1096 1094 /* create new memmap entry */ 1097 1095 if (!strcmp(res->name, "System RAM")) ··· 1098 1098 1099 1099 /* device_online() will take the lock when calling online_pages() */ 1100 1100 mem_hotplug_done(); 1101 1101 + 1102 1102 + /* 1103 1103 + * In case we're allowed to merge the resource, flag it and trigger 1104 1104 + * merging now that adding succeeded. 1105 1105 + */ 1106 1106 + if (mhp_flags & MEMHP_MERGE_RESOURCE) 1107 1107 + merge_system_ram_resource(res); 1101 1108 1102 1109 /* online pages if requested */ 1103 1110 if (memhp_default_online_type != MMOP_OFFLINE) ··· 1122 1115 } 1123 1116 1124 1117 /* requires device_hotplug_lock, see add_memory_resource() */ 1125 1125 - int __ref __add_memory(int nid, u64 start, u64 size) 1118 1118 + int __ref __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) 1126 1119 { 1127 1120 struct resource *res; 1128 1121 int ret; ··· 1131 1124 if (IS_ERR(res)) 1132 1125 return PTR_ERR(res); 1133 1126 1134 1134 - ret = add_memory_resource(nid, res); 1127 1127 + ret = add_memory_resource(nid, res, mhp_flags); 1135 1128 if (ret < 0) 1136 1129 release_memory_resource(res); 1137 1130 return ret; 1138 1131 } 1139 1132 1140 1140 - int add_memory(int nid, u64 start, u64 size) 1133 1133 + int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) 1141 1134 { 1142 1135 int rc; 1143 1136 1144 1137 lock_device_hotplug(); 1145 1145 - rc = __add_memory(nid, start, size); 1138 1138 + rc = __add_memory(nid, start, size, mhp_flags); 1146 1139 unlock_device_hotplug(); 1147 1140 1148 1141 return rc; ··· 1164 1157 * 1165 1158 * For this memory, no entries in /sys/firmware/memmap ("raw firmware-provided 1166 1159 * memory map") are created. Also, the created memory resource is flagged 1167 1167 - * with IORESOURCE_MEM_DRIVER_MANAGED, so in-kernel users can special-case 1160 1160 + * with IORESOURCE_SYSRAM_DRIVER_MANAGED, so in-kernel users can special-case 1168 1161 * this memory as well (esp., not place kexec images onto it). 1169 1162 * 1170 1163 * The resource_name (visible via /proc/iomem) has to have the format 1171 1164 * "System RAM ($DRIVER)". 1172 1165 */ 1173 1166 int add_memory_driver_managed(int nid, u64 start, u64 size, 1174 1174 - const char *resource_name) 1167 1167 + const char *resource_name, mhp_t mhp_flags) 1175 1168 { 1176 1169 struct resource *res; 1177 1170 int rc; ··· 1189 1182 goto out_unlock; 1190 1183 } 1191 1184 1192 1192 - rc = add_memory_resource(nid, res); 1185 1185 + rc = add_memory_resource(nid, res, mhp_flags); 1193 1186 if (rc < 0) 1194 1187 release_memory_resource(res); 1195 1188 ··· 1386 1379 return ret; 1387 1380 } 1388 1381 1389 1389 - /* Mark all sections offline and remove all free pages from the buddy. */ 1390 1390 - static int 1391 1391 - offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages, 1392 1392 - void *data) 1393 1393 - { 1394 1394 - unsigned long *offlined_pages = (unsigned long *)data; 1395 1395 - 1396 1396 - *offlined_pages += __offline_isolated_pages(start, start + nr_pages); 1397 1397 - return 0; 1398 1398 - } 1399 1399 - 1400 1400 - /* 1401 1401 - * Check all pages in range, recorded as memory resource, are isolated. 1402 1402 - */ 1403 1403 - static int 1404 1404 - check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages, 1405 1405 - void *data) 1406 1406 - { 1407 1407 - return test_pages_isolated(start_pfn, start_pfn + nr_pages, 1408 1408 - MEMORY_OFFLINE); 1409 1409 - } 1410 1410 - 1411 1382 static int __init cmdline_parse_movable_node(char *p) 1412 1383 { 1413 1384 movable_node_enabled = true; ··· 1469 1484 return 0; 1470 1485 } 1471 1486 1472 1472 - static int __ref __offline_pages(unsigned long start_pfn, 1473 1473 - unsigned long end_pfn) 1487 1487 + int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages) 1474 1488 { 1475 1475 - unsigned long pfn, nr_pages = 0; 1476 1476 - unsigned long offlined_pages = 0; 1477 1477 - int ret, node, nr_isolate_pageblock; 1489 1489 + const unsigned long end_pfn = start_pfn + nr_pages; 1490 1490 + unsigned long pfn, system_ram_pages = 0; 1478 1491 unsigned long flags; 1479 1492 struct zone *zone; 1480 1493 struct memory_notify arg; 1494 1494 + int ret, node; 1481 1495 char *reason; 1496 1496 + 1497 1497 + /* We can only offline full sections (e.g., SECTION_IS_ONLINE) */ 1498 1498 + if (WARN_ON_ONCE(!nr_pages || 1499 1499 + !IS_ALIGNED(start_pfn | nr_pages, PAGES_PER_SECTION))) 1500 1500 + return -EINVAL; 1482 1501 1483 1502 mem_hotplug_begin(); 1484 1503 ··· 1494 1505 * memory holes PG_reserved, don't need pfn_valid() checks, and can 1495 1506 * avoid using walk_system_ram_range() later. 1496 1507 */ 1497 1497 - walk_system_ram_range(start_pfn, end_pfn - start_pfn, &nr_pages, 1508 1508 + walk_system_ram_range(start_pfn, nr_pages, &system_ram_pages, 1498 1509 count_system_ram_pages_cb); 1499 1499 - if (nr_pages != end_pfn - start_pfn) { 1510 1510 + if (system_ram_pages != nr_pages) { 1500 1511 ret = -EINVAL; 1501 1512 reason = "memory holes"; 1502 1513 goto failed_removal; ··· 1516 1527 ret = start_isolate_page_range(start_pfn, end_pfn, 1517 1528 MIGRATE_MOVABLE, 1518 1529 MEMORY_OFFLINE | REPORT_FAILURE); 1519 1519 - if (ret < 0) { 1530 1530 + if (ret) { 1520 1531 reason = "failure to isolate range"; 1521 1532 goto failed_removal; 1522 1533 } 1523 1523 - nr_isolate_pageblock = ret; 1524 1534 1525 1535 arg.start_pfn = start_pfn; 1526 1536 arg.nr_pages = nr_pages; ··· 1569 1581 reason = "failure to dissolve huge pages"; 1570 1582 goto failed_removal_isolated; 1571 1583 } 1572 1572 - /* check again */ 1573 1573 - ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, 1574 1574 - NULL, check_pages_isolated_cb); 1584 1584 + 1575 1585 /* 1576 1586 * per-cpu pages are drained in start_isolate_page_range, but if 1577 1587 * there are still pages that are not free, make sure that we ··· 1582 1596 * because has_unmovable_pages explicitly checks for 1583 1597 * PageBuddy on freed pages on other zones. 1584 1598 */ 1599 1599 + ret = test_pages_isolated(start_pfn, end_pfn, MEMORY_OFFLINE); 1585 1600 if (ret) 1586 1601 drain_all_pages(zone); 1587 1602 } while (ret); 1588 1603 1589 1589 - /* Ok, all of our target is isolated. 1590 1590 - We cannot do rollback at this point. */ 1591 1591 - walk_system_ram_range(start_pfn, end_pfn - start_pfn, 1592 1592 - &offlined_pages, offline_isolated_pages_cb); 1593 1593 - pr_info("Offlined Pages %ld\n", offlined_pages); 1604 1604 + /* Mark all sections offline and remove free pages from the buddy. */ 1605 1605 + __offline_isolated_pages(start_pfn, end_pfn); 1606 1606 + pr_info("Offlined Pages %ld\n", nr_pages); 1607 1607 + 1594 1608 /* 1595 1595 - * Onlining will reset pagetype flags and makes migrate type 1596 1596 - * MOVABLE, so just need to decrease the number of isolated 1597 1597 - * pageblocks zone counter here. 1609 1609 + * The memory sections are marked offline, and the pageblock flags 1610 1610 + * effectively stale; nobody should be touching them. Fixup the number 1611 1611 + * of isolated pageblocks, memory onlining will properly revert this. 1598 1612 */ 1599 1613 spin_lock_irqsave(&zone->lock, flags); 1600 1600 - zone->nr_isolate_pageblock -= nr_isolate_pageblock; 1614 1614 + zone->nr_isolate_pageblock -= nr_pages / pageblock_nr_pages; 1601 1615 spin_unlock_irqrestore(&zone->lock, flags); 1602 1616 1603 1617 /* removal success */ 1604 1604 - adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages); 1605 1605 - zone->present_pages -= offlined_pages; 1618 1618 + adjust_managed_page_count(pfn_to_page(start_pfn), -nr_pages); 1619 1619 + zone->present_pages -= nr_pages; 1606 1620 1607 1621 pgdat_resize_lock(zone->zone_pgdat, &flags); 1608 1608 - zone->zone_pgdat->node_present_pages -= offlined_pages; 1622 1622 + zone->zone_pgdat->node_present_pages -= nr_pages; 1609 1623 pgdat_resize_unlock(zone->zone_pgdat, &flags); 1610 1624 1611 1625 init_per_zone_wmark_min(); ··· 1640 1654 /* pushback to free area */ 1641 1655 mem_hotplug_done(); 1642 1656 return ret; 1643 1643 - } 1644 1644 - 1645 1645 - int offline_pages(unsigned long start_pfn, unsigned long nr_pages) 1646 1646 - { 1647 1647 - return __offline_pages(start_pfn, start_pfn + nr_pages); 1648 1657 } 1649 1658 1650 1659 static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) ··· 1730 1749 } 1731 1750 EXPORT_SYMBOL(try_offline_node); 1732 1751 1733 1733 - static void __release_memory_resource(resource_size_t start, 1734 1734 - resource_size_t size) 1735 1735 - { 1736 1736 - int ret; 1737 1737 - 1738 1738 - /* 1739 1739 - * When removing memory in the same granularity as it was added, 1740 1740 - * this function never fails. It might only fail if resources 1741 1741 - * have to be adjusted or split. We'll ignore the error, as 1742 1742 - * removing of memory cannot fail. 1743 1743 - */ 1744 1744 - ret = release_mem_region_adjustable(&iomem_resource, start, size); 1745 1745 - if (ret) { 1746 1746 - resource_size_t endres = start + size - 1; 1747 1747 - 1748 1748 - pr_warn("Unable to release resource <%pa-%pa> (%d)\n", 1749 1749 - &start, &endres, ret); 1750 1750 - } 1751 1751 - } 1752 1752 - 1753 1752 static int __ref try_remove_memory(int nid, u64 start, u64 size) 1754 1753 { 1755 1754 int rc = 0; ··· 1763 1802 memblock_remove(start, size); 1764 1803 } 1765 1804 1766 1766 - __release_memory_resource(start, size); 1805 1805 + release_mem_region_adjustable(start, size); 1767 1806 1768 1807 try_offline_node(nid); 1769 1808

+2 -1

mm/memremap.c

reviewed

··· 266 266 267 267 zone = &NODE_DATA(nid)->node_zones[ZONE_DEVICE]; 268 268 move_pfn_range_to_zone(zone, PHYS_PFN(range->start), 269 269 - PHYS_PFN(range_len(range)), params->altmap); 269 269 + PHYS_PFN(range_len(range)), params->altmap, 270 270 + MIGRATE_MOVABLE); 270 271 } 271 272 272 273 mem_hotplug_done();

+3 -8

mm/migrate.c

reviewed

··· 1223 1223 * we want to retry. 1224 1224 */ 1225 1225 if (rc == MIGRATEPAGE_SUCCESS) { 1226 1226 - put_page(page); 1227 1227 - if (reason == MR_MEMORY_FAILURE) { 1226 1226 + if (reason != MR_MEMORY_FAILURE) 1228 1227 /* 1229 1229 - * Set PG_HWPoison on just freed page 1230 1230 - * intentionally. Although it's rather weird, 1231 1231 - * it's how HWPoison flag works at the moment. 1228 1228 + * We release the page in page_handle_poison. 1232 1229 */ 1233 1233 - if (set_hwpoison_free_buddy_page(page)) 1234 1234 - num_poisoned_pages_inc(); 1235 1235 - } 1230 1230 + put_page(page); 1236 1231 } else { 1237 1232 if (rc != -EAGAIN) { 1238 1233 if (likely(!__PageMovable(page))) {

+2 -5

mm/mmap.c

reviewed

··· 619 619 struct address_space *mapping = file->f_mapping; 620 620 621 621 if (vma->vm_flags & VM_DENYWRITE) 622 622 - atomic_dec(&file_inode(file)->i_writecount); 622 622 + put_write_access(file_inode(file)); 623 623 if (vma->vm_flags & VM_SHARED) 624 624 mapping_allow_writable(mapping); 625 625 ··· 2562 2562 if (vma && (vma->vm_start <= addr)) 2563 2563 return vma; 2564 2564 /* don't alter vm_end if the coredump is running */ 2565 2565 - if (!prev || !mmget_still_valid(mm) || expand_stack(prev, addr)) 2565 2565 + if (!prev || expand_stack(prev, addr)) 2566 2566 return NULL; 2567 2567 if (prev->vm_flags & VM_LOCKED) 2568 2568 populate_vma_page_range(prev, addr, prev->vm_end, NULL); ··· 2587 2587 if (vma->vm_start <= addr) 2588 2588 return vma; 2589 2589 if (!(vma->vm_flags & VM_GROWSDOWN)) 2590 2590 - return NULL; 2591 2591 - /* don't alter vm_start if the coredump is running */ 2592 2592 - if (!mmget_still_valid(mm)) 2593 2590 return NULL; 2594 2591 start = vma->vm_start; 2595 2592 if (expand_stack(vma, addr))

+1 -1

mm/mmu_notifier.c

reviewed

··· 913 913 return -EOVERFLOW; 914 914 915 915 /* Must call with a mmget() held */ 916 916 - if (WARN_ON(atomic_read(&mm->mm_count) <= 0)) 916 916 + if (WARN_ON(atomic_read(&mm->mm_users) <= 0)) 917 917 return -EINVAL; 918 918 919 919 /* pairs with mmdrop in mmu_interval_notifier_remove() */

+1

mm/page-writeback.c

reviewed

··· 2849 2849 */ 2850 2850 void wait_for_stable_page(struct page *page) 2851 2851 { 2852 2852 + page = thp_head(page); 2852 2853 if (page->mapping->host->i_sb->s_iflags & SB_I_STABLE_WRITES) 2853 2854 wait_on_page_writeback(page); 2854 2855 }

+157 -84

mm/page_alloc.c

reviewed

··· 78 78 #include "shuffle.h" 79 79 #include "page_reporting.h" 80 80 81 81 + /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */ 82 82 + typedef int __bitwise fpi_t; 83 83 + 84 84 + /* No special request */ 85 85 + #define FPI_NONE ((__force fpi_t)0) 86 86 + 87 87 + /* 88 88 + * Skip free page reporting notification for the (possibly merged) page. 89 89 + * This does not hinder free page reporting from grabbing the page, 90 90 + * reporting it and marking it "reported" - it only skips notifying 91 91 + * the free page reporting infrastructure about a newly freed page. For 92 92 + * example, used when temporarily pulling a page from a freelist and 93 93 + * putting it back unmodified. 94 94 + */ 95 95 + #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) 96 96 + 97 97 + /* 98 98 + * Place the (possibly merged) page to the tail of the freelist. Will ignore 99 99 + * page shuffling (relevant code - e.g., memory onlining - is expected to 100 100 + * shuffle the whole zone). 101 101 + * 102 102 + * Note: No code should rely on this flag for correctness - it's purely 103 103 + * to allow for optimizations when handing back either fresh pages 104 104 + * (memory onlining) or untouched pages (page isolation, free page 105 105 + * reporting). 106 106 + */ 107 107 + #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) 108 108 + 81 109 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 82 110 static DEFINE_MUTEX(pcp_batch_high_lock); 83 111 #define MIN_PERCPU_PAGELIST_FRACTION (8) ··· 275 247 unsigned int pageblock_order __read_mostly; 276 248 #endif 277 249 278 278 - static void __free_pages_ok(struct page *page, unsigned int order); 250 250 + static void __free_pages_ok(struct page *page, unsigned int order, 251 251 + fpi_t fpi_flags); 279 252 280 253 /* 281 254 * results with 256, 32 in the lowmem_reserve sysctl: ··· 688 659 void free_compound_page(struct page *page) 689 660 { 690 661 mem_cgroup_uncharge(page); 691 691 - __free_pages_ok(page, compound_order(page)); 662 662 + __free_pages_ok(page, compound_order(page), FPI_NONE); 692 663 } 693 664 694 665 void prep_compound_page(struct page *page, unsigned int order) ··· 792 763 unsigned int order, int migratetype) {} 793 764 #endif 794 765 795 795 - static inline void set_page_order(struct page *page, unsigned int order) 766 766 + static inline void set_buddy_order(struct page *page, unsigned int order) 796 767 { 797 768 set_page_private(page, order); 798 769 __SetPageBuddy(page); ··· 817 788 if (!page_is_guard(buddy) && !PageBuddy(buddy)) 818 789 return false; 819 790 820 820 - if (page_order(buddy) != order) 791 791 + if (buddy_order(buddy) != order) 821 792 return false; 822 793 823 794 /* ··· 902 873 area->nr_free++; 903 874 } 904 875 905 905 - /* Used for pages which are on another list */ 876 876 + /* 877 877 + * Used for pages which are on another list. Move the pages to the tail 878 878 + * of the list - so the moved pages won't immediately be considered for 879 879 + * allocation again (e.g., optimization for memory onlining). 880 880 + */ 906 881 static inline void move_to_free_list(struct page *page, struct zone *zone, 907 882 unsigned int order, int migratetype) 908 883 { 909 884 struct free_area *area = &zone->free_area[order]; 910 885 911 911 - list_move(&page->lru, &area->free_list[migratetype]); 886 886 + list_move_tail(&page->lru, &area->free_list[migratetype]); 912 887 } 913 888 914 889 static inline void del_page_from_free_list(struct page *page, struct zone *zone, ··· 985 952 static inline void __free_one_page(struct page *page, 986 953 unsigned long pfn, 987 954 struct zone *zone, unsigned int order, 988 988 - int migratetype, bool report) 955 955 + int migratetype, fpi_t fpi_flags) 989 956 { 990 957 struct capture_control *capc = task_capc(zone); 991 958 unsigned long buddy_pfn; ··· 1059 1026 } 1060 1027 1061 1028 done_merging: 1062 1062 - set_page_order(page, order); 1029 1029 + set_buddy_order(page, order); 1063 1030 1064 1064 - if (is_shuffle_order(order)) 1031 1031 + if (fpi_flags & FPI_TO_TAIL) 1032 1032 + to_tail = true; 1033 1033 + else if (is_shuffle_order(order)) 1065 1034 to_tail = shuffle_pick_tail(); 1066 1035 else 1067 1036 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); ··· 1074 1039 add_to_free_list(page, zone, order, migratetype); 1075 1040 1076 1041 /* Notify page reporting subsystem of freed page */ 1077 1077 - if (report) 1042 1042 + if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY)) 1078 1043 page_reporting_notify_free(order); 1079 1044 } 1080 1045 ··· 1208 1173 VM_BUG_ON_PAGE(PageTail(page), page); 1209 1174 1210 1175 trace_mm_page_free(page, order); 1176 1176 + 1177 1177 + if (unlikely(PageHWPoison(page)) && !order) { 1178 1178 + /* 1179 1179 + * Do not let hwpoison pages hit pcplists/buddy 1180 1180 + * Untie memcg state and reset page's owner 1181 1181 + */ 1182 1182 + if (memcg_kmem_enabled() && PageKmemcg(page)) 1183 1183 + __memcg_kmem_uncharge_page(page, order); 1184 1184 + reset_page_owner(page, order); 1185 1185 + return false; 1186 1186 + } 1211 1187 1212 1188 /* 1213 1189 * Check tail pages before head page information is cleared to ··· 1415 1369 if (unlikely(isolated_pageblocks)) 1416 1370 mt = get_pageblock_migratetype(page); 1417 1371 1418 1418 - __free_one_page(page, page_to_pfn(page), zone, 0, mt, true); 1372 1372 + __free_one_page(page, page_to_pfn(page), zone, 0, mt, FPI_NONE); 1419 1373 trace_mm_page_pcpu_drain(page, 0, mt); 1420 1374 } 1421 1375 spin_unlock(&zone->lock); ··· 1424 1378 static void free_one_page(struct zone *zone, 1425 1379 struct page *page, unsigned long pfn, 1426 1380 unsigned int order, 1427 1427 - int migratetype) 1381 1381 + int migratetype, fpi_t fpi_flags) 1428 1382 { 1429 1383 spin_lock(&zone->lock); 1430 1384 if (unlikely(has_isolate_pageblock(zone) || 1431 1385 is_migrate_isolate(migratetype))) { 1432 1386 migratetype = get_pfnblock_migratetype(page, pfn); 1433 1387 } 1434 1434 - __free_one_page(page, pfn, zone, order, migratetype, true); 1388 1388 + __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1435 1389 spin_unlock(&zone->lock); 1436 1390 } 1437 1391 ··· 1509 1463 } 1510 1464 } 1511 1465 1512 1512 - static void __free_pages_ok(struct page *page, unsigned int order) 1466 1466 + static void __free_pages_ok(struct page *page, unsigned int order, 1467 1467 + fpi_t fpi_flags) 1513 1468 { 1514 1469 unsigned long flags; 1515 1470 int migratetype; ··· 1522 1475 migratetype = get_pfnblock_migratetype(page, pfn); 1523 1476 local_irq_save(flags); 1524 1477 __count_vm_events(PGFREE, 1 << order); 1525 1525 - free_one_page(page_zone(page), page, pfn, order, migratetype); 1478 1478 + free_one_page(page_zone(page), page, pfn, order, migratetype, 1479 1479 + fpi_flags); 1526 1480 local_irq_restore(flags); 1527 1481 } 1528 1482 ··· 1533 1485 struct page *p = page; 1534 1486 unsigned int loop; 1535 1487 1488 1488 + /* 1489 1489 + * When initializing the memmap, __init_single_page() sets the refcount 1490 1490 + * of all pages to 1 ("allocated"/"not free"). We have to set the 1491 1491 + * refcount of all involved pages to 0. 1492 1492 + */ 1536 1493 prefetchw(p); 1537 1494 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1538 1495 prefetchw(p + 1); ··· 1548 1495 set_page_count(p, 0); 1549 1496 1550 1497 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1551 1551 - set_page_refcounted(page); 1552 1552 - __free_pages(page, order); 1498 1498 + 1499 1499 + /* 1500 1500 + * Bypass PCP and place fresh pages right to the tail, primarily 1501 1501 + * relevant for memory onlining. 1502 1502 + */ 1503 1503 + __free_pages_ok(page, order, FPI_TO_TAIL); 1553 1504 } 1554 1505 1555 1506 #ifdef CONFIG_NEED_MULTIPLE_NODES ··· 2178 2121 continue; 2179 2122 2180 2123 add_to_free_list(&page[size], zone, high, migratetype); 2181 2181 - set_page_order(&page[size], high); 2124 2124 + set_buddy_order(&page[size], high); 2182 2125 } 2183 2126 } 2184 2127 ··· 2356 2299 #endif 2357 2300 2358 2301 /* 2359 2359 - * Move the free pages in a range to the free lists of the requested type. 2302 2302 + * Move the free pages in a range to the freelist tail of the requested type. 2360 2303 * Note that start_page and end_pages are not aligned on a pageblock 2361 2304 * boundary. If alignment is required, use move_freepages_block() 2362 2305 */ ··· 2392 2335 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2393 2336 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2394 2337 2395 2395 - order = page_order(page); 2338 2338 + order = buddy_order(page); 2396 2339 move_to_free_list(page, zone, order, migratetype); 2397 2340 page += 1 << order; 2398 2341 pages_moved += 1 << order; ··· 2516 2459 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2517 2460 unsigned int alloc_flags, int start_type, bool whole_block) 2518 2461 { 2519 2519 - unsigned int current_order = page_order(page); 2462 2462 + unsigned int current_order = buddy_order(page); 2520 2463 int free_pages, movable_pages, alike_pages; 2521 2464 int old_block_type; 2522 2465 ··· 3180 3123 */ 3181 3124 if (migratetype >= MIGRATE_PCPTYPES) { 3182 3125 if (unlikely(is_migrate_isolate(migratetype))) { 3183 3183 - free_one_page(zone, page, pfn, 0, migratetype); 3126 3126 + free_one_page(zone, page, pfn, 0, migratetype, 3127 3127 + FPI_NONE); 3184 3128 return; 3185 3129 } 3186 3130 migratetype = MIGRATE_MOVABLE; ··· 3267 3209 3268 3210 for (i = 1; i < (1 << order); i++) 3269 3211 set_page_refcounted(page + i); 3270 3270 - split_page_owner(page, order); 3212 3212 + split_page_owner(page, 1 << order); 3271 3213 } 3272 3214 EXPORT_SYMBOL_GPL(split_page); 3273 3215 ··· 3336 3278 lockdep_assert_held(&zone->lock); 3337 3279 3338 3280 /* Return isolated page to tail of freelist. */ 3339 3339 - __free_one_page(page, page_to_pfn(page), zone, order, mt, false); 3281 3281 + __free_one_page(page, page_to_pfn(page), zone, order, mt, 3282 3282 + FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL); 3340 3283 } 3341 3284 3342 3285 /* ··· 5004 4945 if (order == 0) /* Via pcp? */ 5005 4946 free_unref_page(page); 5006 4947 else 5007 5007 - __free_pages_ok(page, order); 4948 4948 + __free_pages_ok(page, order, FPI_NONE); 5008 4949 } 5009 4950 5010 4951 void __free_pages(struct page *page, unsigned int order) ··· 6038 5979 * Initially all pages are reserved - free ones are freed 6039 5980 * up by memblock_free_all() once the early boot process is 6040 5981 * done. Non-atomic initialization, single-pass. 5982 5982 + * 5983 5983 + * All aligned pageblocks are initialized to the specified migratetype 5984 5984 + * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 5985 5985 + * zone stats (e.g., nr_isolate_pageblock) are touched. 6041 5986 */ 6042 5987 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 6043 6043 - unsigned long start_pfn, enum meminit_context context, 6044 6044 - struct vmem_altmap *altmap) 5988 5988 + unsigned long start_pfn, 5989 5989 + enum meminit_context context, 5990 5990 + struct vmem_altmap *altmap, int migratetype) 6045 5991 { 6046 5992 unsigned long pfn, end_pfn = start_pfn + size; 6047 5993 struct page *page; ··· 6090 6026 __SetPageReserved(page); 6091 6027 6092 6028 /* 6093 6093 - * Mark the block movable so that blocks are reserved for 6094 6094 - * movable at startup. This will force kernel allocations 6095 6095 - * to reserve their blocks rather than leaking throughout 6096 6096 - * the address space during boot when many long-lived 6097 6097 - * kernel allocations are made. 6098 6098 - * 6099 6099 - * bitmap is created for zone's valid pfn range. but memmap 6100 6100 - * can be created for invalid pages (for alignment) 6101 6101 - * check here not to call set_pageblock_migratetype() against 6102 6102 - * pfn out of zone. 6029 6029 + * Usually, we want to mark the pageblock MIGRATE_MOVABLE, 6030 6030 + * such that unmovable allocations won't be scattered all 6031 6031 + * over the place during system boot. 6103 6032 */ 6104 6104 - if (!(pfn & (pageblock_nr_pages - 1))) { 6105 6105 - set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6033 6033 + if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6034 6034 + set_pageblock_migratetype(page, migratetype); 6106 6035 cond_resched(); 6107 6036 } 6108 6037 pfn++; ··· 6157 6100 * the address space during boot when many long-lived 6158 6101 * kernel allocations are made. 6159 6102 * 6160 6160 - * bitmap is created for zone's valid pfn range. but memmap 6161 6161 - * can be created for invalid pages (for alignment) 6162 6162 - * check here not to call set_pageblock_migratetype() against 6163 6163 - * pfn out of zone. 6164 6164 - * 6165 6103 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 6166 6104 * because this is done early in section_activate() 6167 6105 */ 6168 6168 - if (!(pfn & (pageblock_nr_pages - 1))) { 6106 6106 + if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6169 6107 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6170 6108 cond_resched(); 6171 6109 } ··· 6195 6143 if (end_pfn > start_pfn) { 6196 6144 size = end_pfn - start_pfn; 6197 6145 memmap_init_zone(size, nid, zone, start_pfn, 6198 6198 - MEMINIT_EARLY, NULL); 6146 6146 + MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 6199 6147 } 6200 6148 } 6201 6149 } ··· 8344 8292 */ 8345 8293 if (!page_ref_count(page)) { 8346 8294 if (PageBuddy(page)) 8347 8347 - iter += (1 << page_order(page)) - 1; 8295 8295 + iter += (1 << buddy_order(page)) - 1; 8348 8296 continue; 8349 8297 } 8350 8298 ··· 8509 8457 8510 8458 ret = start_isolate_page_range(pfn_max_align_down(start), 8511 8459 pfn_max_align_up(end), migratetype, 0); 8512 8512 - if (ret < 0) 8460 8460 + if (ret) 8513 8461 return ret; 8514 8462 8515 8463 /* ··· 8557 8505 } 8558 8506 8559 8507 if (outer_start != start) { 8560 8560 - order = page_order(pfn_to_page(outer_start)); 8508 8508 + order = buddy_order(pfn_to_page(outer_start)); 8561 8509 8562 8510 /* 8563 8511 * outer_start page could be small order buddy page and ··· 8745 8693 8746 8694 #ifdef CONFIG_MEMORY_HOTREMOVE 8747 8695 /* 8748 8748 - * All pages in the range must be in a single zone and isolated 8749 8749 - * before calling this. 8696 8696 + * All pages in the range must be in a single zone, must not contain holes, 8697 8697 + * must span full sections, and must be isolated before calling this function. 8750 8698 */ 8751 8751 - unsigned long 8752 8752 - __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 8699 8699 + void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 8753 8700 { 8701 8701 + unsigned long pfn = start_pfn; 8754 8702 struct page *page; 8755 8703 struct zone *zone; 8756 8704 unsigned int order; 8757 8757 - unsigned long pfn; 8758 8705 unsigned long flags; 8759 8759 - unsigned long offlined_pages = 0; 8760 8760 - 8761 8761 - /* find the first valid pfn */ 8762 8762 - for (pfn = start_pfn; pfn < end_pfn; pfn++) 8763 8763 - if (pfn_valid(pfn)) 8764 8764 - break; 8765 8765 - if (pfn == end_pfn) 8766 8766 - return offlined_pages; 8767 8706 8768 8707 offline_mem_sections(pfn, end_pfn); 8769 8708 zone = page_zone(pfn_to_page(pfn)); 8770 8709 spin_lock_irqsave(&zone->lock, flags); 8771 8771 - pfn = start_pfn; 8772 8710 while (pfn < end_pfn) { 8773 8773 - if (!pfn_valid(pfn)) { 8774 8774 - pfn++; 8775 8775 - continue; 8776 8776 - } 8777 8711 page = pfn_to_page(pfn); 8778 8712 /* 8779 8713 * The HWPoisoned page may be not in buddy system, and ··· 8767 8729 */ 8768 8730 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 8769 8731 pfn++; 8770 8770 - offlined_pages++; 8771 8732 continue; 8772 8733 } 8773 8734 /* ··· 8777 8740 BUG_ON(page_count(page)); 8778 8741 BUG_ON(PageBuddy(page)); 8779 8742 pfn++; 8780 8780 - offlined_pages++; 8781 8743 continue; 8782 8744 } 8783 8745 8784 8746 BUG_ON(page_count(page)); 8785 8747 BUG_ON(!PageBuddy(page)); 8786 8786 - order = page_order(page); 8787 8787 - offlined_pages += 1 << order; 8748 8748 + order = buddy_order(page); 8788 8749 del_page_from_free_list(page, zone, order); 8789 8750 pfn += (1 << order); 8790 8751 } 8791 8752 spin_unlock_irqrestore(&zone->lock, flags); 8792 8792 - 8793 8793 - return offlined_pages; 8794 8753 } 8795 8754 #endif 8796 8755 ··· 8801 8768 for (order = 0; order < MAX_ORDER; order++) { 8802 8769 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8803 8770 8804 8804 - if (PageBuddy(page_head) && page_order(page_head) >= order) 8771 8771 + if (PageBuddy(page_head) && buddy_order(page_head) >= order) 8805 8772 break; 8806 8773 } 8807 8774 spin_unlock_irqrestore(&zone->lock, flags); ··· 8811 8778 8812 8779 #ifdef CONFIG_MEMORY_FAILURE 8813 8780 /* 8814 8814 - * Set PG_hwpoison flag if a given page is confirmed to be a free page. This 8815 8815 - * test is performed under the zone lock to prevent a race against page 8816 8816 - * allocation. 8781 8781 + * Break down a higher-order page in sub-pages, and keep our target out of 8782 8782 + * buddy allocator. 8817 8783 */ 8818 8818 - bool set_hwpoison_free_buddy_page(struct page *page) 8784 8784 + static void break_down_buddy_pages(struct zone *zone, struct page *page, 8785 8785 + struct page *target, int low, int high, 8786 8786 + int migratetype) 8787 8787 + { 8788 8788 + unsigned long size = 1 << high; 8789 8789 + struct page *current_buddy, *next_page; 8790 8790 + 8791 8791 + while (high > low) { 8792 8792 + high--; 8793 8793 + size >>= 1; 8794 8794 + 8795 8795 + if (target >= &page[size]) { 8796 8796 + next_page = page + size; 8797 8797 + current_buddy = page; 8798 8798 + } else { 8799 8799 + next_page = page; 8800 8800 + current_buddy = page + size; 8801 8801 + } 8802 8802 + 8803 8803 + if (set_page_guard(zone, current_buddy, high, migratetype)) 8804 8804 + continue; 8805 8805 + 8806 8806 + if (current_buddy != target) { 8807 8807 + add_to_free_list(current_buddy, zone, high, migratetype); 8808 8808 + set_buddy_order(current_buddy, high); 8809 8809 + page = next_page; 8810 8810 + } 8811 8811 + } 8812 8812 + } 8813 8813 + 8814 8814 + /* 8815 8815 + * Take a page that will be marked as poisoned off the buddy allocator. 8816 8816 + */ 8817 8817 + bool take_page_off_buddy(struct page *page) 8819 8818 { 8820 8819 struct zone *zone = page_zone(page); 8821 8820 unsigned long pfn = page_to_pfn(page); 8822 8821 unsigned long flags; 8823 8822 unsigned int order; 8824 8824 - bool hwpoisoned = false; 8823 8823 + bool ret = false; 8825 8824 8826 8825 spin_lock_irqsave(&zone->lock, flags); 8827 8826 for (order = 0; order < MAX_ORDER; order++) { 8828 8827 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8828 8828 + int page_order = buddy_order(page_head); 8829 8829 8830 8830 - if (PageBuddy(page_head) && page_order(page_head) >= order) { 8831 8831 - if (!TestSetPageHWPoison(page)) 8832 8832 - hwpoisoned = true; 8830 8830 + if (PageBuddy(page_head) && page_order >= order) { 8831 8831 + unsigned long pfn_head = page_to_pfn(page_head); 8832 8832 + int migratetype = get_pfnblock_migratetype(page_head, 8833 8833 + pfn_head); 8834 8834 + 8835 8835 + del_page_from_free_list(page_head, zone, page_order); 8836 8836 + break_down_buddy_pages(zone, page_head, page, 0, 8837 8837 + page_order, migratetype); 8838 8838 + ret = true; 8833 8839 break; 8834 8840 } 8841 8841 + if (page_count(page_head) > 0) 8842 8842 + break; 8835 8843 } 8836 8844 spin_unlock_irqrestore(&zone->lock, flags); 8837 8837 - 8838 8838 - return hwpoisoned; 8845 8845 + return ret; 8839 8846 } 8840 8847 #endif

+9 -7

mm/page_isolation.c

reviewed

··· 88 88 * these pages to be merged. 89 89 */ 90 90 if (PageBuddy(page)) { 91 91 - order = page_order(page); 91 91 + order = buddy_order(page); 92 92 if (order >= pageblock_order) { 93 93 pfn = page_to_pfn(page); 94 94 buddy_pfn = __find_buddy_pfn(pfn, order); ··· 106 106 * If we isolate freepage with more than pageblock_order, there 107 107 * should be no freepage in the range, so we could avoid costly 108 108 * pageblock scanning for freepage moving. 109 109 + * 110 110 + * We didn't actually touch any of the isolated pages, so place them 111 111 + * to the tail of the freelist. This is an optimization for memory 112 112 + * onlining - just onlined memory won't immediately be considered for 113 113 + * allocation. 109 114 */ 110 115 if (!isolated_page) { 111 116 nr_pages = move_freepages_block(zone, page, migratetype, NULL); ··· 178 173 * (e.g. __offline_pages will need to call it after check for isolated range for 179 174 * a next retry). 180 175 * 181 181 - * Return: the number of isolated pageblocks on success and -EBUSY if any part 182 182 - * of range cannot be isolated. 176 176 + * Return: 0 on success and -EBUSY if any part of range cannot be isolated. 183 177 */ 184 178 int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, 185 179 unsigned migratetype, int flags) ··· 186 182 unsigned long pfn; 187 183 unsigned long undo_pfn; 188 184 struct page *page; 189 189 - int nr_isolate_pageblock = 0; 190 185 191 186 BUG_ON(!IS_ALIGNED(start_pfn, pageblock_nr_pages)); 192 187 BUG_ON(!IS_ALIGNED(end_pfn, pageblock_nr_pages)); ··· 199 196 undo_pfn = pfn; 200 197 goto undo; 201 198 } 202 202 - nr_isolate_pageblock++; 203 199 } 204 200 } 205 205 - return nr_isolate_pageblock; 201 201 + return 0; 206 202 undo: 207 203 for (pfn = start_pfn; 208 204 pfn < undo_pfn; ··· 261 259 * the correct MIGRATE_ISOLATE freelist. There is no 262 260 * simple way to verify that as VM_BUG_ON(), though. 263 261 */ 264 264 - pfn += 1 << page_order(page); 262 262 + pfn += 1 << buddy_order(page); 265 263 else if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) 266 264 /* A HWPoisoned page cannot be also PageBuddy */ 267 265 pfn++;

+5 -5

mm/page_owner.c

reviewed

··· 204 204 page_owner->last_migrate_reason = reason; 205 205 } 206 206 207 207 - void __split_page_owner(struct page *page, unsigned int order) 207 207 + void __split_page_owner(struct page *page, unsigned int nr) 208 208 { 209 209 int i; 210 210 struct page_ext *page_ext = lookup_page_ext(page); ··· 213 213 if (unlikely(!page_ext)) 214 214 return; 215 215 216 216 - for (i = 0; i < (1 << order); i++) { 216 216 + for (i = 0; i < nr; i++) { 217 217 page_owner = get_page_owner(page_ext); 218 218 page_owner->order = 0; 219 219 page_ext = page_ext_next(page_ext); ··· 295 295 if (PageBuddy(page)) { 296 296 unsigned long freepage_order; 297 297 298 298 - freepage_order = page_order_unsafe(page); 298 298 + freepage_order = buddy_order_unsafe(page); 299 299 if (freepage_order < MAX_ORDER) 300 300 pfn += (1UL << freepage_order) - 1; 301 301 continue; ··· 490 490 491 491 page = pfn_to_page(pfn); 492 492 if (PageBuddy(page)) { 493 493 - unsigned long freepage_order = page_order_unsafe(page); 493 493 + unsigned long freepage_order = buddy_order_unsafe(page); 494 494 495 495 if (freepage_order < MAX_ORDER) 496 496 pfn += (1UL << freepage_order) - 1; ··· 584 584 * heavy lock contention. 585 585 */ 586 586 if (PageBuddy(page)) { 587 587 - unsigned long order = page_order_unsafe(page); 587 587 + unsigned long order = buddy_order_unsafe(page); 588 588 589 589 if (order > 0 && order < MAX_ORDER) 590 590 pfn += (1UL << order) - 1;

+15 -5

mm/page_poison.c

reviewed

··· 8 8 #include <linux/ratelimit.h> 9 9 #include <linux/kasan.h> 10 10 11 11 - static bool want_page_poisoning __read_mostly; 11 11 + static DEFINE_STATIC_KEY_FALSE_RO(want_page_poisoning); 12 12 13 13 static int __init early_page_poison_param(char *buf) 14 14 { 15 15 - if (!buf) 16 16 - return -EINVAL; 17 17 - return strtobool(buf, &want_page_poisoning); 15 15 + int ret; 16 16 + bool tmp; 17 17 + 18 18 + ret = strtobool(buf, &tmp); 19 19 + if (ret) 20 20 + return ret; 21 21 + 22 22 + if (tmp) 23 23 + static_branch_enable(&want_page_poisoning); 24 24 + else 25 25 + static_branch_disable(&want_page_poisoning); 26 26 + 27 27 + return 0; 18 28 } 19 29 early_param("page_poison", early_page_poison_param); 20 30 ··· 41 31 * Page poisoning is debug page alloc for some arches. If 42 32 * either of those options are enabled, enable poisoning. 43 33 */ 44 44 - return (want_page_poisoning || 34 34 + return (static_branch_unlikely(&want_page_poisoning) || 45 35 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && 46 36 debug_pagealloc_enabled())); 47 37 }

+2 -2

mm/page_reporting.c

reviewed

··· 92 92 * report on the new larger page when we make our way 93 93 * up to that higher order. 94 94 */ 95 95 - if (PageBuddy(page) && page_order(page) == order) 95 95 + if (PageBuddy(page) && buddy_order(page) == order) 96 96 __SetPageReported(page); 97 97 } while ((sg = sg_next(sg))); 98 98 ··· 178 178 * the new head of the free list before we release the 179 179 * zone lock. 180 180 */ 181 181 - if (&page->lru != list && !list_is_first(&page->lru, list)) 181 181 + if (!list_is_first(&page->lru, list)) 182 182 list_rotate_to_front(&page->lru, list); 183 183 184 184 /* release lock before waiting on report processing */

+49 -81

mm/readahead.c

reviewed

··· 158 158 } 159 159 160 160 /** 161 161 - * page_cache_readahead_unbounded - Start unchecked readahead. 162 162 - * @mapping: File address space. 163 163 - * @file: This instance of the open file; used for authentication. 164 164 - * @index: First page index to read. 161 161 + * page_cache_ra_unbounded - Start unchecked readahead. 162 162 + * @ractl: Readahead control. 165 163 * @nr_to_read: The number of pages to read. 166 164 * @lookahead_size: Where to start the next readahead. 167 165 * ··· 171 173 * Context: File is referenced by caller. Mutexes may be held by caller. 172 174 * May sleep, but will not reenter filesystem to reclaim memory. 173 175 */ 174 174 - void page_cache_readahead_unbounded(struct address_space *mapping, 175 175 - struct file *file, pgoff_t index, unsigned long nr_to_read, 176 176 - unsigned long lookahead_size) 176 176 + void page_cache_ra_unbounded(struct readahead_control *ractl, 177 177 + unsigned long nr_to_read, unsigned long lookahead_size) 177 178 { 179 179 + struct address_space *mapping = ractl->mapping; 180 180 + unsigned long index = readahead_index(ractl); 178 181 LIST_HEAD(page_pool); 179 182 gfp_t gfp_mask = readahead_gfp_mask(mapping); 180 180 - struct readahead_control rac = { 181 181 - .mapping = mapping, 182 182 - .file = file, 183 183 - ._index = index, 184 184 - }; 185 183 unsigned long i; 186 184 187 185 /* ··· 198 204 for (i = 0; i < nr_to_read; i++) { 199 205 struct page *page = xa_load(&mapping->i_pages, index + i); 200 206 201 201 - BUG_ON(index + i != rac._index + rac._nr_pages); 207 207 + BUG_ON(index + i != ractl->_index + ractl->_nr_pages); 202 208 203 209 if (page && !xa_is_value(page)) { 204 210 /* ··· 209 215 * have a stable reference to this page, and it's 210 216 * not worth getting one just for that. 211 217 */ 212 212 - read_pages(&rac, &page_pool, true); 218 218 + read_pages(ractl, &page_pool, true); 213 219 continue; 214 220 } 215 221 ··· 222 228 } else if (add_to_page_cache_lru(page, mapping, index + i, 223 229 gfp_mask) < 0) { 224 230 put_page(page); 225 225 - read_pages(&rac, &page_pool, true); 231 231 + read_pages(ractl, &page_pool, true); 226 232 continue; 227 233 } 228 234 if (i == nr_to_read - lookahead_size) 229 235 SetPageReadahead(page); 230 230 - rac._nr_pages++; 236 236 + ractl->_nr_pages++; 231 237 } 232 238 233 239 /* ··· 235 241 * uptodate then the caller will launch readpage again, and 236 242 * will then handle the error. 237 243 */ 238 238 - read_pages(&rac, &page_pool, false); 244 244 + read_pages(ractl, &page_pool, false); 239 245 memalloc_nofs_restore(nofs); 240 246 } 241 241 - EXPORT_SYMBOL_GPL(page_cache_readahead_unbounded); 247 247 + EXPORT_SYMBOL_GPL(page_cache_ra_unbounded); 242 248 243 249 /* 244 244 - * __do_page_cache_readahead() actually reads a chunk of disk. It allocates 250 250 + * do_page_cache_ra() actually reads a chunk of disk. It allocates 245 251 * the pages first, then submits them for I/O. This avoids the very bad 246 252 * behaviour which would occur if page allocations are causing VM writeback. 247 253 * We really don't want to intermingle reads and writes like that. 248 254 */ 249 249 - void __do_page_cache_readahead(struct address_space *mapping, 250 250 - struct file *file, pgoff_t index, unsigned long nr_to_read, 251 251 - unsigned long lookahead_size) 255 255 + void do_page_cache_ra(struct readahead_control *ractl, 256 256 + unsigned long nr_to_read, unsigned long lookahead_size) 252 257 { 253 253 - struct inode *inode = mapping->host; 258 258 + struct inode *inode = ractl->mapping->host; 259 259 + unsigned long index = readahead_index(ractl); 254 260 loff_t isize = i_size_read(inode); 255 261 pgoff_t end_index; /* The last page we want to read */ 256 262 ··· 264 270 if (nr_to_read > end_index - index) 265 271 nr_to_read = end_index - index + 1; 266 272 267 267 - page_cache_readahead_unbounded(mapping, file, index, nr_to_read, 268 268 - lookahead_size); 273 273 + page_cache_ra_unbounded(ractl, nr_to_read, lookahead_size); 269 274 } 270 275 271 276 /* 272 277 * Chunk the readahead into 2 megabyte units, so that we don't pin too much 273 278 * memory at once. 274 279 */ 275 275 - void force_page_cache_readahead(struct address_space *mapping, 276 276 - struct file *filp, pgoff_t index, unsigned long nr_to_read) 280 280 + void force_page_cache_ra(struct readahead_control *ractl, 281 281 + struct file_ra_state *ra, unsigned long nr_to_read) 277 282 { 283 283 + struct address_space *mapping = ractl->mapping; 278 284 struct backing_dev_info *bdi = inode_to_bdi(mapping->host); 279 279 - struct file_ra_state *ra = &filp->f_ra; 280 280 - unsigned long max_pages; 285 285 + unsigned long max_pages, index; 281 286 282 287 if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readpages && 283 288 !mapping->a_ops->readahead)) ··· 286 293 * If the request exceeds the readahead window, allow the read to 287 294 * be up to the optimal hardware IO size 288 295 */ 296 296 + index = readahead_index(ractl); 289 297 max_pages = max_t(unsigned long, bdi->io_pages, ra->ra_pages); 290 290 - nr_to_read = min(nr_to_read, max_pages); 298 298 + nr_to_read = min_t(unsigned long, nr_to_read, max_pages); 291 299 while (nr_to_read) { 292 300 unsigned long this_chunk = (2 * 1024 * 1024) / PAGE_SIZE; 293 301 294 302 if (this_chunk > nr_to_read) 295 303 this_chunk = nr_to_read; 296 296 - __do_page_cache_readahead(mapping, filp, index, this_chunk, 0); 304 304 + ractl->_index = index; 305 305 + do_page_cache_ra(ractl, this_chunk, 0); 297 306 298 307 index += this_chunk; 299 308 nr_to_read -= this_chunk; ··· 432 437 /* 433 438 * A minimal readahead algorithm for trivial sequential/random reads. 434 439 */ 435 435 - static void ondemand_readahead(struct address_space *mapping, 436 436 - struct file_ra_state *ra, struct file *filp, 437 437 - bool hit_readahead_marker, pgoff_t index, 440 440 + static void ondemand_readahead(struct readahead_control *ractl, 441 441 + struct file_ra_state *ra, bool hit_readahead_marker, 438 442 unsigned long req_size) 439 443 { 440 440 - struct backing_dev_info *bdi = inode_to_bdi(mapping->host); 444 444 + struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host); 441 445 unsigned long max_pages = ra->ra_pages; 442 446 unsigned long add_pages; 447 447 + unsigned long index = readahead_index(ractl); 443 448 pgoff_t prev_index; 444 449 445 450 /* ··· 477 482 pgoff_t start; 478 483 479 484 rcu_read_lock(); 480 480 - start = page_cache_next_miss(mapping, index + 1, max_pages); 485 485 + start = page_cache_next_miss(ractl->mapping, index + 1, 486 486 + max_pages); 481 487 rcu_read_unlock(); 482 488 483 489 if (!start || start - index > max_pages) ··· 511 515 * Query the page cache and look for the traces(cached history pages) 512 516 * that a sequential stream would leave behind. 513 517 */ 514 514 - if (try_context_readahead(mapping, ra, index, req_size, max_pages)) 518 518 + if (try_context_readahead(ractl->mapping, ra, index, req_size, 519 519 + max_pages)) 515 520 goto readit; 516 521 517 522 /* 518 523 * standalone, small random read 519 524 * Read as is, and do not pollute the readahead state. 520 525 */ 521 521 - __do_page_cache_readahead(mapping, filp, index, req_size, 0); 526 526 + do_page_cache_ra(ractl, req_size, 0); 522 527 return; 523 528 524 529 initial_readahead: ··· 545 548 } 546 549 } 547 550 548 548 - ra_submit(ra, mapping, filp); 551 551 + ractl->_index = ra->start; 552 552 + do_page_cache_ra(ractl, ra->size, ra->async_size); 549 553 } 550 554 551 551 - /** 552 552 - * page_cache_sync_readahead - generic file readahead 553 553 - * @mapping: address_space which holds the pagecache and I/O vectors 554 554 - * @ra: file_ra_state which holds the readahead state 555 555 - * @filp: passed on to ->readpage() and ->readpages() 556 556 - * @index: Index of first page to be read. 557 557 - * @req_count: Total number of pages being read by the caller. 558 558 - * 559 559 - * page_cache_sync_readahead() should be called when a cache miss happened: 560 560 - * it will submit the read. The readahead logic may decide to piggyback more 561 561 - * pages onto the read request if access patterns suggest it will improve 562 562 - * performance. 563 563 - */ 564 564 - void page_cache_sync_readahead(struct address_space *mapping, 565 565 - struct file_ra_state *ra, struct file *filp, 566 566 - pgoff_t index, unsigned long req_count) 555 555 + void page_cache_sync_ra(struct readahead_control *ractl, 556 556 + struct file_ra_state *ra, unsigned long req_count) 567 557 { 568 558 /* no read-ahead */ 569 559 if (!ra->ra_pages) ··· 560 576 return; 561 577 562 578 /* be dumb */ 563 563 - if (filp && (filp->f_mode & FMODE_RANDOM)) { 564 564 - force_page_cache_readahead(mapping, filp, index, req_count); 579 579 + if (ractl->file && (ractl->file->f_mode & FMODE_RANDOM)) { 580 580 + force_page_cache_ra(ractl, ra, req_count); 565 581 return; 566 582 } 567 583 568 584 /* do read-ahead */ 569 569 - ondemand_readahead(mapping, ra, filp, false, index, req_count); 585 585 + ondemand_readahead(ractl, ra, false, req_count); 570 586 } 571 571 - EXPORT_SYMBOL_GPL(page_cache_sync_readahead); 587 587 + EXPORT_SYMBOL_GPL(page_cache_sync_ra); 572 588 573 573 - /** 574 574 - * page_cache_async_readahead - file readahead for marked pages 575 575 - * @mapping: address_space which holds the pagecache and I/O vectors 576 576 - * @ra: file_ra_state which holds the readahead state 577 577 - * @filp: passed on to ->readpage() and ->readpages() 578 578 - * @page: The page at @index which triggered the readahead call. 579 579 - * @index: Index of first page to be read. 580 580 - * @req_count: Total number of pages being read by the caller. 581 581 - * 582 582 - * page_cache_async_readahead() should be called when a page is used which 583 583 - * is marked as PageReadahead; this is a marker to suggest that the application 584 584 - * has used up enough of the readahead window that we should start pulling in 585 585 - * more pages. 586 586 - */ 587 587 - void 588 588 - page_cache_async_readahead(struct address_space *mapping, 589 589 - struct file_ra_state *ra, struct file *filp, 590 590 - struct page *page, pgoff_t index, 591 591 - unsigned long req_count) 589 589 + void page_cache_async_ra(struct readahead_control *ractl, 590 590 + struct file_ra_state *ra, struct page *page, 591 591 + unsigned long req_count) 592 592 { 593 593 /* no read-ahead */ 594 594 if (!ra->ra_pages) ··· 589 621 /* 590 622 * Defer asynchronous read-ahead on IO congestion. 591 623 */ 592 592 - if (inode_read_congested(mapping->host)) 624 624 + if (inode_read_congested(ractl->mapping->host)) 593 625 return; 594 626 595 627 if (blk_cgroup_congested()) 596 628 return; 597 629 598 630 /* do read-ahead */ 599 599 - ondemand_readahead(mapping, ra, filp, true, index, req_count); 631 631 + ondemand_readahead(ractl, ra, true, req_count); 600 632 } 601 601 - EXPORT_SYMBOL_GPL(page_cache_async_readahead); 633 633 + EXPORT_SYMBOL_GPL(page_cache_async_ra); 602 634 603 635 ssize_t ksys_readahead(int fd, loff_t offset, size_t count) 604 636 {

+5 -5

mm/rmap.c

reviewed

··· 1205 1205 VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page); 1206 1206 lock_page_memcg(page); 1207 1207 if (compound && PageTransHuge(page)) { 1208 1208 - for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { 1208 1208 + for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { 1209 1209 if (atomic_inc_and_test(&page[i]._mapcount)) 1210 1210 nr++; 1211 1211 } ··· 1246 1246 1247 1247 /* page still mapped by someone else? */ 1248 1248 if (compound && PageTransHuge(page)) { 1249 1249 - for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { 1249 1249 + for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { 1250 1250 if (atomic_add_negative(-1, &page[i]._mapcount)) 1251 1251 nr++; 1252 1252 } ··· 1293 1293 * Subpages can be mapped with PTEs too. Check how many of 1294 1294 * them are still mapped. 1295 1295 */ 1296 1296 - for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { 1296 1296 + for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { 1297 1297 if (atomic_add_negative(-1, &page[i]._mapcount)) 1298 1298 nr++; 1299 1299 } ··· 1303 1303 * page of the compound page is unmapped, but at least one 1304 1304 * small page is still mapped. 1305 1305 */ 1306 1306 - if (nr && nr < HPAGE_PMD_NR) 1306 1306 + if (nr && nr < thp_nr_pages(page)) 1307 1307 deferred_split_huge_page(page); 1308 1308 } else { 1309 1309 - nr = HPAGE_PMD_NR; 1309 1309 + nr = thp_nr_pages(page); 1310 1310 } 1311 1311 1312 1312 if (unlikely(PageMlocked(page)))

+1 -1

mm/shmem.c

reviewed

··· 3984 3984 .parameters = shmem_fs_parameters, 3985 3985 #endif 3986 3986 .kill_sb = kill_litter_super, 3987 3987 - .fs_flags = FS_USERNS_MOUNT, 3987 3987 + .fs_flags = FS_USERNS_MOUNT | FS_THP_SUPPORT, 3988 3988 }; 3989 3989 3990 3990 int __init shmem_init(void)

+1 -1

mm/shuffle.c

reviewed

··· 60 60 * ...is the page on the same list as the page we will 61 61 * shuffle it with? 62 62 */ 63 63 - if (page_order(page) != order) 63 63 + if (buddy_order(page) != order) 64 64 return NULL; 65 65 66 66 return page;

+1 -1

mm/slab.c

reviewed

··· 1062 1062 * Even if all the cpus of a node are down, we don't free the 1063 1063 * kmem_cache_node of any cache. This to avoid a race between cpu_down, and 1064 1064 * a kmalloc allocation from another cpu for memory from the node of 1065 1065 - * the cpu going down. The list3 structure is usually allocated from 1065 1065 + * the cpu going down. The kmem_cache_node structure is usually allocated from 1066 1066 * kmem_cache_create() and gets destroyed at kmem_cache_destroy(). 1067 1067 */ 1068 1068 int slab_dead_cpu(unsigned int cpu)

-1

mm/slab.h

reviewed

··· 46 46 #include <linux/kmemleak.h> 47 47 #include <linux/random.h> 48 48 #include <linux/sched/mm.h> 49 49 - #include <linux/kmemleak.h> 50 49 51 50 /* 52 51 * State of the slab allocator.

+1 -1

mm/slub.c

reviewed

··· 1956 1956 /* 1957 1957 * Racy check. If we mistakenly see no partial slabs then we 1958 1958 * just allocate an empty slab. If we mistakenly try to get a 1959 1959 - * partial slab and there is none available then get_partials() 1959 1959 + * partial slab and there is none available then get_partial() 1960 1960 * will return NULL. 1961 1961 */ 1962 1962 if (!n || !n->nr_partial)

+2

mm/sparse.c

reviewed

··· 312 312 return coded_mem_map; 313 313 } 314 314 315 315 + #ifdef CONFIG_MEMORY_HOTPLUG 315 316 /* 316 317 * Decode mem_map from the coded memmap 317 318 */ ··· 322 321 coded_mem_map &= SECTION_MAP_MASK; 323 322 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 324 323 } 324 324 + #endif /* CONFIG_MEMORY_HOTPLUG */ 325 325 326 326 static void __meminit sparse_init_one_section(struct mem_section *ms, 327 327 unsigned long pnum, struct page *mem_map,

+1 -1

mm/swap_state.c

reviewed

··· 246 246 goto fail; 247 247 /* 248 248 * Normally the page will be dirtied in unmap because its pte should be 249 249 - * dirty. A special case is MADV_FREE page. The page'e pte could have 249 249 + * dirty. A special case is MADV_FREE page. The page's pte could have 250 250 * dirty bit cleared but the page's SwapBacked bit is still set because 251 251 * clearing the dirty bit and SwapBacked bit has no lock protected. For 252 252 * such page, unmap will not set dirty bit for it, so page reclaim will

+3 -3

mm/truncate.c

reviewed

··· 168 168 * becomes orphaned. It will be left on the LRU and may even be mapped into 169 169 * user pagetables if we're racing with filemap_fault(). 170 170 * 171 171 - * We need to bale out if page->mapping is no longer equal to the original 171 171 + * We need to bail out if page->mapping is no longer equal to the original 172 172 * mapping. This happens a) when the VM reclaimed the page while we waited on 173 173 * its lock, b) when a concurrent invalidate_mapping_pages got there first and 174 174 * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space. ··· 177 177 truncate_cleanup_page(struct address_space *mapping, struct page *page) 178 178 { 179 179 if (page_mapped(page)) { 180 180 - pgoff_t nr = PageTransHuge(page) ? HPAGE_PMD_NR : 1; 180 180 + unsigned int nr = thp_nr_pages(page); 181 181 unmap_mapping_pages(mapping, page->index, nr, false); 182 182 } 183 183 184 184 if (page_has_private(page)) 185 185 - do_invalidatepage(page, 0, PAGE_SIZE); 185 185 + do_invalidatepage(page, 0, thp_size(page)); 186 186 187 187 /* 188 188 * Some filesystems seem to re-dirty the page even after

+2 -1

mm/util.c

reviewed

··· 69 69 * @s: the string to duplicate 70 70 * @gfp: the GFP mask used in the kmalloc() call when allocating memory 71 71 * 72 72 - * Note: Strings allocated by kstrdup_const should be freed by kfree_const. 72 72 + * Note: Strings allocated by kstrdup_const should be freed by kfree_const and 73 73 + * must not be passed to krealloc(). 73 74 * 74 75 * Return: source string if it is in .rodata section otherwise 75 76 * fallback to kstrdup.

+2 -3

mm/vmscan.c

reviewed

··· 725 725 * that isolated the page, the page cache and optional buffer 726 726 * heads at page->private. 727 727 */ 728 728 - int page_cache_pins = PageTransHuge(page) && PageSwapCache(page) ? 729 729 - HPAGE_PMD_NR : 1; 728 728 + int page_cache_pins = thp_nr_pages(page); 730 729 return page_count(page) - page_has_private(page) == 1 + page_cache_pins; 731 730 } 732 731 ··· 2239 2240 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 2240 2241 unsigned long anon_cost, file_cost, total_cost; 2241 2242 int swappiness = mem_cgroup_swappiness(memcg); 2242 2242 - u64 fraction[2]; 2243 2243 + u64 fraction[ANON_AND_FILE]; 2243 2244 u64 denominator = 0; /* gcc */ 2244 2245 enum scan_balance scan_balance; 2245 2246 unsigned long ap, fp;

+4 -4

mm/vmstat.c

reviewed

··· 325 325 326 326 t = __this_cpu_read(pcp->stat_threshold); 327 327 328 328 - if (unlikely(x > t || x < -t)) { 328 328 + if (unlikely(abs(x) > t)) { 329 329 zone_page_state_add(x, zone, item); 330 330 x = 0; 331 331 } ··· 350 350 351 351 t = __this_cpu_read(pcp->stat_threshold); 352 352 353 353 - if (unlikely(x > t || x < -t)) { 353 353 + if (unlikely(abs(x) > t)) { 354 354 node_page_state_add(x, pgdat, item); 355 355 x = 0; 356 356 } ··· 511 511 o = this_cpu_read(*p); 512 512 n = delta + o; 513 513 514 514 - if (n > t || n < -t) { 514 514 + if (abs(n) > t) { 515 515 int os = overstep_mode * (t >> 1) ; 516 516 517 517 /* Overflow must be added to zone counters */ ··· 573 573 o = this_cpu_read(*p); 574 574 n = delta + o; 575 575 576 576 - if (n > t || n < -t) { 576 576 + if (abs(n) > t) { 577 577 int os = overstep_mode * (t >> 1) ; 578 578 579 579 /* Overflow must be added to node counters */

+1 -1

mm/workingset.c

reviewed

··· 216 216 217 217 /** 218 218 * workingset_age_nonresident - age non-resident entries as LRU ages 219 219 - * @memcg: the lruvec that was aged 219 219 + * @lruvec: the lruvec that was aged 220 220 * @nr_pages: the number of pages to count 221 221 * 222 222 * As in-memory pages are aged, non-resident pages need to be aged as

+9 -1

scripts/Makefile.ubsan

reviewed

··· 4 4 endif 5 5 6 6 ifdef CONFIG_UBSAN_BOUNDS 7 7 - CFLAGS_UBSAN += $(call cc-option, -fsanitize=bounds) 7 7 + ifdef CONFIG_CC_IS_CLANG 8 8 + CFLAGS_UBSAN += -fsanitize=array-bounds 9 9 + else 10 10 + CFLAGS_UBSAN += $(call cc-option, -fsanitize=bounds) 11 11 + endif 12 12 + endif 13 13 + 14 14 + ifdef CONFIG_UBSAN_LOCAL_BOUNDS 15 15 + CFLAGS_UBSAN += -fsanitize=local-bounds 8 16 endif 9 17 10 18 ifdef CONFIG_UBSAN_MISC

+178 -60

scripts/checkpatch.pl

reviewed

··· 65 65 # git output parsing needs US English output, so first set backtick child process LANGUAGE 66 66 my $git_command ='export LANGUAGE=en_US.UTF-8; git'; 67 67 my $tabsize = 8; 68 68 + my ${CONFIG_} = "CONFIG_"; 68 69 69 70 sub help { 70 71 my ($exitcode) = @_; ··· 128 127 --typedefsfile Read additional types from this file 129 128 --color[=WHEN] Use colors 'always', 'never', or only when output 130 129 is a terminal ('auto'). Default is 'auto'. 130 130 + --kconfig-prefix=WORD use WORD as a prefix for Kconfig symbols (default 131 131 + ${CONFIG_}) 131 132 -h, --help, --version display this help and exit 132 133 133 134 When FILE is - read standard input. ··· 238 235 'color=s' => \$color, 239 236 'no-color' => \$color, #keep old behaviors of -nocolor 240 237 'nocolor' => \$color, #keep old behaviors of -nocolor 238 238 + 'kconfig-prefix=s' => \${CONFIG_}, 241 239 'h|help' => \$help, 242 240 'version' => \$help 243 241 ) or help(1); ··· 974 970 } 975 971 } 976 972 973 973 + sub git_is_single_file { 974 974 + my ($filename) = @_; 975 975 + 976 976 + return 0 if ((which("git") eq "") || !(-e "$gitroot")); 977 977 + 978 978 + my $output = `${git_command} ls-files -- $filename 2>/dev/null`; 979 979 + my $count = $output =~ tr/\n//; 980 980 + return $count eq 1 && $output =~ m{^${filename}$}; 981 981 + } 982 982 + 977 983 sub git_commit_info { 978 984 my ($commit, $id, $desc) = @_; 979 985 ··· 1057 1043 $allow_c99_comments = !defined $ignore_type{"C99_COMMENT_TOLERANCE"}; 1058 1044 for my $filename (@ARGV) { 1059 1045 my $FILE; 1046 1046 + my $is_git_file = git_is_single_file($filename); 1047 1047 + my $oldfile = $file; 1048 1048 + $file = 1 if ($is_git_file); 1060 1049 if ($git) { 1061 1050 open($FILE, '-|', "git format-patch -M --stdout -1 $filename") || 1062 1051 die "$P: $filename: git format-patch failed - $!\n"; ··· 1104 1087 @modifierListFile = (); 1105 1088 @typeListFile = (); 1106 1089 build_types(); 1090 1090 + $file = $oldfile if ($is_git_file); 1107 1091 } 1108 1092 1109 1093 if (!$quiet) { ··· 1181 1163 } 1182 1164 } 1183 1165 1166 1166 + $comment = trim($comment); 1184 1167 $name = trim($name); 1185 1168 $name =~ s/^\"|\"$//g; 1186 1186 - $name =~ s/(\s*\([^\)]+\))\s*//; 1187 1187 - if (defined($1)) { 1169 1169 + if ($name =~ s/(\s*\([^\)]+\))\s*//) { 1188 1170 $name_comment = trim($1); 1189 1171 } 1190 1172 $address = trim($address); ··· 1199 1181 } 1200 1182 1201 1183 sub format_email { 1202 1202 - my ($name, $address) = @_; 1184 1184 + my ($name, $name_comment, $address, $comment) = @_; 1203 1185 1204 1186 my $formatted_email; 1205 1187 1188 1188 + $name_comment = trim($name_comment); 1189 1189 + $comment = trim($comment); 1206 1190 $name = trim($name); 1207 1191 $name =~ s/^\"|\"$//g; 1208 1192 $address = trim($address); ··· 1217 1197 if ("$name" eq "") { 1218 1198 $formatted_email = "$address"; 1219 1199 } else { 1220 1220 - $formatted_email = "$name <$address>"; 1200 1200 + $formatted_email = "$name$name_comment <$address>"; 1221 1201 } 1222 1222 - 1202 1202 + $formatted_email .= "$comment"; 1223 1203 return $formatted_email; 1224 1204 } 1225 1205 ··· 1227 1207 my ($email) = @_; 1228 1208 1229 1209 my ($email_name, $name_comment, $email_address, $comment) = parse_email($email); 1230 1230 - return format_email($email_name, $email_address); 1210 1210 + return format_email($email_name, $name_comment, $email_address, $comment); 1231 1211 } 1232 1212 1233 1213 sub same_email_addresses { 1234 1234 - my ($email1, $email2) = @_; 1214 1214 + my ($email1, $email2, $match_comment) = @_; 1235 1215 1236 1216 my ($email1_name, $name1_comment, $email1_address, $comment1) = parse_email($email1); 1237 1217 my ($email2_name, $name2_comment, $email2_address, $comment2) = parse_email($email2); 1238 1218 1219 1219 + if ($match_comment != 1) { 1220 1220 + return $email1_name eq $email2_name && 1221 1221 + $email1_address eq $email2_address; 1222 1222 + } 1239 1223 return $email1_name eq $email2_name && 1240 1240 - $email1_address eq $email2_address; 1224 1224 + $email1_address eq $email2_address && 1225 1225 + $name1_comment eq $name2_comment && 1226 1226 + $comment1 eq $comment2; 1241 1227 } 1242 1228 1243 1229 sub which { ··· 2373 2347 my $signoff = 0; 2374 2348 my $author = ''; 2375 2349 my $authorsignoff = 0; 2350 2350 + my $author_sob = ''; 2376 2351 my $is_patch = 0; 2377 2352 my $is_binding_patch = -1; 2378 2353 my $in_header_lines = $file ? 0 : 1; ··· 2688 2661 # Check the patch for a From: 2689 2662 if (decode("MIME-Header", $line) =~ /^From:\s*(.*)/) { 2690 2663 $author = $1; 2664 2664 + my $curline = $linenr; 2665 2665 + while(defined($rawlines[$curline]) && ($rawlines[$curline++] =~ /^[ \t]\s*(.*)/)) { 2666 2666 + $author .= $1; 2667 2667 + } 2691 2668 $author = encode("utf8", $author) if ($line =~ /=\?utf-8\?/i); 2692 2669 $author =~ s/"//g; 2693 2670 $author = reformat_email($author); ··· 2701 2670 if ($line =~ /^\s*signed-off-by:\s*(.*)/i) { 2702 2671 $signoff++; 2703 2672 $in_commit_log = 0; 2704 2704 - if ($author ne '') { 2705 2705 - if (same_email_addresses($1, $author)) { 2673 2673 + if ($author ne '' && $authorsignoff != 1) { 2674 2674 + if (same_email_addresses($1, $author, 1)) { 2706 2675 $authorsignoff = 1; 2676 2676 + } else { 2677 2677 + my $ctx = $1; 2678 2678 + my ($email_name, $email_comment, $email_address, $comment1) = parse_email($ctx); 2679 2679 + my ($author_name, $author_comment, $author_address, $comment2) = parse_email($author); 2680 2680 + 2681 2681 + if ($email_address eq $author_address && $email_name eq $author_name) { 2682 2682 + $author_sob = $ctx; 2683 2683 + $authorsignoff = 2; 2684 2684 + } elsif ($email_address eq $author_address) { 2685 2685 + $author_sob = $ctx; 2686 2686 + $authorsignoff = 3; 2687 2687 + } elsif ($email_name eq $author_name) { 2688 2688 + $author_sob = $ctx; 2689 2689 + $authorsignoff = 4; 2690 2690 + 2691 2691 + my $address1 = $email_address; 2692 2692 + my $address2 = $author_address; 2693 2693 + 2694 2694 + if ($address1 =~ /(\S+)\+\S+(\@.*)/) { 2695 2695 + $address1 = "$1$2"; 2696 2696 + } 2697 2697 + if ($address2 =~ /(\S+)\+\S+(\@.*)/) { 2698 2698 + $address2 = "$1$2"; 2699 2699 + } 2700 2700 + if ($address1 eq $address2) { 2701 2701 + $authorsignoff = 5; 2702 2702 + } 2703 2703 + } 2707 2704 } 2708 2705 } 2709 2706 } ··· 2788 2729 } 2789 2730 2790 2731 my ($email_name, $name_comment, $email_address, $comment) = parse_email($email); 2791 2791 - my $suggested_email = format_email(($email_name, $email_address)); 2732 2732 + my $suggested_email = format_email(($email_name, $name_comment, $email_address, $comment)); 2792 2733 if ($suggested_email eq "") { 2793 2734 ERROR("BAD_SIGN_OFF", 2794 2735 "Unrecognized email address: '$email'\n" . $herecurr); ··· 2798 2739 $dequoted =~ s/" </ </; 2799 2740 # Don't force email to have quotes 2800 2741 # Allow just an angle bracketed address 2801 2801 - if (!same_email_addresses($email, $suggested_email)) { 2742 2742 + if (!same_email_addresses($email, $suggested_email, 0)) { 2802 2743 WARN("BAD_SIGN_OFF", 2803 2803 - "email address '$email' might be better as '$suggested_email$comment'\n" . $herecurr); 2744 2744 + "email address '$email' might be better as '$suggested_email'\n" . $herecurr); 2804 2745 } 2805 2746 } 2806 2747 ··· 3046 2987 } 3047 2988 } 3048 2989 2990 2990 + # check for repeated words separated by a single space 2991 2991 + if ($rawline =~ /^\+/ || $in_commit_log) { 2992 2992 + while ($rawline =~ /\b($word_pattern) (?=($word_pattern))/g) { 2993 2993 + 2994 2994 + my $first = $1; 2995 2995 + my $second = $2; 2996 2996 + 2997 2997 + if ($first =~ /(?:struct|union|enum)/) { 2998 2998 + pos($rawline) += length($first) + length($second) + 1; 2999 2999 + next; 3000 3000 + } 3001 3001 + 3002 3002 + next if ($first ne $second); 3003 3003 + next if ($first eq 'long'); 3004 3004 + 3005 3005 + if (WARN("REPEATED_WORD", 3006 3006 + "Possible repeated word: '$first'\n" . $herecurr) && 3007 3007 + $fix) { 3008 3008 + $fixed[$fixlinenr] =~ s/\b$first $second\b/$first/; 3009 3009 + } 3010 3010 + } 3011 3011 + 3012 3012 + # if it's a repeated word on consecutive lines in a comment block 3013 3013 + if ($prevline =~ /$;+\s*$/ && 3014 3014 + $prevrawline =~ /($word_pattern)\s*$/) { 3015 3015 + my $last_word = $1; 3016 3016 + if ($rawline =~ /^\+\s*\*\s*$last_word /) { 3017 3017 + if (WARN("REPEATED_WORD", 3018 3018 + "Possible repeated word: '$last_word'\n" . $hereprev) && 3019 3019 + $fix) { 3020 3020 + $fixed[$fixlinenr] =~ s/(\+\s*\*\s*)$last_word /$1/; 3021 3021 + } 3022 3022 + } 3023 3023 + } 3024 3024 + } 3025 3025 + 3049 3026 # ignore non-hunk lines and lines being removed 3050 3027 next if (!$hunk_line || $line =~ /^-/); 3051 3028 ··· 3308 3213 } 3309 3214 } 3310 3215 3216 3216 + # check for embedded filenames 3217 3217 + if ($rawline =~ /^\+.*\Q$realfile\E/) { 3218 3218 + WARN("EMBEDDED_FILENAME", 3219 3219 + "It's generally not useful to have the filename in the file\n" . $herecurr); 3220 3220 + } 3221 3221 + 3311 3222 # check we are in a valid source file if not then ignore this hunk 3312 3223 next if ($realfile !~ /\.(h|c|s|S|sh|dtsi|dts)$/); 3313 3224 ··· 3411 3310 } 3412 3311 } 3413 3312 3414 3414 - # check for repeated words separated by a single space 3415 3415 - if ($rawline =~ /^\+/) { 3416 3416 - while ($rawline =~ /\b($word_pattern) (?=($word_pattern))/g) { 3417 3417 - 3418 3418 - my $first = $1; 3419 3419 - my $second = $2; 3420 3420 - 3421 3421 - if ($first =~ /(?:struct|union|enum)/) { 3422 3422 - pos($rawline) += length($first) + length($second) + 1; 3423 3423 - next; 3424 3424 - } 3425 3425 - 3426 3426 - next if ($first ne $second); 3427 3427 - next if ($first eq 'long'); 3428 3428 - 3429 3429 - if (WARN("REPEATED_WORD", 3430 3430 - "Possible repeated word: '$first'\n" . $herecurr) && 3431 3431 - $fix) { 3432 3432 - $fixed[$fixlinenr] =~ s/\b$first $second\b/$first/; 3433 3433 - } 3434 3434 - } 3435 3435 - 3436 3436 - # if it's a repeated word on consecutive lines in a comment block 3437 3437 - if ($prevline =~ /$;+\s*$/ && 3438 3438 - $prevrawline =~ /($word_pattern)\s*$/) { 3439 3439 - my $last_word = $1; 3440 3440 - if ($rawline =~ /^\+\s*\*\s*$last_word /) { 3441 3441 - if (WARN("REPEATED_WORD", 3442 3442 - "Possible repeated word: '$last_word'\n" . $hereprev) && 3443 3443 - $fix) { 3444 3444 - $fixed[$fixlinenr] =~ s/(\+\s*\*\s*)$last_word /$1/; 3445 3445 - } 3446 3446 - } 3447 3447 - } 3448 3448 - } 3449 3449 - 3450 3313 # check for space before tabs. 3451 3314 if ($rawline =~ /^\+/ && $rawline =~ / \t/) { 3452 3315 my $herevet = "$here\n" . cat_vet($rawline) . "\n"; ··· 3501 3436 if ($realfile =~ m@^(drivers/net/|net/)@ && 3502 3437 $prevrawline =~ /^\+[ \t]*\/\*[ \t]*$/ && 3503 3438 $rawline =~ /^\+[ \t]*\*/ && 3504 3504 - $realline > 2) { 3439 3439 + $realline > 3) { # Do not warn about the initial copyright comment block after SPDX-License-Identifier 3505 3440 WARN("NETWORKING_BLOCK_COMMENT_STYLE", 3506 3441 "networking block comments don't use an empty /* line, use /* Comment...\n" . $hereprev); 3507 3442 } ··· 3960 3895 #ignore lines not being added 3961 3896 next if ($line =~ /^[^\+]/); 3962 3897 3898 3898 + # check for self assignments used to avoid compiler warnings 3899 3899 + # e.g.: int foo = foo, *bar = NULL; 3900 3900 + # struct foo bar = *(&(bar)); 3901 3901 + if ($line =~ /^\+\s*(?:$Declare)?([A-Za-z_][A-Za-z\d_]*)\s*=/) { 3902 3902 + my $var = $1; 3903 3903 + if ($line =~ /^\+\s*(?:$Declare)?$var\s*=\s*(?:$var|\*\s*\(?\s*&\s*\(?\s*$var\s*\)?\s*\)?)\s*[;,]/) { 3904 3904 + WARN("SELF_ASSIGNMENT", 3905 3905 + "Do not use self-assignments to avoid compiler warnings\n" . $herecurr); 3906 3906 + } 3907 3907 + } 3908 3908 + 3963 3909 # check for dereferences that span multiple lines 3964 3910 if ($prevline =~ /^\+.*$Lval\s*(?:\.|->)\s*$/ && 3965 3911 $line =~ /^\+\s*(?!\#\s*(?!define\s+|if))\s*$Lval/) { ··· 4344 4268 $level = "dbg" if ($level eq "debug"); 4345 4269 WARN("PREFER_DEV_LEVEL", 4346 4270 "Prefer dev_$level(... to dev_printk(KERN_$orig, ...\n" . $herecurr); 4271 4271 + } 4272 4272 + 4273 4273 + # trace_printk should not be used in production code. 4274 4274 + if ($line =~ /\b(trace_printk|trace_puts|ftrace_vprintk)\s*\(/) { 4275 4275 + WARN("TRACE_PRINTK", 4276 4276 + "Do not use $1() in production code (this can be ignored if built only with a debug config option)\n" . $herecurr); 4347 4277 } 4348 4278 4349 4279 # ENOSYS means "bad syscall nr" and nothing else. This will have a small ··· 5018 4936 } 5019 4937 } 5020 4938 4939 4939 + # check if a statement with a comma should be two statements like: 4940 4940 + # foo = bar(), /* comma should be semicolon */ 4941 4941 + # bar = baz(); 4942 4942 + if (defined($stat) && 4943 4943 + $stat =~ /^\+\s*(?:$Lval\s*$Assignment\s*)?$FuncArg\s*,\s*(?:$Lval\s*$Assignment\s*)?$FuncArg\s*;\s*$/) { 4944 4944 + my $cnt = statement_rawlines($stat); 4945 4945 + my $herectx = get_stat_here($linenr, $cnt, $here); 4946 4946 + WARN("SUSPECT_COMMA_SEMICOLON", 4947 4947 + "Possible comma where semicolon could be used\n" . $herectx); 4948 4948 + } 4949 4949 + 5021 4950 # return is not a function 5022 4951 if (defined($stat) && $stat =~ /^.\s*return(\s*)\(/s) { 5023 4952 my $spacing = $1; ··· 5388 5295 $dstat =~ s/\s*$//s; 5389 5296 5390 5297 # Flatten any parentheses and braces 5391 5391 - while ($dstat =~ s/\([^\(\)]*\)/1/ || 5392 5392 - $dstat =~ s/\{[^\{\}]*\}/1/ || 5393 5393 - $dstat =~ s/.\[[^\[\]]*\]/1/) 5298 5298 + while ($dstat =~ s/\([^\(\)]*\)/1u/ || 5299 5299 + $dstat =~ s/\{[^\{\}]*\}/1u/ || 5300 5300 + $dstat =~ s/.\[[^\[\]]*\]/1u/) 5394 5301 { 5395 5302 } 5396 5303 ··· 5431 5338 $dstat !~ /^\.$Ident\s*=/ && # .foo = 5432 5339 $dstat !~ /^(?:\#\s*$Ident|\#\s*$Constant)\s*$/ && # stringification #foo 5433 5340 $dstat !~ /^do\s*$Constant\s*while\s*$Constant;?$/ && # do {...} while (...); // do {...} while (...) 5341 5341 + $dstat !~ /^while\s*$Constant\s*$Constant\s*$/ && # while (...) {...} 5434 5342 $dstat !~ /^for\s*$Constant$/ && # for (...) 5435 5343 $dstat !~ /^for\s*$Constant\s+(?:$Ident|-?$Constant)$/ && # for (...) bar() 5436 5344 $dstat !~ /^do\s*{/ && # do {... ··· 6618 6524 } 6619 6525 6620 6526 # check for IS_ENABLED() without CONFIG_<FOO> ($rawline for comments too) 6621 6621 - if ($rawline =~ /\bIS_ENABLED\s*\(\s*(\w+)\s*\)/ && $1 !~ /^CONFIG_/) { 6527 6527 + if ($rawline =~ /\bIS_ENABLED\s*\(\s*(\w+)\s*\)/ && $1 !~ /^${CONFIG_}/) { 6622 6528 WARN("IS_ENABLED_CONFIG", 6623 6623 - "IS_ENABLED($1) is normally used as IS_ENABLED(CONFIG_$1)\n" . $herecurr); 6529 6529 + "IS_ENABLED($1) is normally used as IS_ENABLED(${CONFIG_}$1)\n" . $herecurr); 6624 6530 } 6625 6531 6626 6532 # check for #if defined CONFIG_<FOO> || defined CONFIG_<FOO>_MODULE 6627 6627 - if ($line =~ /^\+\s*#\s*if\s+defined(?:\s*\(?\s*|\s+)(CONFIG_[A-Z_]+)\s*\)?\s*\|\|\s*defined(?:\s*\(?\s*|\s+)\1_MODULE\s*\)?\s*$/) { 6533 6533 + if ($line =~ /^\+\s*#\s*if\s+defined(?:\s*\(?\s*|\s+)(${CONFIG_}[A-Z_]+)\s*\)?\s*\|\|\s*defined(?:\s*\(?\s*|\s+)\1_MODULE\s*\)?\s*$/) { 6628 6534 my $config = $1; 6629 6535 if (WARN("PREFER_IS_ENABLED", 6630 6630 - "Prefer IS_ENABLED(<FOO>) to CONFIG_<FOO> || CONFIG_<FOO>_MODULE\n" . $herecurr) && 6536 6536 + "Prefer IS_ENABLED(<FOO>) to ${CONFIG_}<FOO> || ${CONFIG_}<FOO>_MODULE\n" . $herecurr) && 6631 6537 $fix) { 6632 6538 $fixed[$fixlinenr] = "\+#if IS_ENABLED($config)"; 6633 6539 } ··· 6980 6886 if ($signoff == 0) { 6981 6887 ERROR("MISSING_SIGN_OFF", 6982 6888 "Missing Signed-off-by: line(s)\n"); 6983 6983 - } elsif (!$authorsignoff) { 6984 6984 - WARN("NO_AUTHOR_SIGN_OFF", 6985 6985 - "Missing Signed-off-by: line by nominal patch author '$author'\n"); 6889 6889 + } elsif ($authorsignoff != 1) { 6890 6890 + # authorsignoff values: 6891 6891 + # 0 -> missing sign off 6892 6892 + # 1 -> sign off identical 6893 6893 + # 2 -> names and addresses match, comments mismatch 6894 6894 + # 3 -> addresses match, names different 6895 6895 + # 4 -> names match, addresses different 6896 6896 + # 5 -> names match, addresses excluding subaddress details (refer RFC 5233) match 6897 6897 + 6898 6898 + my $sob_msg = "'From: $author' != 'Signed-off-by: $author_sob'"; 6899 6899 + 6900 6900 + if ($authorsignoff == 0) { 6901 6901 + ERROR("NO_AUTHOR_SIGN_OFF", 6902 6902 + "Missing Signed-off-by: line by nominal patch author '$author'\n"); 6903 6903 + } elsif ($authorsignoff == 2) { 6904 6904 + CHK("FROM_SIGN_OFF_MISMATCH", 6905 6905 + "From:/Signed-off-by: email comments mismatch: $sob_msg\n"); 6906 6906 + } elsif ($authorsignoff == 3) { 6907 6907 + WARN("FROM_SIGN_OFF_MISMATCH", 6908 6908 + "From:/Signed-off-by: email name mismatch: $sob_msg\n"); 6909 6909 + } elsif ($authorsignoff == 4) { 6910 6910 + WARN("FROM_SIGN_OFF_MISMATCH", 6911 6911 + "From:/Signed-off-by: email address mismatch: $sob_msg\n"); 6912 6912 + } elsif ($authorsignoff == 5) { 6913 6913 + WARN("FROM_SIGN_OFF_MISMATCH", 6914 6914 + "From:/Signed-off-by: email subaddress mismatch: $sob_msg\n"); 6915 6915 + } 6986 6916 } 6987 6917 } 6988 6918

+3

scripts/const_structs.checkpatch

reviewed

··· 39 39 nvkm_device_chip 40 40 of_device_id 41 41 pci_raw_ops 42 42 + phy_ops 43 43 + pinctrl_ops 44 44 + pinmux_ops 42 45 pipe_buf_operations 43 46 platform_hibernation_ops 44 47 platform_suspend_ops

+7 -8

scripts/gdb/linux/proc.py

reviewed

··· 167 167 if not namespace: 168 168 raise gdb.GdbError("No namespace for current process") 169 169 170 170 + gdb.write("{:^18} {:^15} {:>9} {} {} options\n".format( 171 171 + "mount", "super_block", "devname", "pathname", "fstype")) 172 172 + 170 173 for vfs in lists.list_for_each_entry(namespace['list'], 171 174 mount_ptr_type, "mnt_list"): 172 175 devname = vfs['mnt_devname'].string() ··· 193 190 m_flags = int(vfs['mnt']['mnt_flags']) 194 191 rd = "ro" if (s_flags & constants.LX_SB_RDONLY) else "rw" 195 192 196 196 - gdb.write( 197 197 - "{} {} {} {}{}{} 0 0\n" 198 198 - .format(devname, 199 199 - pathname, 200 200 - fstype, 201 201 - rd, 202 202 - info_opts(FS_INFO, s_flags), 203 203 - info_opts(MNT_INFO, m_flags))) 193 193 + gdb.write("{} {} {} {} {} {}{}{} 0 0\n".format( 194 194 + vfs.format_string(), superblock.format_string(), devname, 195 195 + pathname, fstype, rd, info_opts(FS_INFO, s_flags), 196 196 + info_opts(MNT_INFO, m_flags))) 204 197 205 198 206 199 LxMounts()

+5 -4

scripts/gdb/linux/tasks.py

reviewed

··· 73 73 super(LxPs, self).__init__("lx-ps", gdb.COMMAND_DATA) 74 74 75 75 def invoke(self, arg, from_tty): 76 76 + gdb.write("{:>10} {:>12} {:>7}\n".format("TASK", "PID", "COMM")) 76 77 for task in task_lists(): 77 77 - gdb.write("{address} {pid} {comm}\n".format( 78 78 - address=task, 79 79 - pid=task["pid"], 80 80 - comm=task["comm"].string())) 78 78 + gdb.write("{} {:^5} {}\n".format( 79 79 + task.format_string().split()[0], 80 80 + task["pid"].format_string(), 81 81 + task["comm"].string())) 81 82 82 83 83 84 LxPs()

+7 -2

scripts/get_maintainer.pl

reviewed

··· 541 541 die "$P: file '${file}' not found\n"; 542 542 } 543 543 } 544 544 + if ($from_filename && (vcs_exists() && !vcs_file_exists($file))) { 545 545 + warn "$P: file '$file' not found in version control $!\n"; 546 546 + } 544 547 if ($from_filename || ($file ne "&STDIN" && vcs_file_exists($file))) { 545 548 $file =~ s/^\Q${cur_path}\E//; #strip any absolute path 546 549 $file =~ s/^\Q${lk_path}\E//; #or the path to the lk tree ··· 957 954 958 955 foreach my $file (@files) { 959 956 if ($email && 960 960 - ($email_git || ($email_git_fallback && 961 961 - !$exact_pattern_match_hash{$file}))) { 957 957 + ($email_git || 958 958 + ($email_git_fallback && 959 959 + $file !~ /MAINTAINERS$/ && 960 960 + !$exact_pattern_match_hash{$file}))) { 962 961 vcs_file_signoffs($file); 963 962 } 964 963 if ($email && $email_git_blame) {

+1

tools/testing/selftests/exec/.gitignore

reviewed

··· 7 7 execveat.path.ephemeral 8 8 execveat.ephemeral 9 9 execveat.denatured 10 10 + /load_address_* 10 11 /recursion-depth 11 12 xxxxxxxx* 12 13 pipe

+7 -2

tools/testing/selftests/exec/Makefile

reviewed

··· 4 4 CFLAGS += -D_GNU_SOURCE 5 5 6 6 TEST_PROGS := binfmt_script non-regular 7 7 - TEST_GEN_PROGS := execveat 7 7 + TEST_GEN_PROGS := execveat load_address_4096 load_address_2097152 load_address_16777216 8 8 TEST_GEN_FILES := execveat.symlink execveat.denatured script subdir pipe 9 9 # Makefile is a run-time dependency, since it's accessed by the execveat test 10 10 TEST_FILES := Makefile ··· 27 27 $(OUTPUT)/execveat.denatured: $(OUTPUT)/execveat 28 28 cp $< $@ 29 29 chmod -x $@ 30 30 - 30 30 + $(OUTPUT)/load_address_4096: load_address.c 31 31 + $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=0x1000 -pie $< -o $@ 32 32 + $(OUTPUT)/load_address_2097152: load_address.c 33 33 + $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=0x200000 -pie $< -o $@ 34 34 + $(OUTPUT)/load_address_16777216: load_address.c 35 35 + $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=0x1000000 -pie $< -o $@

+68

tools/testing/selftests/exec/load_address.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + #ifndef _GNU_SOURCE 3 3 + #define _GNU_SOURCE 4 4 + #endif 5 5 + #include <link.h> 6 6 + #include <stdio.h> 7 7 + #include <stdlib.h> 8 8 + 9 9 + struct Statistics { 10 10 + unsigned long long load_address; 11 11 + unsigned long long alignment; 12 12 + }; 13 13 + 14 14 + int ExtractStatistics(struct dl_phdr_info *info, size_t size, void *data) 15 15 + { 16 16 + struct Statistics *stats = (struct Statistics *) data; 17 17 + int i; 18 18 + 19 19 + if (info->dlpi_name != NULL && info->dlpi_name[0] != '\0') { 20 20 + // Ignore headers from other than the executable. 21 21 + return 2; 22 22 + } 23 23 + 24 24 + stats->load_address = (unsigned long long) info->dlpi_addr; 25 25 + stats->alignment = 0; 26 26 + 27 27 + for (i = 0; i < info->dlpi_phnum; i++) { 28 28 + if (info->dlpi_phdr[i].p_type != PT_LOAD) 29 29 + continue; 30 30 + 31 31 + if (info->dlpi_phdr[i].p_align > stats->alignment) 32 32 + stats->alignment = info->dlpi_phdr[i].p_align; 33 33 + } 34 34 + 35 35 + return 1; // Terminate dl_iterate_phdr. 36 36 + } 37 37 + 38 38 + int main(int argc, char **argv) 39 39 + { 40 40 + struct Statistics extracted; 41 41 + unsigned long long misalign; 42 42 + int ret; 43 43 + 44 44 + ret = dl_iterate_phdr(ExtractStatistics, &extracted); 45 45 + if (ret != 1) { 46 46 + fprintf(stderr, "FAILED\n"); 47 47 + return 1; 48 48 + } 49 49 + 50 50 + if (extracted.alignment == 0) { 51 51 + fprintf(stderr, "No alignment found\n"); 52 52 + return 1; 53 53 + } else if (extracted.alignment & (extracted.alignment - 1)) { 54 54 + fprintf(stderr, "Alignment is not a power of 2\n"); 55 55 + return 1; 56 56 + } 57 57 + 58 58 + misalign = extracted.load_address & (extracted.alignment - 1); 59 59 + if (misalign) { 60 60 + printf("alignment = %llu, load_address = %llu\n", 61 61 + extracted.alignment, extracted.load_address); 62 62 + fprintf(stderr, "FAILED\n"); 63 63 + return 1; 64 64 + } 65 65 + 66 66 + fprintf(stderr, "PASS\n"); 67 67 + return 0; 68 68 + }