mm: move mmap/vma locking logic into specific files

+5 -228

include/linux/mm.h

··· 671 671 static inline void vma_numab_state_free(struct vm_area_struct *vma) {} 672 672 #endif /* CONFIG_NUMA_BALANCING */ 673 673 674 + /* 675 + * These must be here rather than mmap_lock.h as dependent on vm_fault type, 676 + * declared in this header. 677 + */ 674 678 #ifdef CONFIG_PER_VMA_LOCK 675 - static inline void vma_lock_init(struct vm_area_struct *vma, bool reset_refcnt) 676 - { 677 - #ifdef CONFIG_DEBUG_LOCK_ALLOC 678 - static struct lock_class_key lockdep_key; 679 - 680 - lockdep_init_map(&vma->vmlock_dep_map, "vm_lock", &lockdep_key, 0); 681 - #endif 682 - if (reset_refcnt) 683 - refcount_set(&vma->vm_refcnt, 0); 684 - vma->vm_lock_seq = UINT_MAX; 685 - } 686 - 687 - static inline bool is_vma_writer_only(int refcnt) 688 - { 689 - /* 690 - * With a writer and no readers, refcnt is VMA_LOCK_OFFSET if the vma 691 - * is detached and (VMA_LOCK_OFFSET + 1) if it is attached. Waiting on 692 - * a detached vma happens only in vma_mark_detached() and is a rare 693 - * case, therefore most of the time there will be no unnecessary wakeup. 694 - */ 695 - return refcnt & VMA_LOCK_OFFSET && refcnt <= VMA_LOCK_OFFSET + 1; 696 - } 697 - 698 - static inline void vma_refcount_put(struct vm_area_struct *vma) 699 - { 700 - /* Use a copy of vm_mm in case vma is freed after we drop vm_refcnt */ 701 - struct mm_struct *mm = vma->vm_mm; 702 - int oldcnt; 703 - 704 - rwsem_release(&vma->vmlock_dep_map, _RET_IP_); 705 - if (!__refcount_dec_and_test(&vma->vm_refcnt, &oldcnt)) { 706 - 707 - if (is_vma_writer_only(oldcnt - 1)) 708 - rcuwait_wake_up(&mm->vma_writer_wait); 709 - } 710 - } 711 - 712 - /* 713 - * Try to read-lock a vma. The function is allowed to occasionally yield false 714 - * locked result to avoid performance overhead, in which case we fall back to 715 - * using mmap_lock. The function should never yield false unlocked result. 716 - * False locked result is possible if mm_lock_seq overflows or if vma gets 717 - * reused and attached to a different mm before we lock it. 718 - * Returns the vma on success, NULL on failure to lock and EAGAIN if vma got 719 - * detached. 720 - */ 721 - static inline struct vm_area_struct *vma_start_read(struct mm_struct *mm, 722 - struct vm_area_struct *vma) 723 - { 724 - int oldcnt; 725 - 726 - /* 727 - * Check before locking. A race might cause false locked result. 728 - * We can use READ_ONCE() for the mm_lock_seq here, and don't need 729 - * ACQUIRE semantics, because this is just a lockless check whose result 730 - * we don't rely on for anything - the mm_lock_seq read against which we 731 - * need ordering is below. 732 - */ 733 - if (READ_ONCE(vma->vm_lock_seq) == READ_ONCE(mm->mm_lock_seq.sequence)) 734 - return NULL; 735 - 736 - /* 737 - * If VMA_LOCK_OFFSET is set, __refcount_inc_not_zero_limited_acquire() 738 - * will fail because VMA_REF_LIMIT is less than VMA_LOCK_OFFSET. 739 - * Acquire fence is required here to avoid reordering against later 740 - * vm_lock_seq check and checks inside lock_vma_under_rcu(). 741 - */ 742 - if (unlikely(!__refcount_inc_not_zero_limited_acquire(&vma->vm_refcnt, &oldcnt, 743 - VMA_REF_LIMIT))) { 744 - /* return EAGAIN if vma got detached from under us */ 745 - return oldcnt ? NULL : ERR_PTR(-EAGAIN); 746 - } 747 - 748 - rwsem_acquire_read(&vma->vmlock_dep_map, 0, 1, _RET_IP_); 749 - /* 750 - * Overflow of vm_lock_seq/mm_lock_seq might produce false locked result. 751 - * False unlocked result is impossible because we modify and check 752 - * vma->vm_lock_seq under vma->vm_refcnt protection and mm->mm_lock_seq 753 - * modification invalidates all existing locks. 754 - * 755 - * We must use ACQUIRE semantics for the mm_lock_seq so that if we are 756 - * racing with vma_end_write_all(), we only start reading from the VMA 757 - * after it has been unlocked. 758 - * This pairs with RELEASE semantics in vma_end_write_all(). 759 - */ 760 - if (unlikely(vma->vm_lock_seq == raw_read_seqcount(&mm->mm_lock_seq))) { 761 - vma_refcount_put(vma); 762 - return NULL; 763 - } 764 - 765 - return vma; 766 - } 767 - 768 - /* 769 - * Use only while holding mmap read lock which guarantees that locking will not 770 - * fail (nobody can concurrently write-lock the vma). vma_start_read() should 771 - * not be used in such cases because it might fail due to mm_lock_seq overflow. 772 - * This functionality is used to obtain vma read lock and drop the mmap read lock. 773 - */ 774 - static inline bool vma_start_read_locked_nested(struct vm_area_struct *vma, int subclass) 775 - { 776 - int oldcnt; 777 - 778 - mmap_assert_locked(vma->vm_mm); 779 - if (unlikely(!__refcount_inc_not_zero_limited_acquire(&vma->vm_refcnt, &oldcnt, 780 - VMA_REF_LIMIT))) 781 - return false; 782 - 783 - rwsem_acquire_read(&vma->vmlock_dep_map, 0, 1, _RET_IP_); 784 - return true; 785 - } 786 - 787 - /* 788 - * Use only while holding mmap read lock which guarantees that locking will not 789 - * fail (nobody can concurrently write-lock the vma). vma_start_read() should 790 - * not be used in such cases because it might fail due to mm_lock_seq overflow. 791 - * This functionality is used to obtain vma read lock and drop the mmap read lock. 792 - */ 793 - static inline bool vma_start_read_locked(struct vm_area_struct *vma) 794 - { 795 - return vma_start_read_locked_nested(vma, 0); 796 - } 797 - 798 - static inline void vma_end_read(struct vm_area_struct *vma) 799 - { 800 - vma_refcount_put(vma); 801 - } 802 - 803 - /* WARNING! Can only be used if mmap_lock is expected to be write-locked */ 804 - static bool __is_vma_write_locked(struct vm_area_struct *vma, unsigned int *mm_lock_seq) 805 - { 806 - mmap_assert_write_locked(vma->vm_mm); 807 - 808 - /* 809 - * current task is holding mmap_write_lock, both vma->vm_lock_seq and 810 - * mm->mm_lock_seq can't be concurrently modified. 811 - */ 812 - *mm_lock_seq = vma->vm_mm->mm_lock_seq.sequence; 813 - return (vma->vm_lock_seq == *mm_lock_seq); 814 - } 815 - 816 - void __vma_start_write(struct vm_area_struct *vma, unsigned int mm_lock_seq); 817 - 818 - /* 819 - * Begin writing to a VMA. 820 - * Exclude concurrent readers under the per-VMA lock until the currently 821 - * write-locked mmap_lock is dropped or downgraded. 822 - */ 823 - static inline void vma_start_write(struct vm_area_struct *vma) 824 - { 825 - unsigned int mm_lock_seq; 826 - 827 - if (__is_vma_write_locked(vma, &mm_lock_seq)) 828 - return; 829 - 830 - __vma_start_write(vma, mm_lock_seq); 831 - } 832 - 833 - static inline void vma_assert_write_locked(struct vm_area_struct *vma) 834 - { 835 - unsigned int mm_lock_seq; 836 - 837 - VM_BUG_ON_VMA(!__is_vma_write_locked(vma, &mm_lock_seq), vma); 838 - } 839 - 840 - static inline void vma_assert_locked(struct vm_area_struct *vma) 841 - { 842 - unsigned int mm_lock_seq; 843 - 844 - VM_BUG_ON_VMA(refcount_read(&vma->vm_refcnt) <= 1 && 845 - !__is_vma_write_locked(vma, &mm_lock_seq), vma); 846 - } 847 - 848 - /* 849 - * WARNING: to avoid racing with vma_mark_attached()/vma_mark_detached(), these 850 - * assertions should be made either under mmap_write_lock or when the object 851 - * has been isolated under mmap_write_lock, ensuring no competing writers. 852 - */ 853 - static inline void vma_assert_attached(struct vm_area_struct *vma) 854 - { 855 - WARN_ON_ONCE(!refcount_read(&vma->vm_refcnt)); 856 - } 857 - 858 - static inline void vma_assert_detached(struct vm_area_struct *vma) 859 - { 860 - WARN_ON_ONCE(refcount_read(&vma->vm_refcnt)); 861 - } 862 - 863 - static inline void vma_mark_attached(struct vm_area_struct *vma) 864 - { 865 - vma_assert_write_locked(vma); 866 - vma_assert_detached(vma); 867 - refcount_set_release(&vma->vm_refcnt, 1); 868 - } 869 - 870 - void vma_mark_detached(struct vm_area_struct *vma); 871 - 872 679 static inline void release_fault_lock(struct vm_fault *vmf) 873 680 { 874 681 if (vmf->flags & FAULT_FLAG_VMA_LOCK) ··· 691 884 else 692 885 mmap_assert_locked(vmf->vma->vm_mm); 693 886 } 694 - 695 - struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, 696 - unsigned long address); 697 - 698 - #else /* CONFIG_PER_VMA_LOCK */ 699 - 700 - static inline void vma_lock_init(struct vm_area_struct *vma, bool reset_refcnt) {} 701 - static inline struct vm_area_struct *vma_start_read(struct mm_struct *mm, 702 - struct vm_area_struct *vma) 703 - { return NULL; } 704 - static inline void vma_end_read(struct vm_area_struct *vma) {} 705 - static inline void vma_start_write(struct vm_area_struct *vma) {} 706 - static inline void vma_assert_write_locked(struct vm_area_struct *vma) 707 - { mmap_assert_write_locked(vma->vm_mm); } 708 - static inline void vma_assert_attached(struct vm_area_struct *vma) {} 709 - static inline void vma_assert_detached(struct vm_area_struct *vma) {} 710 - static inline void vma_mark_attached(struct vm_area_struct *vma) {} 711 - static inline void vma_mark_detached(struct vm_area_struct *vma) {} 712 - 713 - static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, 714 - unsigned long address) 715 - { 716 - return NULL; 717 - } 718 - 719 - static inline void vma_assert_locked(struct vm_area_struct *vma) 720 - { 721 - mmap_assert_locked(vma->vm_mm); 722 - } 723 - 887 + #else 724 888 static inline void release_fault_lock(struct vm_fault *vmf) 725 889 { 726 890 mmap_read_unlock(vmf->vma->vm_mm); ··· 701 923 { 702 924 mmap_assert_locked(vmf->vma->vm_mm); 703 925 } 704 - 705 926 #endif /* CONFIG_PER_VMA_LOCK */ 706 927 707 928 extern const struct vm_operations_struct vma_dummy_vm_ops;

+227

include/linux/mmap_lock.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 1 2 #ifndef _LINUX_MMAP_LOCK_H 2 3 #define _LINUX_MMAP_LOCK_H 4 + 5 + /* Avoid a dependency loop by declaring here. */ 6 + extern int rcuwait_wake_up(struct rcuwait *w); 3 7 4 8 #include <linux/lockdep.h> 5 9 #include <linux/mm_types.h> ··· 108 104 return read_seqcount_retry(&mm->mm_lock_seq, seq); 109 105 } 110 106 107 + static inline void vma_lock_init(struct vm_area_struct *vma, bool reset_refcnt) 108 + { 109 + #ifdef CONFIG_DEBUG_LOCK_ALLOC 110 + static struct lock_class_key lockdep_key; 111 + 112 + lockdep_init_map(&vma->vmlock_dep_map, "vm_lock", &lockdep_key, 0); 113 + #endif 114 + if (reset_refcnt) 115 + refcount_set(&vma->vm_refcnt, 0); 116 + vma->vm_lock_seq = UINT_MAX; 117 + } 118 + 119 + static inline bool is_vma_writer_only(int refcnt) 120 + { 121 + /* 122 + * With a writer and no readers, refcnt is VMA_LOCK_OFFSET if the vma 123 + * is detached and (VMA_LOCK_OFFSET + 1) if it is attached. Waiting on 124 + * a detached vma happens only in vma_mark_detached() and is a rare 125 + * case, therefore most of the time there will be no unnecessary wakeup. 126 + */ 127 + return refcnt & VMA_LOCK_OFFSET && refcnt <= VMA_LOCK_OFFSET + 1; 128 + } 129 + 130 + static inline void vma_refcount_put(struct vm_area_struct *vma) 131 + { 132 + /* Use a copy of vm_mm in case vma is freed after we drop vm_refcnt */ 133 + struct mm_struct *mm = vma->vm_mm; 134 + int oldcnt; 135 + 136 + rwsem_release(&vma->vmlock_dep_map, _RET_IP_); 137 + if (!__refcount_dec_and_test(&vma->vm_refcnt, &oldcnt)) { 138 + 139 + if (is_vma_writer_only(oldcnt - 1)) 140 + rcuwait_wake_up(&mm->vma_writer_wait); 141 + } 142 + } 143 + 144 + /* 145 + * Try to read-lock a vma. The function is allowed to occasionally yield false 146 + * locked result to avoid performance overhead, in which case we fall back to 147 + * using mmap_lock. The function should never yield false unlocked result. 148 + * False locked result is possible if mm_lock_seq overflows or if vma gets 149 + * reused and attached to a different mm before we lock it. 150 + * Returns the vma on success, NULL on failure to lock and EAGAIN if vma got 151 + * detached. 152 + */ 153 + static inline struct vm_area_struct *vma_start_read(struct mm_struct *mm, 154 + struct vm_area_struct *vma) 155 + { 156 + int oldcnt; 157 + 158 + /* 159 + * Check before locking. A race might cause false locked result. 160 + * We can use READ_ONCE() for the mm_lock_seq here, and don't need 161 + * ACQUIRE semantics, because this is just a lockless check whose result 162 + * we don't rely on for anything - the mm_lock_seq read against which we 163 + * need ordering is below. 164 + */ 165 + if (READ_ONCE(vma->vm_lock_seq) == READ_ONCE(mm->mm_lock_seq.sequence)) 166 + return NULL; 167 + 168 + /* 169 + * If VMA_LOCK_OFFSET is set, __refcount_inc_not_zero_limited_acquire() 170 + * will fail because VMA_REF_LIMIT is less than VMA_LOCK_OFFSET. 171 + * Acquire fence is required here to avoid reordering against later 172 + * vm_lock_seq check and checks inside lock_vma_under_rcu(). 173 + */ 174 + if (unlikely(!__refcount_inc_not_zero_limited_acquire(&vma->vm_refcnt, &oldcnt, 175 + VMA_REF_LIMIT))) { 176 + /* return EAGAIN if vma got detached from under us */ 177 + return oldcnt ? NULL : ERR_PTR(-EAGAIN); 178 + } 179 + 180 + rwsem_acquire_read(&vma->vmlock_dep_map, 0, 1, _RET_IP_); 181 + /* 182 + * Overflow of vm_lock_seq/mm_lock_seq might produce false locked result. 183 + * False unlocked result is impossible because we modify and check 184 + * vma->vm_lock_seq under vma->vm_refcnt protection and mm->mm_lock_seq 185 + * modification invalidates all existing locks. 186 + * 187 + * We must use ACQUIRE semantics for the mm_lock_seq so that if we are 188 + * racing with vma_end_write_all(), we only start reading from the VMA 189 + * after it has been unlocked. 190 + * This pairs with RELEASE semantics in vma_end_write_all(). 191 + */ 192 + if (unlikely(vma->vm_lock_seq == raw_read_seqcount(&mm->mm_lock_seq))) { 193 + vma_refcount_put(vma); 194 + return NULL; 195 + } 196 + 197 + return vma; 198 + } 199 + 200 + /* 201 + * Use only while holding mmap read lock which guarantees that locking will not 202 + * fail (nobody can concurrently write-lock the vma). vma_start_read() should 203 + * not be used in such cases because it might fail due to mm_lock_seq overflow. 204 + * This functionality is used to obtain vma read lock and drop the mmap read lock. 205 + */ 206 + static inline bool vma_start_read_locked_nested(struct vm_area_struct *vma, int subclass) 207 + { 208 + int oldcnt; 209 + 210 + mmap_assert_locked(vma->vm_mm); 211 + if (unlikely(!__refcount_inc_not_zero_limited_acquire(&vma->vm_refcnt, &oldcnt, 212 + VMA_REF_LIMIT))) 213 + return false; 214 + 215 + rwsem_acquire_read(&vma->vmlock_dep_map, 0, 1, _RET_IP_); 216 + return true; 217 + } 218 + 219 + /* 220 + * Use only while holding mmap read lock which guarantees that locking will not 221 + * fail (nobody can concurrently write-lock the vma). vma_start_read() should 222 + * not be used in such cases because it might fail due to mm_lock_seq overflow. 223 + * This functionality is used to obtain vma read lock and drop the mmap read lock. 224 + */ 225 + static inline bool vma_start_read_locked(struct vm_area_struct *vma) 226 + { 227 + return vma_start_read_locked_nested(vma, 0); 228 + } 229 + 230 + static inline void vma_end_read(struct vm_area_struct *vma) 231 + { 232 + vma_refcount_put(vma); 233 + } 234 + 235 + /* WARNING! Can only be used if mmap_lock is expected to be write-locked */ 236 + static bool __is_vma_write_locked(struct vm_area_struct *vma, unsigned int *mm_lock_seq) 237 + { 238 + mmap_assert_write_locked(vma->vm_mm); 239 + 240 + /* 241 + * current task is holding mmap_write_lock, both vma->vm_lock_seq and 242 + * mm->mm_lock_seq can't be concurrently modified. 243 + */ 244 + *mm_lock_seq = vma->vm_mm->mm_lock_seq.sequence; 245 + return (vma->vm_lock_seq == *mm_lock_seq); 246 + } 247 + 248 + void __vma_start_write(struct vm_area_struct *vma, unsigned int mm_lock_seq); 249 + 250 + /* 251 + * Begin writing to a VMA. 252 + * Exclude concurrent readers under the per-VMA lock until the currently 253 + * write-locked mmap_lock is dropped or downgraded. 254 + */ 255 + static inline void vma_start_write(struct vm_area_struct *vma) 256 + { 257 + unsigned int mm_lock_seq; 258 + 259 + if (__is_vma_write_locked(vma, &mm_lock_seq)) 260 + return; 261 + 262 + __vma_start_write(vma, mm_lock_seq); 263 + } 264 + 265 + static inline void vma_assert_write_locked(struct vm_area_struct *vma) 266 + { 267 + unsigned int mm_lock_seq; 268 + 269 + VM_BUG_ON_VMA(!__is_vma_write_locked(vma, &mm_lock_seq), vma); 270 + } 271 + 272 + static inline void vma_assert_locked(struct vm_area_struct *vma) 273 + { 274 + unsigned int mm_lock_seq; 275 + 276 + VM_BUG_ON_VMA(refcount_read(&vma->vm_refcnt) <= 1 && 277 + !__is_vma_write_locked(vma, &mm_lock_seq), vma); 278 + } 279 + 280 + /* 281 + * WARNING: to avoid racing with vma_mark_attached()/vma_mark_detached(), these 282 + * assertions should be made either under mmap_write_lock or when the object 283 + * has been isolated under mmap_write_lock, ensuring no competing writers. 284 + */ 285 + static inline void vma_assert_attached(struct vm_area_struct *vma) 286 + { 287 + WARN_ON_ONCE(!refcount_read(&vma->vm_refcnt)); 288 + } 289 + 290 + static inline void vma_assert_detached(struct vm_area_struct *vma) 291 + { 292 + WARN_ON_ONCE(refcount_read(&vma->vm_refcnt)); 293 + } 294 + 295 + static inline void vma_mark_attached(struct vm_area_struct *vma) 296 + { 297 + vma_assert_write_locked(vma); 298 + vma_assert_detached(vma); 299 + refcount_set_release(&vma->vm_refcnt, 1); 300 + } 301 + 302 + void vma_mark_detached(struct vm_area_struct *vma); 303 + 304 + struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, 305 + unsigned long address); 306 + 111 307 #else /* CONFIG_PER_VMA_LOCK */ 112 308 113 309 static inline void mm_lock_seqcount_init(struct mm_struct *mm) {} ··· 322 118 static inline bool mmap_lock_speculate_retry(struct mm_struct *mm, unsigned int seq) 323 119 { 324 120 return true; 121 + } 122 + static inline void vma_lock_init(struct vm_area_struct *vma, bool reset_refcnt) {} 123 + static inline struct vm_area_struct *vma_start_read(struct mm_struct *mm, 124 + struct vm_area_struct *vma) 125 + { return NULL; } 126 + static inline void vma_end_read(struct vm_area_struct *vma) {} 127 + static inline void vma_start_write(struct vm_area_struct *vma) {} 128 + static inline void vma_assert_write_locked(struct vm_area_struct *vma) 129 + { mmap_assert_write_locked(vma->vm_mm); } 130 + static inline void vma_assert_attached(struct vm_area_struct *vma) {} 131 + static inline void vma_assert_detached(struct vm_area_struct *vma) {} 132 + static inline void vma_mark_attached(struct vm_area_struct *vma) {} 133 + static inline void vma_mark_detached(struct vm_area_struct *vma) {} 134 + 135 + static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, 136 + unsigned long address) 137 + { 138 + return NULL; 139 + } 140 + 141 + static inline void vma_assert_locked(struct vm_area_struct *vma) 142 + { 143 + mmap_assert_locked(vma->vm_mm); 325 144 } 326 145 327 146 #endif /* CONFIG_PER_VMA_LOCK */

-252

mm/memory.c

··· 6378 6378 } 6379 6379 EXPORT_SYMBOL_GPL(handle_mm_fault); 6380 6380 6381 - #ifdef CONFIG_LOCK_MM_AND_FIND_VMA 6382 - #include <linux/extable.h> 6383 - 6384 - static inline bool get_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs) 6385 - { 6386 - if (likely(mmap_read_trylock(mm))) 6387 - return true; 6388 - 6389 - if (regs && !user_mode(regs)) { 6390 - unsigned long ip = exception_ip(regs); 6391 - if (!search_exception_tables(ip)) 6392 - return false; 6393 - } 6394 - 6395 - return !mmap_read_lock_killable(mm); 6396 - } 6397 - 6398 - static inline bool mmap_upgrade_trylock(struct mm_struct *mm) 6399 - { 6400 - /* 6401 - * We don't have this operation yet. 6402 - * 6403 - * It should be easy enough to do: it's basically a 6404 - * atomic_long_try_cmpxchg_acquire() 6405 - * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but 6406 - * it also needs the proper lockdep magic etc. 6407 - */ 6408 - return false; 6409 - } 6410 - 6411 - static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs) 6412 - { 6413 - mmap_read_unlock(mm); 6414 - if (regs && !user_mode(regs)) { 6415 - unsigned long ip = exception_ip(regs); 6416 - if (!search_exception_tables(ip)) 6417 - return false; 6418 - } 6419 - return !mmap_write_lock_killable(mm); 6420 - } 6421 - 6422 - /* 6423 - * Helper for page fault handling. 6424 - * 6425 - * This is kind of equivalent to "mmap_read_lock()" followed 6426 - * by "find_extend_vma()", except it's a lot more careful about 6427 - * the locking (and will drop the lock on failure). 6428 - * 6429 - * For example, if we have a kernel bug that causes a page 6430 - * fault, we don't want to just use mmap_read_lock() to get 6431 - * the mm lock, because that would deadlock if the bug were 6432 - * to happen while we're holding the mm lock for writing. 6433 - * 6434 - * So this checks the exception tables on kernel faults in 6435 - * order to only do this all for instructions that are actually 6436 - * expected to fault. 6437 - * 6438 - * We can also actually take the mm lock for writing if we 6439 - * need to extend the vma, which helps the VM layer a lot. 6440 - */ 6441 - struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, 6442 - unsigned long addr, struct pt_regs *regs) 6443 - { 6444 - struct vm_area_struct *vma; 6445 - 6446 - if (!get_mmap_lock_carefully(mm, regs)) 6447 - return NULL; 6448 - 6449 - vma = find_vma(mm, addr); 6450 - if (likely(vma && (vma->vm_start <= addr))) 6451 - return vma; 6452 - 6453 - /* 6454 - * Well, dang. We might still be successful, but only 6455 - * if we can extend a vma to do so. 6456 - */ 6457 - if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) { 6458 - mmap_read_unlock(mm); 6459 - return NULL; 6460 - } 6461 - 6462 - /* 6463 - * We can try to upgrade the mmap lock atomically, 6464 - * in which case we can continue to use the vma 6465 - * we already looked up. 6466 - * 6467 - * Otherwise we'll have to drop the mmap lock and 6468 - * re-take it, and also look up the vma again, 6469 - * re-checking it. 6470 - */ 6471 - if (!mmap_upgrade_trylock(mm)) { 6472 - if (!upgrade_mmap_lock_carefully(mm, regs)) 6473 - return NULL; 6474 - 6475 - vma = find_vma(mm, addr); 6476 - if (!vma) 6477 - goto fail; 6478 - if (vma->vm_start <= addr) 6479 - goto success; 6480 - if (!(vma->vm_flags & VM_GROWSDOWN)) 6481 - goto fail; 6482 - } 6483 - 6484 - if (expand_stack_locked(vma, addr)) 6485 - goto fail; 6486 - 6487 - success: 6488 - mmap_write_downgrade(mm); 6489 - return vma; 6490 - 6491 - fail: 6492 - mmap_write_unlock(mm); 6493 - return NULL; 6494 - } 6495 - #endif 6496 - 6497 - #ifdef CONFIG_PER_VMA_LOCK 6498 - static inline bool __vma_enter_locked(struct vm_area_struct *vma, bool detaching) 6499 - { 6500 - unsigned int tgt_refcnt = VMA_LOCK_OFFSET; 6501 - 6502 - /* Additional refcnt if the vma is attached. */ 6503 - if (!detaching) 6504 - tgt_refcnt++; 6505 - 6506 - /* 6507 - * If vma is detached then only vma_mark_attached() can raise the 6508 - * vm_refcnt. mmap_write_lock prevents racing with vma_mark_attached(). 6509 - */ 6510 - if (!refcount_add_not_zero(VMA_LOCK_OFFSET, &vma->vm_refcnt)) 6511 - return false; 6512 - 6513 - rwsem_acquire(&vma->vmlock_dep_map, 0, 0, _RET_IP_); 6514 - rcuwait_wait_event(&vma->vm_mm->vma_writer_wait, 6515 - refcount_read(&vma->vm_refcnt) == tgt_refcnt, 6516 - TASK_UNINTERRUPTIBLE); 6517 - lock_acquired(&vma->vmlock_dep_map, _RET_IP_); 6518 - 6519 - return true; 6520 - } 6521 - 6522 - static inline void __vma_exit_locked(struct vm_area_struct *vma, bool *detached) 6523 - { 6524 - *detached = refcount_sub_and_test(VMA_LOCK_OFFSET, &vma->vm_refcnt); 6525 - rwsem_release(&vma->vmlock_dep_map, _RET_IP_); 6526 - } 6527 - 6528 - void __vma_start_write(struct vm_area_struct *vma, unsigned int mm_lock_seq) 6529 - { 6530 - bool locked; 6531 - 6532 - /* 6533 - * __vma_enter_locked() returns false immediately if the vma is not 6534 - * attached, otherwise it waits until refcnt is indicating that vma 6535 - * is attached with no readers. 6536 - */ 6537 - locked = __vma_enter_locked(vma, false); 6538 - 6539 - /* 6540 - * We should use WRITE_ONCE() here because we can have concurrent reads 6541 - * from the early lockless pessimistic check in vma_start_read(). 6542 - * We don't really care about the correctness of that early check, but 6543 - * we should use WRITE_ONCE() for cleanliness and to keep KCSAN happy. 6544 - */ 6545 - WRITE_ONCE(vma->vm_lock_seq, mm_lock_seq); 6546 - 6547 - if (locked) { 6548 - bool detached; 6549 - 6550 - __vma_exit_locked(vma, &detached); 6551 - WARN_ON_ONCE(detached); /* vma should remain attached */ 6552 - } 6553 - } 6554 - EXPORT_SYMBOL_GPL(__vma_start_write); 6555 - 6556 - void vma_mark_detached(struct vm_area_struct *vma) 6557 - { 6558 - vma_assert_write_locked(vma); 6559 - vma_assert_attached(vma); 6560 - 6561 - /* 6562 - * We are the only writer, so no need to use vma_refcount_put(). 6563 - * The condition below is unlikely because the vma has been already 6564 - * write-locked and readers can increment vm_refcnt only temporarily 6565 - * before they check vm_lock_seq, realize the vma is locked and drop 6566 - * back the vm_refcnt. That is a narrow window for observing a raised 6567 - * vm_refcnt. 6568 - */ 6569 - if (unlikely(!refcount_dec_and_test(&vma->vm_refcnt))) { 6570 - /* Wait until vma is detached with no readers. */ 6571 - if (__vma_enter_locked(vma, true)) { 6572 - bool detached; 6573 - 6574 - __vma_exit_locked(vma, &detached); 6575 - WARN_ON_ONCE(!detached); 6576 - } 6577 - } 6578 - } 6579 - 6580 - /* 6581 - * Lookup and lock a VMA under RCU protection. Returned VMA is guaranteed to be 6582 - * stable and not isolated. If the VMA is not found or is being modified the 6583 - * function returns NULL. 6584 - */ 6585 - struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, 6586 - unsigned long address) 6587 - { 6588 - MA_STATE(mas, &mm->mm_mt, address, address); 6589 - struct vm_area_struct *vma; 6590 - 6591 - rcu_read_lock(); 6592 - retry: 6593 - vma = mas_walk(&mas); 6594 - if (!vma) 6595 - goto inval; 6596 - 6597 - vma = vma_start_read(mm, vma); 6598 - if (IS_ERR_OR_NULL(vma)) { 6599 - /* Check if the VMA got isolated after we found it */ 6600 - if (PTR_ERR(vma) == -EAGAIN) { 6601 - count_vm_vma_lock_event(VMA_LOCK_MISS); 6602 - /* The area was replaced with another one */ 6603 - goto retry; 6604 - } 6605 - 6606 - /* Failed to lock the VMA */ 6607 - goto inval; 6608 - } 6609 - /* 6610 - * At this point, we have a stable reference to a VMA: The VMA is 6611 - * locked and we know it hasn't already been isolated. 6612 - * From here on, we can access the VMA without worrying about which 6613 - * fields are accessible for RCU readers. 6614 - */ 6615 - 6616 - /* Check if the vma we locked is the right one. */ 6617 - if (unlikely(vma->vm_mm != mm || 6618 - address < vma->vm_start || address >= vma->vm_end)) 6619 - goto inval_end_read; 6620 - 6621 - rcu_read_unlock(); 6622 - return vma; 6623 - 6624 - inval_end_read: 6625 - vma_end_read(vma); 6626 - inval: 6627 - rcu_read_unlock(); 6628 - count_vm_vma_lock_event(VMA_LOCK_ABORT); 6629 - return NULL; 6630 - } 6631 - #endif /* CONFIG_PER_VMA_LOCK */ 6632 - 6633 6381 #ifndef __PAGETABLE_P4D_FOLDED 6634 6382 /* 6635 6383 * Allocate p4d page table.

+273

mm/mmap_lock.c

··· 42 42 } 43 43 EXPORT_SYMBOL(__mmap_lock_do_trace_released); 44 44 #endif /* CONFIG_TRACING */ 45 + 46 + #ifdef CONFIG_MMU 47 + #ifdef CONFIG_PER_VMA_LOCK 48 + static inline bool __vma_enter_locked(struct vm_area_struct *vma, bool detaching) 49 + { 50 + unsigned int tgt_refcnt = VMA_LOCK_OFFSET; 51 + 52 + /* Additional refcnt if the vma is attached. */ 53 + if (!detaching) 54 + tgt_refcnt++; 55 + 56 + /* 57 + * If vma is detached then only vma_mark_attached() can raise the 58 + * vm_refcnt. mmap_write_lock prevents racing with vma_mark_attached(). 59 + */ 60 + if (!refcount_add_not_zero(VMA_LOCK_OFFSET, &vma->vm_refcnt)) 61 + return false; 62 + 63 + rwsem_acquire(&vma->vmlock_dep_map, 0, 0, _RET_IP_); 64 + rcuwait_wait_event(&vma->vm_mm->vma_writer_wait, 65 + refcount_read(&vma->vm_refcnt) == tgt_refcnt, 66 + TASK_UNINTERRUPTIBLE); 67 + lock_acquired(&vma->vmlock_dep_map, _RET_IP_); 68 + 69 + return true; 70 + } 71 + 72 + static inline void __vma_exit_locked(struct vm_area_struct *vma, bool *detached) 73 + { 74 + *detached = refcount_sub_and_test(VMA_LOCK_OFFSET, &vma->vm_refcnt); 75 + rwsem_release(&vma->vmlock_dep_map, _RET_IP_); 76 + } 77 + 78 + void __vma_start_write(struct vm_area_struct *vma, unsigned int mm_lock_seq) 79 + { 80 + bool locked; 81 + 82 + /* 83 + * __vma_enter_locked() returns false immediately if the vma is not 84 + * attached, otherwise it waits until refcnt is indicating that vma 85 + * is attached with no readers. 86 + */ 87 + locked = __vma_enter_locked(vma, false); 88 + 89 + /* 90 + * We should use WRITE_ONCE() here because we can have concurrent reads 91 + * from the early lockless pessimistic check in vma_start_read(). 92 + * We don't really care about the correctness of that early check, but 93 + * we should use WRITE_ONCE() for cleanliness and to keep KCSAN happy. 94 + */ 95 + WRITE_ONCE(vma->vm_lock_seq, mm_lock_seq); 96 + 97 + if (locked) { 98 + bool detached; 99 + 100 + __vma_exit_locked(vma, &detached); 101 + WARN_ON_ONCE(detached); /* vma should remain attached */ 102 + } 103 + } 104 + EXPORT_SYMBOL_GPL(__vma_start_write); 105 + 106 + void vma_mark_detached(struct vm_area_struct *vma) 107 + { 108 + vma_assert_write_locked(vma); 109 + vma_assert_attached(vma); 110 + 111 + /* 112 + * We are the only writer, so no need to use vma_refcount_put(). 113 + * The condition below is unlikely because the vma has been already 114 + * write-locked and readers can increment vm_refcnt only temporarily 115 + * before they check vm_lock_seq, realize the vma is locked and drop 116 + * back the vm_refcnt. That is a narrow window for observing a raised 117 + * vm_refcnt. 118 + */ 119 + if (unlikely(!refcount_dec_and_test(&vma->vm_refcnt))) { 120 + /* Wait until vma is detached with no readers. */ 121 + if (__vma_enter_locked(vma, true)) { 122 + bool detached; 123 + 124 + __vma_exit_locked(vma, &detached); 125 + WARN_ON_ONCE(!detached); 126 + } 127 + } 128 + } 129 + 130 + /* 131 + * Lookup and lock a VMA under RCU protection. Returned VMA is guaranteed to be 132 + * stable and not isolated. If the VMA is not found or is being modified the 133 + * function returns NULL. 134 + */ 135 + struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, 136 + unsigned long address) 137 + { 138 + MA_STATE(mas, &mm->mm_mt, address, address); 139 + struct vm_area_struct *vma; 140 + 141 + rcu_read_lock(); 142 + retry: 143 + vma = mas_walk(&mas); 144 + if (!vma) 145 + goto inval; 146 + 147 + vma = vma_start_read(mm, vma); 148 + if (IS_ERR_OR_NULL(vma)) { 149 + /* Check if the VMA got isolated after we found it */ 150 + if (PTR_ERR(vma) == -EAGAIN) { 151 + count_vm_vma_lock_event(VMA_LOCK_MISS); 152 + /* The area was replaced with another one */ 153 + goto retry; 154 + } 155 + 156 + /* Failed to lock the VMA */ 157 + goto inval; 158 + } 159 + /* 160 + * At this point, we have a stable reference to a VMA: The VMA is 161 + * locked and we know it hasn't already been isolated. 162 + * From here on, we can access the VMA without worrying about which 163 + * fields are accessible for RCU readers. 164 + */ 165 + 166 + /* Check if the vma we locked is the right one. */ 167 + if (unlikely(vma->vm_mm != mm || 168 + address < vma->vm_start || address >= vma->vm_end)) 169 + goto inval_end_read; 170 + 171 + rcu_read_unlock(); 172 + return vma; 173 + 174 + inval_end_read: 175 + vma_end_read(vma); 176 + inval: 177 + rcu_read_unlock(); 178 + count_vm_vma_lock_event(VMA_LOCK_ABORT); 179 + return NULL; 180 + } 181 + #endif /* CONFIG_PER_VMA_LOCK */ 182 + 183 + #ifdef CONFIG_LOCK_MM_AND_FIND_VMA 184 + #include <linux/extable.h> 185 + 186 + static inline bool get_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs) 187 + { 188 + if (likely(mmap_read_trylock(mm))) 189 + return true; 190 + 191 + if (regs && !user_mode(regs)) { 192 + unsigned long ip = exception_ip(regs); 193 + if (!search_exception_tables(ip)) 194 + return false; 195 + } 196 + 197 + return !mmap_read_lock_killable(mm); 198 + } 199 + 200 + static inline bool mmap_upgrade_trylock(struct mm_struct *mm) 201 + { 202 + /* 203 + * We don't have this operation yet. 204 + * 205 + * It should be easy enough to do: it's basically a 206 + * atomic_long_try_cmpxchg_acquire() 207 + * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but 208 + * it also needs the proper lockdep magic etc. 209 + */ 210 + return false; 211 + } 212 + 213 + static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs) 214 + { 215 + mmap_read_unlock(mm); 216 + if (regs && !user_mode(regs)) { 217 + unsigned long ip = exception_ip(regs); 218 + if (!search_exception_tables(ip)) 219 + return false; 220 + } 221 + return !mmap_write_lock_killable(mm); 222 + } 223 + 224 + /* 225 + * Helper for page fault handling. 226 + * 227 + * This is kind of equivalent to "mmap_read_lock()" followed 228 + * by "find_extend_vma()", except it's a lot more careful about 229 + * the locking (and will drop the lock on failure). 230 + * 231 + * For example, if we have a kernel bug that causes a page 232 + * fault, we don't want to just use mmap_read_lock() to get 233 + * the mm lock, because that would deadlock if the bug were 234 + * to happen while we're holding the mm lock for writing. 235 + * 236 + * So this checks the exception tables on kernel faults in 237 + * order to only do this all for instructions that are actually 238 + * expected to fault. 239 + * 240 + * We can also actually take the mm lock for writing if we 241 + * need to extend the vma, which helps the VM layer a lot. 242 + */ 243 + struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, 244 + unsigned long addr, struct pt_regs *regs) 245 + { 246 + struct vm_area_struct *vma; 247 + 248 + if (!get_mmap_lock_carefully(mm, regs)) 249 + return NULL; 250 + 251 + vma = find_vma(mm, addr); 252 + if (likely(vma && (vma->vm_start <= addr))) 253 + return vma; 254 + 255 + /* 256 + * Well, dang. We might still be successful, but only 257 + * if we can extend a vma to do so. 258 + */ 259 + if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) { 260 + mmap_read_unlock(mm); 261 + return NULL; 262 + } 263 + 264 + /* 265 + * We can try to upgrade the mmap lock atomically, 266 + * in which case we can continue to use the vma 267 + * we already looked up. 268 + * 269 + * Otherwise we'll have to drop the mmap lock and 270 + * re-take it, and also look up the vma again, 271 + * re-checking it. 272 + */ 273 + if (!mmap_upgrade_trylock(mm)) { 274 + if (!upgrade_mmap_lock_carefully(mm, regs)) 275 + return NULL; 276 + 277 + vma = find_vma(mm, addr); 278 + if (!vma) 279 + goto fail; 280 + if (vma->vm_start <= addr) 281 + goto success; 282 + if (!(vma->vm_flags & VM_GROWSDOWN)) 283 + goto fail; 284 + } 285 + 286 + if (expand_stack_locked(vma, addr)) 287 + goto fail; 288 + 289 + success: 290 + mmap_write_downgrade(mm); 291 + return vma; 292 + 293 + fail: 294 + mmap_write_unlock(mm); 295 + return NULL; 296 + } 297 + #endif /* CONFIG_LOCK_MM_AND_FIND_VMA */ 298 + 299 + #else /* CONFIG_MMU */ 300 + 301 + /* 302 + * At least xtensa ends up having protection faults even with no 303 + * MMU.. No stack expansion, at least. 304 + */ 305 + struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, 306 + unsigned long addr, struct pt_regs *regs) 307 + { 308 + struct vm_area_struct *vma; 309 + 310 + mmap_read_lock(mm); 311 + vma = vma_lookup(mm, addr); 312 + if (!vma) 313 + mmap_read_unlock(mm); 314 + return vma; 315 + } 316 + 317 + #endif /* CONFIG_MMU */

-16

mm/nommu.c

··· 627 627 EXPORT_SYMBOL(find_vma); 628 628 629 629 /* 630 - * At least xtensa ends up having protection faults even with no 631 - * MMU.. No stack expansion, at least. 632 - */ 633 - struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, 634 - unsigned long addr, struct pt_regs *regs) 635 - { 636 - struct vm_area_struct *vma; 637 - 638 - mmap_read_lock(mm); 639 - vma = vma_lookup(mm, addr); 640 - if (!vma) 641 - mmap_read_unlock(mm); 642 - return vma; 643 - } 644 - 645 - /* 646 630 * expand a stack to a given address 647 631 * - not supported under NOMMU conditions 648 632 */

Configure Feed

Configure Feed