mm/vma: improve and document __is_vma_write_locked()

We don't actually need to return an output parameter providing mm sequence
number, rather we can separate that out into another function -
__vma_raw_mm_seqnum() - and have any callers which need to obtain that
invoke that instead.

The access to the raw sequence number requires that we hold the exclusive
mmap lock such that we know we can't race vma_end_write_all(), so move the
assert to __vma_raw_mm_seqnum() to make this requirement clear.

Also while we're here, convert all of the VM_BUG_ON_VMA()'s to
VM_WARN_ON_ONCE_VMA()'s in line with the convention that we do not invoke
oopses when we can avoid it.

[lorenzo.stoakes@oracle.com: minor tweaks, per Vlastimil]
Link: https://lkml.kernel.org/r/3fa89c13-232d-4eee-86cc-96caa75c2c67@lucifer.local
Link: https://lkml.kernel.org/r/ef6c415c2d2c03f529dca124ccaed66bc2f60edc.1769198904.git.lorenzo.stoakes@oracle.com
Signed-off-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Waiman Long <longman@redhat.com>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Lorenzo Stoakes and committed by

Andrew Morton 4 months ago 22f7639f e28e575a

+28 -25

2 changed files

expand all

include

linux

mmap_lock.h

mmap_lock.c

+25 -22

include/linux/mmap_lock.h

··· 258 258 vma_refcount_put(vma); 259 259 } 260 260 261 - /* WARNING! Can only be used if mmap_lock is expected to be write-locked */ 262 - static inline bool __is_vma_write_locked(struct vm_area_struct *vma, unsigned int *mm_lock_seq) 261 + static inline unsigned int __vma_raw_mm_seqnum(struct vm_area_struct *vma) 263 262 { 264 - mmap_assert_write_locked(vma->vm_mm); 263 + const struct mm_struct *mm = vma->vm_mm; 265 264 265 + /* We must hold an exclusive write lock for this access to be valid. */ 266 + mmap_assert_write_locked(vma->vm_mm); 267 + return mm->mm_lock_seq.sequence; 268 + } 269 + 270 + /* 271 + * Determine whether a VMA is write-locked. Must be invoked ONLY if the mmap 272 + * write lock is held. 273 + * 274 + * Returns true if write-locked, otherwise false. 275 + */ 276 + static inline bool __is_vma_write_locked(struct vm_area_struct *vma) 277 + { 266 278 /* 267 279 * current task is holding mmap_write_lock, both vma->vm_lock_seq and 268 280 * mm->mm_lock_seq can't be concurrently modified. 269 281 */ 270 - *mm_lock_seq = vma->vm_mm->mm_lock_seq.sequence; 271 - return (vma->vm_lock_seq == *mm_lock_seq); 282 + return vma->vm_lock_seq == __vma_raw_mm_seqnum(vma); 272 283 } 273 284 274 - int __vma_start_write(struct vm_area_struct *vma, unsigned int mm_lock_seq, 275 - int state); 285 + int __vma_start_write(struct vm_area_struct *vma, int state); 276 286 277 287 /* 278 288 * Begin writing to a VMA. ··· 291 281 */ 292 282 static inline void vma_start_write(struct vm_area_struct *vma) 293 283 { 294 - unsigned int mm_lock_seq; 295 - 296 - if (__is_vma_write_locked(vma, &mm_lock_seq)) 284 + if (__is_vma_write_locked(vma)) 297 285 return; 298 286 299 - __vma_start_write(vma, mm_lock_seq, TASK_UNINTERRUPTIBLE); 287 + __vma_start_write(vma, TASK_UNINTERRUPTIBLE); 300 288 } 301 289 302 290 /** ··· 313 305 static inline __must_check 314 306 int vma_start_write_killable(struct vm_area_struct *vma) 315 307 { 316 - unsigned int mm_lock_seq; 317 - 318 - if (__is_vma_write_locked(vma, &mm_lock_seq)) 308 + if (__is_vma_write_locked(vma)) 319 309 return 0; 320 - return __vma_start_write(vma, mm_lock_seq, TASK_KILLABLE); 310 + 311 + return __vma_start_write(vma, TASK_KILLABLE); 321 312 } 322 313 323 314 static inline void vma_assert_write_locked(struct vm_area_struct *vma) 324 315 { 325 - unsigned int mm_lock_seq; 326 - 327 - VM_BUG_ON_VMA(!__is_vma_write_locked(vma, &mm_lock_seq), vma); 316 + VM_WARN_ON_ONCE_VMA(!__is_vma_write_locked(vma), vma); 328 317 } 329 318 330 319 static inline void vma_assert_locked(struct vm_area_struct *vma) 331 320 { 332 - unsigned int mm_lock_seq; 333 - 334 321 /* 335 322 * See the comment describing the vm_area_struct->vm_refcnt field for 336 323 * details of possible refcnt values. 337 324 */ 338 - VM_BUG_ON_VMA(refcount_read(&vma->vm_refcnt) <= 1 && 339 - !__is_vma_write_locked(vma, &mm_lock_seq), vma); 325 + VM_WARN_ON_ONCE_VMA(refcount_read(&vma->vm_refcnt) <= 1 && 326 + !__is_vma_write_locked(vma), vma); 340 327 } 341 328 342 329 static inline bool vma_is_attached(struct vm_area_struct *vma)

+3 -3

mm/mmap_lock.c

··· 136 136 return 0; 137 137 } 138 138 139 - int __vma_start_write(struct vm_area_struct *vma, unsigned int mm_lock_seq, 140 - int state) 139 + int __vma_start_write(struct vm_area_struct *vma, int state) 141 140 { 142 - int err; 141 + const unsigned int mm_lock_seq = __vma_raw_mm_seqnum(vma); 143 142 struct vma_exclude_readers_state ves = { 144 143 .vma = vma, 145 144 .state = state, 146 145 }; 146 + int err; 147 147 148 148 err = __vma_start_exclude_readers(&ves); 149 149 if (err) {

Configure Feed

Configure Feed