locking/semaphore: Remove the list_head from struct semaphore

Instead of embedding a list_head in struct semaphore, store a pointer to
the first waiter. The list of waiters remains a doubly linked list so
we can efficiently add to the tail of the list and remove from the front
(or middle) of the list.

Some of the list manipulation becomes more complicated, but it's a
reasonable tradeoff on the slow paths to shrink data structures
which embed a semaphore.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260305195545.3707590-3-willy@infradead.org

authored by

Matthew Wilcox (Oracle) and committed by

Peter Zijlstra 3 months ago b9bdd4b6 1ea4b473

+33 -12

3 changed files

expand all

drivers

acpi

osl.c

include

linux

semaphore.h

kernel

locking

semaphore.c

+1 -1

drivers/acpi/osl.c

··· 1257 1257 1258 1258 ACPI_DEBUG_PRINT((ACPI_DB_MUTEX, "Deleting semaphore[%p].\n", handle)); 1259 1259 1260 - BUG_ON(!list_empty(&sem->wait_list)); 1260 + BUG_ON(sem->first_waiter); 1261 1261 kfree(sem); 1262 1262 sem = NULL; 1263 1263

+2 -2

include/linux/semaphore.h

··· 15 15 struct semaphore { 16 16 raw_spinlock_t lock; 17 17 unsigned int count; 18 - struct list_head wait_list; 18 + struct semaphore_waiter *first_waiter; 19 19 20 20 #ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER 21 21 unsigned long last_holder; ··· 33 33 { \ 34 34 .lock = __RAW_SPIN_LOCK_UNLOCKED((name).lock), \ 35 35 .count = n, \ 36 - .wait_list = LIST_HEAD_INIT((name).wait_list) \ 36 + .first_waiter = NULL \ 37 37 __LAST_HOLDER_SEMAPHORE_INITIALIZER \ 38 38 } 39 39

+30 -9

kernel/locking/semaphore.c

··· 21 21 * too. 22 22 * 23 23 * The ->count variable represents how many more tasks can acquire this 24 - * semaphore. If it's zero, there may be tasks waiting on the wait_list. 24 + * semaphore. If it's zero, there may be waiters. 25 25 */ 26 26 27 27 #include <linux/compiler.h> ··· 226 226 227 227 hung_task_sem_clear_if_holder(sem); 228 228 229 - if (likely(list_empty(&sem->wait_list))) 229 + if (likely(!sem->first_waiter)) 230 230 sem->count++; 231 231 else 232 232 __up(sem, &wake_q); ··· 244 244 bool up; 245 245 }; 246 246 247 + static inline 248 + void sem_del_waiter(struct semaphore *sem, struct semaphore_waiter *waiter) 249 + { 250 + if (list_empty(&waiter->list)) { 251 + sem->first_waiter = NULL; 252 + return; 253 + } 254 + 255 + if (sem->first_waiter == waiter) { 256 + sem->first_waiter = list_first_entry(&waiter->list, 257 + struct semaphore_waiter, list); 258 + } 259 + list_del(&waiter->list); 260 + } 261 + 247 262 /* 248 263 * Because this function is inlined, the 'state' parameter will be 249 264 * constant, and thus optimised away by the compiler. Likewise the ··· 267 252 static inline int __sched ___down_common(struct semaphore *sem, long state, 268 253 long timeout) 269 254 { 270 - struct semaphore_waiter waiter; 255 + struct semaphore_waiter waiter, *first; 271 256 272 - list_add_tail(&waiter.list, &sem->wait_list); 257 + first = sem->first_waiter; 258 + if (first) { 259 + list_add_tail(&waiter.list, &first->list); 260 + } else { 261 + INIT_LIST_HEAD(&waiter.list); 262 + sem->first_waiter = &waiter; 263 + } 273 264 waiter.task = current; 274 265 waiter.up = false; 275 266 ··· 295 274 } 296 275 297 276 timed_out: 298 - list_del(&waiter.list); 277 + sem_del_waiter(sem, &waiter); 299 278 return -ETIME; 300 279 301 280 interrupted: 302 - list_del(&waiter.list); 281 + sem_del_waiter(sem, &waiter); 303 282 return -EINTR; 304 283 } 305 284 ··· 342 321 static noinline void __sched __up(struct semaphore *sem, 343 322 struct wake_q_head *wake_q) 344 323 { 345 - struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list, 346 - struct semaphore_waiter, list); 347 - list_del(&waiter->list); 324 + struct semaphore_waiter *waiter = sem->first_waiter; 325 + 326 + sem_del_waiter(sem, waiter); 348 327 waiter->up = true; 349 328 wake_q_add(wake_q, waiter->task); 350 329 }

Configure Feed

Configure Feed