net: Use nested-BH locking for napi_alloc_cache.

napi_alloc_cache is a per-CPU variable and relies on disabled BH for its
locking. Without per-CPU locking in local_bh_disable() on PREEMPT_RT
this data structure requires explicit locking.

Add a local_lock_t to the data structure and use local_lock_nested_bh()
for locking. This change adds only lockdep coverage and does not alter
the functional behaviour for !PREEMPT_RT.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Link: https://patch.msgid.link/20240620132727.660738-5-bigeasy@linutronix.de
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

authored by

Sebastian Andrzej Siewior and committed by

Jakub Kicinski 2 years ago bdacf3e3 43d7ca29

+24 -5

1 changed file

expand all

net

core

skbuff.c

+24 -5

net/core/skbuff.c

··· 277 277 #endif 278 278 279 279 struct napi_alloc_cache { 280 + local_lock_t bh_lock; 280 281 struct page_frag_cache page; 281 282 struct page_frag_1k page_small; 282 283 unsigned int skb_count; ··· 285 284 }; 286 285 287 286 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 288 - static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); 287 + static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache) = { 288 + .bh_lock = INIT_LOCAL_LOCK(bh_lock), 289 + }; 289 290 290 291 /* Double check that napi_get_frags() allocates skbs with 291 292 * skb->head being backed by slab, not a page fragment. ··· 309 306 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 310 307 { 311 308 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 309 + void *data; 312 310 313 311 fragsz = SKB_DATA_ALIGN(fragsz); 314 312 315 - return __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, 313 + local_lock_nested_bh(&napi_alloc_cache.bh_lock); 314 + data = __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, 316 315 align_mask); 316 + local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 317 + return data; 318 + 317 319 } 318 320 EXPORT_SYMBOL(__napi_alloc_frag_align); 319 321 ··· 346 338 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 347 339 struct sk_buff *skb; 348 340 341 + local_lock_nested_bh(&napi_alloc_cache.bh_lock); 349 342 if (unlikely(!nc->skb_count)) { 350 343 nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, 351 344 GFP_ATOMIC, 352 345 NAPI_SKB_CACHE_BULK, 353 346 nc->skb_cache); 354 - if (unlikely(!nc->skb_count)) 347 + if (unlikely(!nc->skb_count)) { 348 + local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 355 349 return NULL; 350 + } 356 351 } 357 352 358 353 skb = nc->skb_cache[--nc->skb_count]; 354 + local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 359 355 kasan_mempool_unpoison_object(skb, kmem_cache_size(net_hotdata.skbuff_cache)); 360 356 361 357 return skb; ··· 752 740 pfmemalloc = nc->pfmemalloc; 753 741 } else { 754 742 local_bh_disable(); 743 + local_lock_nested_bh(&napi_alloc_cache.bh_lock); 744 + 755 745 nc = this_cpu_ptr(&napi_alloc_cache.page); 756 746 data = page_frag_alloc(nc, len, gfp_mask); 757 747 pfmemalloc = nc->pfmemalloc; 748 + 749 + local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 758 750 local_bh_enable(); 759 751 } 760 752 ··· 822 806 goto skb_success; 823 807 } 824 808 825 - nc = this_cpu_ptr(&napi_alloc_cache); 826 - 827 809 if (sk_memalloc_socks()) 828 810 gfp_mask |= __GFP_MEMALLOC; 829 811 812 + local_lock_nested_bh(&napi_alloc_cache.bh_lock); 813 + nc = this_cpu_ptr(&napi_alloc_cache); 830 814 if (NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) { 831 815 /* we are artificially inflating the allocation size, but 832 816 * that is not as bad as it may look like, as: ··· 848 832 data = page_frag_alloc(&nc->page, len, gfp_mask); 849 833 pfmemalloc = nc->page.pfmemalloc; 850 834 } 835 + local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 851 836 852 837 if (unlikely(!data)) 853 838 return NULL; ··· 1448 1431 if (!kasan_mempool_poison_object(skb)) 1449 1432 return; 1450 1433 1434 + local_lock_nested_bh(&napi_alloc_cache.bh_lock); 1451 1435 nc->skb_cache[nc->skb_count++] = skb; 1452 1436 1453 1437 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { ··· 1460 1442 nc->skb_cache + NAPI_SKB_CACHE_HALF); 1461 1443 nc->skb_count = NAPI_SKB_CACHE_HALF; 1462 1444 } 1445 + local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 1463 1446 } 1464 1447 1465 1448 void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason)

Configure Feed

Configure Feed