tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

bpf: Select bpf_local_storage_map_bucket based on bpf_local_storage

A later bpf_local_storage refactor will acquire all locks before
performing any update. To simplified the number of locks needed to take
in bpf_local_storage_map_update(), determine the bucket based on the
local_storage an selem belongs to instead of the selem pointer.

Currently, when a new selem needs to be created to replace the old selem
in bpf_local_storage_map_update(), locks of both buckets need to be
acquired to prevent racing. This can be simplified if the two selem
belongs to the same bucket so that only one bucket needs to be locked.
Therefore, instead of hashing selem, hashing the local_storage pointer
the selem belongs.

Performance wise, this is slightly better as update now requires locking
one bucket. It should not change the level of contention on one bucket
as the pointers to local storages of selems in a map are just as unique
as pointers to selems.

Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Amery Hung <ameryhung@gmail.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://patch.msgid.link/20260205222916.1788211-2-ameryhung@gmail.com

authored by

Amery Hung and committed by
Martin KaFai Lau 0ccef707 b8c89f5c

+13 -7
+1
include/linux/bpf_local_storage.h
··· 179 179 void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now); 180 180 181 181 void bpf_selem_link_map(struct bpf_local_storage_map *smap, 182 + struct bpf_local_storage *local_storage, 182 183 struct bpf_local_storage_elem *selem); 183 184 184 185 struct bpf_local_storage_elem *
+11 -6
kernel/bpf/bpf_local_storage.c
··· 19 19 20 20 static struct bpf_local_storage_map_bucket * 21 21 select_bucket(struct bpf_local_storage_map *smap, 22 - struct bpf_local_storage_elem *selem) 22 + struct bpf_local_storage *local_storage) 23 23 { 24 - return &smap->buckets[hash_ptr(selem, smap->bucket_log)]; 24 + return &smap->buckets[hash_ptr(local_storage, smap->bucket_log)]; 25 25 } 26 26 27 27 static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size) ··· 349 349 350 350 static void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) 351 351 { 352 + struct bpf_local_storage *local_storage; 352 353 struct bpf_local_storage_map *smap; 353 354 struct bpf_local_storage_map_bucket *b; 354 355 unsigned long flags; ··· 358 357 /* selem has already be unlinked from smap */ 359 358 return; 360 359 360 + local_storage = rcu_dereference_check(selem->local_storage, 361 + bpf_rcu_lock_held()); 361 362 smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); 362 - b = select_bucket(smap, selem); 363 + b = select_bucket(smap, local_storage); 363 364 raw_spin_lock_irqsave(&b->lock, flags); 364 365 if (likely(selem_linked_to_map(selem))) 365 366 hlist_del_init_rcu(&selem->map_node); ··· 369 366 } 370 367 371 368 void bpf_selem_link_map(struct bpf_local_storage_map *smap, 369 + struct bpf_local_storage *local_storage, 372 370 struct bpf_local_storage_elem *selem) 373 371 { 374 - struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem); 372 + struct bpf_local_storage_map_bucket *b; 375 373 unsigned long flags; 376 374 375 + b = select_bucket(smap, local_storage); 377 376 raw_spin_lock_irqsave(&b->lock, flags); 378 377 hlist_add_head_rcu(&selem->map_node, &b->list); 379 378 raw_spin_unlock_irqrestore(&b->lock, flags); ··· 453 448 storage->use_kmalloc_nolock = smap->use_kmalloc_nolock; 454 449 455 450 bpf_selem_link_storage_nolock(storage, first_selem); 456 - bpf_selem_link_map(smap, first_selem); 451 + bpf_selem_link_map(smap, storage, first_selem); 457 452 458 453 owner_storage_ptr = 459 454 (struct bpf_local_storage **)owner_storage(smap, owner); ··· 581 576 582 577 alloc_selem = NULL; 583 578 /* First, link the new selem to the map */ 584 - bpf_selem_link_map(smap, selem); 579 + bpf_selem_link_map(smap, local_storage, selem); 585 580 586 581 /* Second, link (and publish) the new selem to local_storage */ 587 582 bpf_selem_link_storage_nolock(local_storage, selem);
+1 -1
net/core/bpf_sk_storage.c
··· 191 191 } 192 192 193 193 if (new_sk_storage) { 194 - bpf_selem_link_map(smap, copy_selem); 194 + bpf_selem_link_map(smap, new_sk_storage, copy_selem); 195 195 bpf_selem_link_storage_nolock(new_sk_storage, copy_selem); 196 196 } else { 197 197 ret = bpf_local_storage_alloc(newsk, smap, copy_selem, GFP_ATOMIC);