tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

bpf: Prepare for bpf_selem_unlink_nofail()

The next patch will introduce bpf_selem_unlink_nofail() to handle
rqspinlock errors. bpf_selem_unlink_nofail() will allow an selem to be
partially unlinked from map or local storage. Save memory allocation
method in selem so that later an selem can be correctly freed even when
SDATA(selem)->smap is init to NULL.

In addition, keep track of memory charge to the owner in local storage
so that later bpf_selem_unlink_nofail() can return the correct memory
charge to the owner. Updating local_storage->mem_charge is protected by
local_storage->lock.

Finally, extract miscellaneous tasks performed when unlinking an selem
from local_storage into bpf_selem_unlink_storage_nolock_misc(). It will
be reused by bpf_selem_unlink_nofail().

This patch also takes the chance to remove local_storage->smap, which
is no longer used since commit f484f4a3e058 ("bpf: Replace bpf memory
allocator with kmalloc_nolock() in local storage").

Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Amery Hung <ameryhung@gmail.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://patch.msgid.link/20260205222916.1788211-10-ameryhung@gmail.com

authored by

Amery Hung and committed by
Martin KaFai Lau c8be3da1 3417dffb

+37 -37
+3 -2
include/linux/bpf_local_storage.h
··· 80 80 * after raw_spin_unlock 81 81 */ 82 82 }; 83 - /* 8 bytes hole */ 83 + bool use_kmalloc_nolock; 84 + /* 7 bytes hole */ 84 85 /* The data is stored in another cacheline to minimize 85 86 * the number of cachelines access during a cache hit. 86 87 */ ··· 90 89 91 90 struct bpf_local_storage { 92 91 struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE]; 93 - struct bpf_local_storage_map __rcu *smap; 94 92 struct hlist_head list; /* List of bpf_local_storage_elem */ 95 93 void *owner; /* The object that owns the above "list" of 96 94 * bpf_local_storage_elem. 97 95 */ 98 96 struct rcu_head rcu; 99 97 rqspinlock_t lock; /* Protect adding/removing from the "list" */ 98 + u64 mem_charge; /* Copy of mem charged to owner. Protected by "lock" */ 100 99 bool use_kmalloc_nolock; 101 100 }; 102 101
+34 -35
kernel/bpf/bpf_local_storage.c
··· 85 85 86 86 if (selem) { 87 87 RCU_INIT_POINTER(SDATA(selem)->smap, smap); 88 + selem->use_kmalloc_nolock = smap->use_kmalloc_nolock; 88 89 89 90 if (value) { 90 91 /* No need to call check_and_init_map_value as memory is zero init */ ··· 215 214 216 215 smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); 217 216 218 - if (!smap->use_kmalloc_nolock) { 217 + if (!selem->use_kmalloc_nolock) { 219 218 /* 220 219 * No uptr will be unpin even when reuse_now == false since uptr 221 220 * is only supported in task local storage, where ··· 252 251 bpf_selem_free(selem, reuse_now); 253 252 } 254 253 254 + static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem *selem, 255 + struct bpf_local_storage_map *smap, 256 + struct bpf_local_storage *local_storage, 257 + bool free_local_storage) 258 + { 259 + void *owner = local_storage->owner; 260 + u32 uncharge = smap->elem_size; 261 + 262 + if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) == 263 + SDATA(selem)) 264 + RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL); 265 + 266 + uncharge += free_local_storage ? sizeof(*local_storage) : 0; 267 + mem_uncharge(smap, local_storage->owner, uncharge); 268 + local_storage->mem_charge -= uncharge; 269 + 270 + if (free_local_storage) { 271 + local_storage->owner = NULL; 272 + 273 + /* After this RCU_INIT, owner may be freed and cannot be used */ 274 + RCU_INIT_POINTER(*owner_storage(smap, owner), NULL); 275 + } 276 + } 277 + 255 278 /* local_storage->lock must be held and selem->local_storage == local_storage. 256 279 * The caller must ensure selem->smap is still valid to be 257 280 * dereferenced for its smap->elem_size and smap->cache_idx. ··· 286 261 { 287 262 struct bpf_local_storage_map *smap; 288 263 bool free_local_storage; 289 - void *owner; 290 264 291 265 smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); 292 - owner = local_storage->owner; 293 - 294 - /* All uncharging on the owner must be done first. 295 - * The owner may be freed once the last selem is unlinked 296 - * from local_storage. 297 - */ 298 - mem_uncharge(smap, owner, smap->elem_size); 299 266 300 267 free_local_storage = hlist_is_singular_node(&selem->snode, 301 268 &local_storage->list); 302 - if (free_local_storage) { 303 - mem_uncharge(smap, owner, sizeof(struct bpf_local_storage)); 304 - local_storage->owner = NULL; 305 269 306 - /* After this RCU_INIT, owner may be freed and cannot be used */ 307 - RCU_INIT_POINTER(*owner_storage(smap, owner), NULL); 270 + bpf_selem_unlink_storage_nolock_misc(selem, smap, local_storage, 271 + free_local_storage); 308 272 309 - /* local_storage is not freed now. local_storage->lock is 310 - * still held and raw_spin_unlock_bh(&local_storage->lock) 311 - * will be done by the caller. 312 - * 313 - * Although the unlock will be done under 314 - * rcu_read_lock(), it is more intuitive to 315 - * read if the freeing of the storage is done 316 - * after the raw_spin_unlock_bh(&local_storage->lock). 317 - * 318 - * Hence, a "bool free_local_storage" is returned 319 - * to the caller which then calls then frees the storage after 320 - * all the RCU grace periods have expired. 321 - */ 322 - } 323 273 hlist_del_init_rcu(&selem->snode); 324 - if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) == 325 - SDATA(selem)) 326 - RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL); 327 274 328 275 hlist_add_head(&selem->free_node, free_selem_list); 329 - 330 - if (rcu_access_pointer(local_storage->smap) == smap) 331 - RCU_INIT_POINTER(local_storage->smap, NULL); 332 276 333 277 return free_local_storage; 334 278 } ··· 305 311 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, 306 312 struct bpf_local_storage_elem *selem) 307 313 { 314 + struct bpf_local_storage_map *smap; 315 + 316 + smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); 317 + local_storage->mem_charge += smap->elem_size; 318 + 308 319 RCU_INIT_POINTER(selem->local_storage, local_storage); 309 320 hlist_add_head_rcu(&selem->snode, &local_storage->list); 310 321 } ··· 470 471 goto uncharge; 471 472 } 472 473 473 - RCU_INIT_POINTER(storage->smap, smap); 474 474 INIT_HLIST_HEAD(&storage->list); 475 475 raw_res_spin_lock_init(&storage->lock); 476 476 storage->owner = owner; 477 + storage->mem_charge = sizeof(*storage); 477 478 storage->use_kmalloc_nolock = smap->use_kmalloc_nolock; 478 479 479 480 bpf_selem_link_storage_nolock(storage, first_selem);