Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
bpf: Save memory alloction info in bpf_local_storage
Save the memory allocation method used for bpf_local_storage in the
struct explicitly so that we don't need to go through the hassle to
find out the info. When a later patch replaces BPF memory allocator
with kmalloc_noloc(), bpf_local_storage_free() will no longer need
smap->storage_ma to return the memory and completely remove the
dependency on smap in bpf_local_storage_free().
Signed-off-by: Amery Hung <ameryhung@gmail.com>
Reviewed-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20251114201329.3275875-4-ameryhung@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
authored by
Amery Hung
and committed by
Alexei Starovoitov
39a460c4
e76a33e1
+9
-44
+1
include/linux/bpf_local_storage.h
+1
include/linux/bpf_local_storage.h
+8
-44
kernel/bpf/bpf_local_storage.c
+8
-44
kernel/bpf/bpf_local_storage.c
···
157
157
158
158
static void bpf_local_storage_free(struct bpf_local_storage *local_storage,
159
159
struct bpf_local_storage_map *smap,
160
-
bool bpf_ma, bool reuse_now)
160
+
bool reuse_now)
161
161
{
162
162
if (!local_storage)
163
163
return;
164
164
165
-
if (!bpf_ma) {
165
+
if (!local_storage->bpf_ma) {
166
166
__bpf_local_storage_free(local_storage, reuse_now);
167
167
return;
168
168
}
···
336
336
return free_local_storage;
337
337
}
338
338
339
-
static bool check_storage_bpf_ma(struct bpf_local_storage *local_storage,
340
-
struct bpf_local_storage_map *storage_smap,
341
-
struct bpf_local_storage_elem *selem)
342
-
{
343
-
344
-
struct bpf_local_storage_map *selem_smap;
345
-
346
-
/* local_storage->smap may be NULL. If it is, get the bpf_ma
347
-
* from any selem in the local_storage->list. The bpf_ma of all
348
-
* local_storage and selem should have the same value
349
-
* for the same map type.
350
-
*
351
-
* If the local_storage->list is already empty, the caller will not
352
-
* care about the bpf_ma value also because the caller is not
353
-
* responsible to free the local_storage.
354
-
*/
355
-
356
-
if (storage_smap)
357
-
return storage_smap->bpf_ma;
358
-
359
-
if (!selem) {
360
-
struct hlist_node *n;
361
-
362
-
n = rcu_dereference_check(hlist_first_rcu(&local_storage->list),
363
-
bpf_rcu_lock_held());
364
-
if (!n)
365
-
return false;
366
-
367
-
selem = hlist_entry(n, struct bpf_local_storage_elem, snode);
368
-
}
369
-
selem_smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
370
-
371
-
return selem_smap->bpf_ma;
372
-
}
373
-
374
339
static void bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem,
375
340
bool reuse_now)
376
341
{
377
342
struct bpf_local_storage_map *storage_smap;
378
343
struct bpf_local_storage *local_storage;
379
-
bool bpf_ma, free_local_storage = false;
344
+
bool free_local_storage = false;
380
345
HLIST_HEAD(selem_free_list);
381
346
unsigned long flags;
382
347
···
353
388
bpf_rcu_lock_held());
354
389
storage_smap = rcu_dereference_check(local_storage->smap,
355
390
bpf_rcu_lock_held());
356
-
bpf_ma = check_storage_bpf_ma(local_storage, storage_smap, selem);
357
391
358
392
raw_spin_lock_irqsave(&local_storage->lock, flags);
359
393
if (likely(selem_linked_to_storage(selem)))
···
363
399
bpf_selem_free_list(&selem_free_list, reuse_now);
364
400
365
401
if (free_local_storage)
366
-
bpf_local_storage_free(local_storage, storage_smap, bpf_ma, reuse_now);
402
+
bpf_local_storage_free(local_storage, storage_smap, reuse_now);
367
403
}
368
404
369
405
void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
···
470
506
INIT_HLIST_HEAD(&storage->list);
471
507
raw_spin_lock_init(&storage->lock);
472
508
storage->owner = owner;
509
+
storage->bpf_ma = smap->bpf_ma;
473
510
474
511
bpf_selem_link_storage_nolock(storage, first_selem);
475
512
bpf_selem_link_map(smap, first_selem);
···
507
542
return 0;
508
543
509
544
uncharge:
510
-
bpf_local_storage_free(storage, smap, smap->bpf_ma, true);
545
+
bpf_local_storage_free(storage, smap, true);
511
546
mem_uncharge(smap, owner, sizeof(*storage));
512
547
return err;
513
548
}
···
696
731
{
697
732
struct bpf_local_storage_map *storage_smap;
698
733
struct bpf_local_storage_elem *selem;
699
-
bool bpf_ma, free_storage = false;
734
+
bool free_storage = false;
700
735
HLIST_HEAD(free_selem_list);
701
736
struct hlist_node *n;
702
737
unsigned long flags;
703
738
704
739
storage_smap = rcu_dereference_check(local_storage->smap, bpf_rcu_lock_held());
705
-
bpf_ma = check_storage_bpf_ma(local_storage, storage_smap, NULL);
706
740
707
741
/* Neither the bpf_prog nor the bpf_map's syscall
708
742
* could be modifying the local_storage->list now.
···
732
768
bpf_selem_free_list(&free_selem_list, true);
733
769
734
770
if (free_storage)
735
-
bpf_local_storage_free(local_storage, storage_smap, bpf_ma, true);
771
+
bpf_local_storage_free(local_storage, storage_smap, true);
736
772
}
737
773
738
774
u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map)