Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
bpf: Change local_storage->lock and b->lock to rqspinlock
Change bpf_local_storage::lock and bpf_local_storage_map_bucket::lock
from raw_spin_lock to rqspinlock.
Finally, propagate errors from raw_res_spin_lock_irqsave() to syscall
return or BPF helper return.
In bpf_local_storage_destroy(), ignore return from
raw_res_spin_lock_irqsave() for now. A later patch will correctly
handle errors correctly in bpf_local_storage_destroy() so that it can
unlink selems even when failing to acquire locks.
For __bpf_local_storage_map_cache(), instead of handling the error,
skip updating the cache.
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Amery Hung <ameryhung@gmail.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://patch.msgid.link/20260205222916.1788211-6-ameryhung@gmail.com
authored by
Amery Hung
and committed by
Martin KaFai Lau
8dabe34b
403e935f
+47
-22
+3
-2
include/linux/bpf_local_storage.h
+3
-2
include/linux/bpf_local_storage.h
···
15
15
#include <linux/types.h>
16
16
#include <linux/bpf_mem_alloc.h>
17
17
#include <uapi/linux/btf.h>
18
+
#include <asm/rqspinlock.h>
18
19
19
20
#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
20
21
21
22
struct bpf_local_storage_map_bucket {
22
23
struct hlist_head list;
23
-
raw_spinlock_t lock;
24
+
rqspinlock_t lock;
24
25
};
25
26
26
27
/* Thp map is not the primary owner of a bpf_local_storage_elem.
···
95
94
* bpf_local_storage_elem.
96
95
*/
97
96
struct rcu_head rcu;
98
-
raw_spinlock_t lock; /* Protect adding/removing from the "list" */
97
+
rqspinlock_t lock; /* Protect adding/removing from the "list" */
99
98
bool use_kmalloc_nolock;
100
99
};
101
100
+44
-20
kernel/bpf/bpf_local_storage.c
+44
-20
kernel/bpf/bpf_local_storage.c
···
321
321
struct bpf_local_storage_map *smap;
322
322
struct bpf_local_storage_map_bucket *b;
323
323
unsigned long flags;
324
+
int err;
324
325
325
326
local_storage = rcu_dereference_check(selem->local_storage,
326
327
bpf_rcu_lock_held());
327
328
smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
328
329
b = select_bucket(smap, local_storage);
329
-
raw_spin_lock_irqsave(&b->lock, flags);
330
+
err = raw_res_spin_lock_irqsave(&b->lock, flags);
331
+
if (err)
332
+
return err;
333
+
330
334
hlist_del_init_rcu(&selem->map_node);
331
-
raw_spin_unlock_irqrestore(&b->lock, flags);
335
+
raw_res_spin_unlock_irqrestore(&b->lock, flags);
332
336
333
337
return 0;
334
338
}
···
348
344
{
349
345
struct bpf_local_storage_map_bucket *b;
350
346
unsigned long flags;
347
+
int err;
351
348
352
349
b = select_bucket(smap, local_storage);
353
-
raw_spin_lock_irqsave(&b->lock, flags);
350
+
351
+
err = raw_res_spin_lock_irqsave(&b->lock, flags);
352
+
if (err)
353
+
return err;
354
+
354
355
hlist_add_head_rcu(&selem->map_node, &b->list);
355
-
raw_spin_unlock_irqrestore(&b->lock, flags);
356
+
raw_res_spin_unlock_irqrestore(&b->lock, flags);
356
357
357
358
return 0;
358
359
}
···
374
365
bool free_local_storage = false;
375
366
HLIST_HEAD(selem_free_list);
376
367
unsigned long flags;
377
-
int err = 0;
368
+
int err;
378
369
379
370
if (unlikely(!selem_linked_to_storage_lockless(selem)))
380
371
/* selem has already been unlinked from sk */
···
383
374
local_storage = rcu_dereference_check(selem->local_storage,
384
375
bpf_rcu_lock_held());
385
376
386
-
raw_spin_lock_irqsave(&local_storage->lock, flags);
377
+
err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);
378
+
if (err)
379
+
return err;
380
+
387
381
if (likely(selem_linked_to_storage(selem))) {
388
382
/* Always unlink from map before unlinking from local_storage
389
383
* because selem will be freed after successfully unlinked from
···
400
388
local_storage, selem, &selem_free_list);
401
389
}
402
390
out:
403
-
raw_spin_unlock_irqrestore(&local_storage->lock, flags);
391
+
raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);
404
392
405
393
bpf_selem_free_list(&selem_free_list, reuse_now);
406
394
···
415
403
struct bpf_local_storage_elem *selem)
416
404
{
417
405
unsigned long flags;
406
+
int err;
418
407
419
408
/* spinlock is needed to avoid racing with the
420
409
* parallel delete. Otherwise, publishing an already
421
410
* deleted sdata to the cache will become a use-after-free
422
411
* problem in the next bpf_local_storage_lookup().
423
412
*/
424
-
raw_spin_lock_irqsave(&local_storage->lock, flags);
413
+
err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);
414
+
if (err)
415
+
return;
416
+
425
417
if (selem_linked_to_storage(selem))
426
418
rcu_assign_pointer(local_storage->cache[smap->cache_idx], SDATA(selem));
427
-
raw_spin_unlock_irqrestore(&local_storage->lock, flags);
419
+
raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);
428
420
}
429
421
430
422
static int check_flags(const struct bpf_local_storage_data *old_sdata,
···
473
457
474
458
RCU_INIT_POINTER(storage->smap, smap);
475
459
INIT_HLIST_HEAD(&storage->list);
476
-
raw_spin_lock_init(&storage->lock);
460
+
raw_res_spin_lock_init(&storage->lock);
477
461
storage->owner = owner;
478
462
storage->use_kmalloc_nolock = smap->use_kmalloc_nolock;
479
463
480
464
bpf_selem_link_storage_nolock(storage, first_selem);
481
465
482
466
b = select_bucket(smap, storage);
483
-
raw_spin_lock_irqsave(&b->lock, flags);
467
+
err = raw_res_spin_lock_irqsave(&b->lock, flags);
468
+
if (err)
469
+
goto uncharge;
470
+
484
471
bpf_selem_link_map_nolock(b, first_selem);
485
472
486
473
owner_storage_ptr =
···
501
482
prev_storage = cmpxchg(owner_storage_ptr, NULL, storage);
502
483
if (unlikely(prev_storage)) {
503
484
bpf_selem_unlink_map_nolock(first_selem);
504
-
raw_spin_unlock_irqrestore(&b->lock, flags);
485
+
raw_res_spin_unlock_irqrestore(&b->lock, flags);
505
486
err = -EAGAIN;
506
487
goto uncharge;
507
488
}
508
-
raw_spin_unlock_irqrestore(&b->lock, flags);
489
+
raw_res_spin_unlock_irqrestore(&b->lock, flags);
509
490
510
491
return 0;
511
492
···
588
569
if (!alloc_selem)
589
570
return ERR_PTR(-ENOMEM);
590
571
591
-
raw_spin_lock_irqsave(&local_storage->lock, flags);
572
+
err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);
573
+
if (err)
574
+
goto free_selem;
592
575
593
576
/* Recheck local_storage->list under local_storage->lock */
594
577
if (unlikely(hlist_empty(&local_storage->list))) {
···
617
596
618
597
b = select_bucket(smap, local_storage);
619
598
620
-
raw_spin_lock_irqsave(&b->lock, b_flags);
599
+
err = raw_res_spin_lock_irqsave(&b->lock, b_flags);
600
+
if (err)
601
+
goto unlock;
621
602
622
603
alloc_selem = NULL;
623
604
/* First, link the new selem to the map */
···
635
612
&old_selem_free_list);
636
613
}
637
614
638
-
raw_spin_unlock_irqrestore(&b->lock, b_flags);
615
+
raw_res_spin_unlock_irqrestore(&b->lock, b_flags);
639
616
unlock:
640
-
raw_spin_unlock_irqrestore(&local_storage->lock, flags);
617
+
raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);
618
+
free_selem:
641
619
bpf_selem_free_list(&old_selem_free_list, false);
642
620
if (alloc_selem) {
643
621
mem_uncharge(smap, owner, smap->elem_size);
···
723
699
* when unlinking elem from the local_storage->list and
724
700
* the map's bucket->list.
725
701
*/
726
-
raw_spin_lock_irqsave(&local_storage->lock, flags);
702
+
raw_res_spin_lock_irqsave(&local_storage->lock, flags);
727
703
hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
728
704
/* Always unlink from map before unlinking from
729
705
* local_storage.
···
738
714
free_storage = bpf_selem_unlink_storage_nolock(
739
715
local_storage, selem, &free_selem_list);
740
716
}
741
-
raw_spin_unlock_irqrestore(&local_storage->lock, flags);
717
+
raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);
742
718
743
719
bpf_selem_free_list(&free_selem_list, true);
744
720
···
785
761
786
762
for (i = 0; i < nbuckets; i++) {
787
763
INIT_HLIST_HEAD(&smap->buckets[i].list);
788
-
raw_spin_lock_init(&smap->buckets[i].lock);
764
+
raw_res_spin_lock_init(&smap->buckets[i].lock);
789
765
}
790
766
791
767
smap->elem_size = offsetof(struct bpf_local_storage_elem,