Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
rhashtable: use likely for rhashtable lookup
Sometimes, the result of the rhashtable_lookup() is expected to be found.
Therefore, we can use likely() for such cases.
Following new functions are introduced, which will use likely or unlikely
during the lookup:
rhashtable_lookup_likely
rhltable_lookup_likely
A micro-benchmark is made for these new functions: lookup a existed entry
repeatedly for 100000000 times, and rhashtable_lookup_likely() gets ~30%
speedup.
Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
authored by
Menglong Dong
and committed by
Herbert Xu
aa653654
9b04d8f0
+58
-12
+58
-12
include/linux/rhashtable.h
+58
-12
include/linux/rhashtable.h
···
355
355
local_irq_restore(flags);
356
356
}
357
357
358
-
static inline struct rhash_head *__rht_ptr(
359
-
struct rhash_lock_head *p, struct rhash_lock_head __rcu *const *bkt)
358
+
enum rht_lookup_freq {
359
+
RHT_LOOKUP_NORMAL,
360
+
RHT_LOOKUP_LIKELY,
361
+
};
362
+
363
+
static __always_inline struct rhash_head *__rht_ptr(
364
+
struct rhash_lock_head *p, struct rhash_lock_head __rcu *const *bkt,
365
+
const enum rht_lookup_freq freq)
360
366
{
361
-
return (struct rhash_head *)
362
-
((unsigned long)p & ~BIT(0) ?:
363
-
(unsigned long)RHT_NULLS_MARKER(bkt));
367
+
unsigned long p_val = (unsigned long)p & ~BIT(0);
368
+
369
+
BUILD_BUG_ON(!__builtin_constant_p(freq));
370
+
371
+
if (freq == RHT_LOOKUP_LIKELY)
372
+
return (struct rhash_head *)
373
+
(likely(p_val) ? p_val : (unsigned long)RHT_NULLS_MARKER(bkt));
374
+
else
375
+
return (struct rhash_head *)
376
+
(p_val ?: (unsigned long)RHT_NULLS_MARKER(bkt));
364
377
}
365
378
366
379
/*
···
383
370
* rht_ptr_exclusive() dereferences in a context where exclusive
384
371
* access is guaranteed, such as when destroying the table.
385
372
*/
373
+
static __always_inline struct rhash_head *__rht_ptr_rcu(
374
+
struct rhash_lock_head __rcu *const *bkt,
375
+
const enum rht_lookup_freq freq)
376
+
{
377
+
return __rht_ptr(rcu_dereference_all(*bkt), bkt, freq);
378
+
}
379
+
386
380
static inline struct rhash_head *rht_ptr_rcu(
387
381
struct rhash_lock_head __rcu *const *bkt)
388
382
{
389
-
return __rht_ptr(rcu_dereference_all(*bkt), bkt);
383
+
return __rht_ptr_rcu(bkt, RHT_LOOKUP_NORMAL);
390
384
}
391
385
392
386
static inline struct rhash_head *rht_ptr(
···
401
381
struct bucket_table *tbl,
402
382
unsigned int hash)
403
383
{
404
-
return __rht_ptr(rht_dereference_bucket(*bkt, tbl, hash), bkt);
384
+
return __rht_ptr(rht_dereference_bucket(*bkt, tbl, hash), bkt,
385
+
RHT_LOOKUP_NORMAL);
405
386
}
406
387
407
388
static inline struct rhash_head *rht_ptr_exclusive(
408
389
struct rhash_lock_head __rcu *const *bkt)
409
390
{
410
-
return __rht_ptr(rcu_dereference_protected(*bkt, 1), bkt);
391
+
return __rht_ptr(rcu_dereference_protected(*bkt, 1), bkt,
392
+
RHT_LOOKUP_NORMAL);
411
393
}
412
394
413
395
static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt,
···
610
588
/* Internal function, do not use. */
611
589
static __always_inline struct rhash_head *__rhashtable_lookup(
612
590
struct rhashtable *ht, const void *key,
613
-
const struct rhashtable_params params)
591
+
const struct rhashtable_params params,
592
+
const enum rht_lookup_freq freq)
614
593
{
615
594
struct rhashtable_compare_arg arg = {
616
595
.ht = ht,
···
622
599
struct rhash_head *he;
623
600
unsigned int hash;
624
601
602
+
BUILD_BUG_ON(!__builtin_constant_p(freq));
625
603
tbl = rht_dereference_rcu(ht->tbl, ht);
626
604
restart:
627
605
hash = rht_key_hashfn(ht, tbl, key, params);
628
606
bkt = rht_bucket(tbl, hash);
629
607
do {
630
-
rht_for_each_rcu_from(he, rht_ptr_rcu(bkt), tbl, hash) {
608
+
rht_for_each_rcu_from(he, __rht_ptr_rcu(bkt, freq), tbl, hash) {
631
609
if (params.obj_cmpfn ?
632
610
params.obj_cmpfn(&arg, rht_obj(ht, he)) :
633
611
rhashtable_compare(&arg, rht_obj(ht, he)))
···
667
643
struct rhashtable *ht, const void *key,
668
644
const struct rhashtable_params params)
669
645
{
670
-
struct rhash_head *he = __rhashtable_lookup(ht, key, params);
646
+
struct rhash_head *he = __rhashtable_lookup(ht, key, params,
647
+
RHT_LOOKUP_NORMAL);
671
648
672
649
return he ? rht_obj(ht, he) : NULL;
650
+
}
651
+
652
+
static __always_inline void *rhashtable_lookup_likely(
653
+
struct rhashtable *ht, const void *key,
654
+
const struct rhashtable_params params)
655
+
{
656
+
struct rhash_head *he = __rhashtable_lookup(ht, key, params,
657
+
RHT_LOOKUP_LIKELY);
658
+
659
+
return likely(he) ? rht_obj(ht, he) : NULL;
673
660
}
674
661
675
662
/**
···
728
693
struct rhltable *hlt, const void *key,
729
694
const struct rhashtable_params params)
730
695
{
731
-
struct rhash_head *he = __rhashtable_lookup(&hlt->ht, key, params);
696
+
struct rhash_head *he = __rhashtable_lookup(&hlt->ht, key, params,
697
+
RHT_LOOKUP_NORMAL);
732
698
733
699
return he ? container_of(he, struct rhlist_head, rhead) : NULL;
700
+
}
701
+
702
+
static __always_inline struct rhlist_head *rhltable_lookup_likely(
703
+
struct rhltable *hlt, const void *key,
704
+
const struct rhashtable_params params)
705
+
{
706
+
struct rhash_head *he = __rhashtable_lookup(&hlt->ht, key, params,
707
+
RHT_LOOKUP_LIKELY);
708
+
709
+
return likely(he) ? container_of(he, struct rhlist_head, rhead) : NULL;
734
710
}
735
711
736
712
/* Internal function, please use rhashtable_insert_fast() instead. This