Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'locking-futex-2025-07-29' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull futex updates from Thomas Gleixner:
- Switch the reference counting to a RCU based per-CPU reference to
address a performance bottleneck vs the single instance rcuref
variant
- Make the futex selftest build on 32-bit architectures which only
support 64-bit time_t, e.g. RISCV-32
- Cleanups and improvements in selftests and futex bench
* tag 'locking-futex-2025-07-29' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
selftests/futex: Fix spelling mistake "Succeffuly" -> "Successfully"
selftests/futex: Define SYS_futex on 32-bit architectures with 64-bit time_t
perf bench futex: Remove support for IMMUTABLE
selftests/futex: Remove support for IMMUTABLE
futex: Remove support for IMMUTABLE
futex: Make futex_private_hash_get() static
futex: Use RCU-based per-CPU reference counting instead of rcuref_t
selftests/futex: Adapt the private hash test to RCU related changes
+326
-149
+5
-11
include/linux/futex.h
+5
-11
include/linux/futex.h
···
85
85
#ifdef CONFIG_FUTEX_PRIVATE_HASH
86
86
int futex_hash_allocate_default(void);
87
87
void futex_hash_free(struct mm_struct *mm);
88
-
89
-
static inline void futex_mm_init(struct mm_struct *mm)
90
-
{
91
-
RCU_INIT_POINTER(mm->futex_phash, NULL);
92
-
mm->futex_phash_new = NULL;
93
-
mutex_init(&mm->futex_hash_lock);
94
-
}
88
+
int futex_mm_init(struct mm_struct *mm);
95
89
96
90
#else /* !CONFIG_FUTEX_PRIVATE_HASH */
97
91
static inline int futex_hash_allocate_default(void) { return 0; }
98
-
static inline void futex_hash_free(struct mm_struct *mm) { }
99
-
static inline void futex_mm_init(struct mm_struct *mm) { }
92
+
static inline int futex_hash_free(struct mm_struct *mm) { return 0; }
93
+
static inline int futex_mm_init(struct mm_struct *mm) { return 0; }
100
94
#endif /* CONFIG_FUTEX_PRIVATE_HASH */
101
95
102
96
#else /* !CONFIG_FUTEX */
···
112
118
{
113
119
return 0;
114
120
}
115
-
static inline void futex_hash_free(struct mm_struct *mm) { }
116
-
static inline void futex_mm_init(struct mm_struct *mm) { }
121
+
static inline int futex_hash_free(struct mm_struct *mm) { return 0; }
122
+
static inline int futex_mm_init(struct mm_struct *mm) { return 0; }
117
123
118
124
#endif
119
125
+5
include/linux/mm_types.h
+5
include/linux/mm_types.h
···
1070
1070
struct mutex futex_hash_lock;
1071
1071
struct futex_private_hash __rcu *futex_phash;
1072
1072
struct futex_private_hash *futex_phash_new;
1073
+
/* futex-ref */
1074
+
unsigned long futex_batches;
1075
+
struct rcu_head futex_rcu;
1076
+
atomic_long_t futex_atomic;
1077
+
unsigned int __percpu *futex_ref;
1073
1078
#endif
1074
1079
1075
1080
unsigned long hiwater_rss; /* High-watermark of RSS usage */
+1
-1
include/linux/sched/mm.h
+1
-1
include/linux/sched/mm.h
···
140
140
141
141
/* mmput gets rid of the mappings and all user-space */
142
142
extern void mmput(struct mm_struct *);
143
-
#ifdef CONFIG_MMU
143
+
#if defined(CONFIG_MMU) || defined(CONFIG_FUTEX_PRIVATE_HASH)
144
144
/* same as above but performs the slow path from the async context. Can
145
145
* be called from the atomic context as well
146
146
*/
-2
include/uapi/linux/prctl.h
-2
include/uapi/linux/prctl.h
-4
init/Kconfig
-4
init/Kconfig
···
1716
1716
depends on FUTEX && RT_MUTEXES
1717
1717
default y
1718
1718
1719
-
#
1720
-
# marked broken for performance reasons; gives us one more cycle to sort things out.
1721
-
#
1722
1719
config FUTEX_PRIVATE_HASH
1723
1720
bool
1724
1721
depends on FUTEX && !BASE_SMALL && MMU
1725
-
depends on BROKEN
1726
1722
default y
1727
1723
1728
1724
config FUTEX_MPOL
+6
-2
kernel/fork.c
+6
-2
kernel/fork.c
···
1046
1046
RCU_INIT_POINTER(mm->exe_file, NULL);
1047
1047
mmu_notifier_subscriptions_init(mm);
1048
1048
init_tlb_flush_pending(mm);
1049
-
futex_mm_init(mm);
1050
1049
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
1051
1050
mm->pmd_huge_pte = NULL;
1052
1051
#endif
···
1059
1060
mm->flags = default_dump_filter;
1060
1061
mm->def_flags = 0;
1061
1062
}
1063
+
1064
+
if (futex_mm_init(mm))
1065
+
goto fail_mm_init;
1062
1066
1063
1067
if (mm_alloc_pgd(mm))
1064
1068
goto fail_nopgd;
···
1092
1090
fail_noid:
1093
1091
mm_free_pgd(mm);
1094
1092
fail_nopgd:
1093
+
futex_hash_free(mm);
1094
+
fail_mm_init:
1095
1095
free_mm(mm);
1096
1096
return NULL;
1097
1097
}
···
1149
1145
}
1150
1146
EXPORT_SYMBOL_GPL(mmput);
1151
1147
1152
-
#ifdef CONFIG_MMU
1148
+
#if defined(CONFIG_MMU) || defined(CONFIG_FUTEX_PRIVATE_HASH)
1153
1149
static void mmput_async_fn(struct work_struct *work)
1154
1150
{
1155
1151
struct mm_struct *mm = container_of(work, struct mm_struct,
+230
-51
kernel/futex/core.c
+230
-51
kernel/futex/core.c
···
42
42
#include <linux/fault-inject.h>
43
43
#include <linux/slab.h>
44
44
#include <linux/prctl.h>
45
-
#include <linux/rcuref.h>
46
45
#include <linux/mempolicy.h>
47
46
#include <linux/mmap_lock.h>
48
47
···
64
65
#define futex_queues (__futex_data.queues)
65
66
66
67
struct futex_private_hash {
67
-
rcuref_t users;
68
+
int state;
68
69
unsigned int hash_mask;
69
70
struct rcu_head rcu;
70
71
void *mm;
71
72
bool custom;
72
-
bool immutable;
73
73
struct futex_hash_bucket queues[];
74
74
};
75
75
···
127
129
__futex_hash(union futex_key *key, struct futex_private_hash *fph);
128
130
129
131
#ifdef CONFIG_FUTEX_PRIVATE_HASH
132
+
static bool futex_ref_get(struct futex_private_hash *fph);
133
+
static bool futex_ref_put(struct futex_private_hash *fph);
134
+
static bool futex_ref_is_dead(struct futex_private_hash *fph);
135
+
136
+
enum { FR_PERCPU = 0, FR_ATOMIC };
137
+
130
138
static inline bool futex_key_is_private(union futex_key *key)
131
139
{
132
140
/*
···
142
138
return !(key->both.offset & (FUT_OFF_INODE | FUT_OFF_MMSHARED));
143
139
}
144
140
145
-
bool futex_private_hash_get(struct futex_private_hash *fph)
141
+
static bool futex_private_hash_get(struct futex_private_hash *fph)
146
142
{
147
-
if (fph->immutable)
148
-
return true;
149
-
return rcuref_get(&fph->users);
143
+
return futex_ref_get(fph);
150
144
}
151
145
152
146
void futex_private_hash_put(struct futex_private_hash *fph)
153
147
{
154
-
/* Ignore return value, last put is verified via rcuref_is_dead() */
155
-
if (fph->immutable)
156
-
return;
157
-
if (rcuref_put(&fph->users))
148
+
if (futex_ref_put(fph))
158
149
wake_up_var(fph->mm);
159
150
}
160
151
···
242
243
fph = rcu_dereference_protected(mm->futex_phash,
243
244
lockdep_is_held(&mm->futex_hash_lock));
244
245
if (fph) {
245
-
if (!rcuref_is_dead(&fph->users)) {
246
+
if (!futex_ref_is_dead(fph)) {
246
247
mm->futex_phash_new = new;
247
248
return false;
248
249
}
249
250
250
251
futex_rehash_private(fph, new);
251
252
}
252
-
rcu_assign_pointer(mm->futex_phash, new);
253
+
new->state = FR_PERCPU;
254
+
scoped_guard(rcu) {
255
+
mm->futex_batches = get_state_synchronize_rcu();
256
+
rcu_assign_pointer(mm->futex_phash, new);
257
+
}
253
258
kvfree_rcu(fph, rcu);
254
259
return true;
255
260
}
···
292
289
if (!fph)
293
290
return NULL;
294
291
295
-
if (fph->immutable)
296
-
return fph;
297
-
if (rcuref_get(&fph->users))
292
+
if (futex_private_hash_get(fph))
298
293
return fph;
299
294
}
300
295
futex_pivot_hash(mm);
···
1525
1524
}
1526
1525
1527
1526
#define FH_CUSTOM 0x01
1528
-
#define FH_IMMUTABLE 0x02
1529
1527
1530
1528
#ifdef CONFIG_FUTEX_PRIVATE_HASH
1529
+
1530
+
/*
1531
+
* futex-ref
1532
+
*
1533
+
* Heavily inspired by percpu-rwsem/percpu-refcount; not reusing any of that
1534
+
* code because it just doesn't fit right.
1535
+
*
1536
+
* Dual counter, per-cpu / atomic approach like percpu-refcount, except it
1537
+
* re-initializes the state automatically, such that the fph swizzle is also a
1538
+
* transition back to per-cpu.
1539
+
*/
1540
+
1541
+
static void futex_ref_rcu(struct rcu_head *head);
1542
+
1543
+
static void __futex_ref_atomic_begin(struct futex_private_hash *fph)
1544
+
{
1545
+
struct mm_struct *mm = fph->mm;
1546
+
1547
+
/*
1548
+
* The counter we're about to switch to must have fully switched;
1549
+
* otherwise it would be impossible for it to have reported success
1550
+
* from futex_ref_is_dead().
1551
+
*/
1552
+
WARN_ON_ONCE(atomic_long_read(&mm->futex_atomic) != 0);
1553
+
1554
+
/*
1555
+
* Set the atomic to the bias value such that futex_ref_{get,put}()
1556
+
* will never observe 0. Will be fixed up in __futex_ref_atomic_end()
1557
+
* when folding in the percpu count.
1558
+
*/
1559
+
atomic_long_set(&mm->futex_atomic, LONG_MAX);
1560
+
smp_store_release(&fph->state, FR_ATOMIC);
1561
+
1562
+
call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu);
1563
+
}
1564
+
1565
+
static void __futex_ref_atomic_end(struct futex_private_hash *fph)
1566
+
{
1567
+
struct mm_struct *mm = fph->mm;
1568
+
unsigned int count = 0;
1569
+
long ret;
1570
+
int cpu;
1571
+
1572
+
/*
1573
+
* Per __futex_ref_atomic_begin() the state of the fph must be ATOMIC
1574
+
* and per this RCU callback, everybody must now observe this state and
1575
+
* use the atomic variable.
1576
+
*/
1577
+
WARN_ON_ONCE(fph->state != FR_ATOMIC);
1578
+
1579
+
/*
1580
+
* Therefore the per-cpu counter is now stable, sum and reset.
1581
+
*/
1582
+
for_each_possible_cpu(cpu) {
1583
+
unsigned int *ptr = per_cpu_ptr(mm->futex_ref, cpu);
1584
+
count += *ptr;
1585
+
*ptr = 0;
1586
+
}
1587
+
1588
+
/*
1589
+
* Re-init for the next cycle.
1590
+
*/
1591
+
this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */
1592
+
1593
+
/*
1594
+
* Add actual count, subtract bias and initial refcount.
1595
+
*
1596
+
* The moment this atomic operation happens, futex_ref_is_dead() can
1597
+
* become true.
1598
+
*/
1599
+
ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex_atomic);
1600
+
if (!ret)
1601
+
wake_up_var(mm);
1602
+
1603
+
WARN_ON_ONCE(ret < 0);
1604
+
mmput_async(mm);
1605
+
}
1606
+
1607
+
static void futex_ref_rcu(struct rcu_head *head)
1608
+
{
1609
+
struct mm_struct *mm = container_of(head, struct mm_struct, futex_rcu);
1610
+
struct futex_private_hash *fph = rcu_dereference_raw(mm->futex_phash);
1611
+
1612
+
if (fph->state == FR_PERCPU) {
1613
+
/*
1614
+
* Per this extra grace-period, everybody must now observe
1615
+
* fph as the current fph and no previously observed fph's
1616
+
* are in-flight.
1617
+
*
1618
+
* Notably, nobody will now rely on the atomic
1619
+
* futex_ref_is_dead() state anymore so we can begin the
1620
+
* migration of the per-cpu counter into the atomic.
1621
+
*/
1622
+
__futex_ref_atomic_begin(fph);
1623
+
return;
1624
+
}
1625
+
1626
+
__futex_ref_atomic_end(fph);
1627
+
}
1628
+
1629
+
/*
1630
+
* Drop the initial refcount and transition to atomics.
1631
+
*/
1632
+
static void futex_ref_drop(struct futex_private_hash *fph)
1633
+
{
1634
+
struct mm_struct *mm = fph->mm;
1635
+
1636
+
/*
1637
+
* Can only transition the current fph;
1638
+
*/
1639
+
WARN_ON_ONCE(rcu_dereference_raw(mm->futex_phash) != fph);
1640
+
/*
1641
+
* We enqueue at least one RCU callback. Ensure mm stays if the task
1642
+
* exits before the transition is completed.
1643
+
*/
1644
+
mmget(mm);
1645
+
1646
+
/*
1647
+
* In order to avoid the following scenario:
1648
+
*
1649
+
* futex_hash() __futex_pivot_hash()
1650
+
* guard(rcu); guard(mm->futex_hash_lock);
1651
+
* fph = mm->futex_phash;
1652
+
* rcu_assign_pointer(&mm->futex_phash, new);
1653
+
* futex_hash_allocate()
1654
+
* futex_ref_drop()
1655
+
* fph->state = FR_ATOMIC;
1656
+
* atomic_set(, BIAS);
1657
+
*
1658
+
* futex_private_hash_get(fph); // OOPS
1659
+
*
1660
+
* Where an old fph (which is FR_ATOMIC) and should fail on
1661
+
* inc_not_zero, will succeed because a new transition is started and
1662
+
* the atomic is bias'ed away from 0.
1663
+
*
1664
+
* There must be at least one full grace-period between publishing a
1665
+
* new fph and trying to replace it.
1666
+
*/
1667
+
if (poll_state_synchronize_rcu(mm->futex_batches)) {
1668
+
/*
1669
+
* There was a grace-period, we can begin now.
1670
+
*/
1671
+
__futex_ref_atomic_begin(fph);
1672
+
return;
1673
+
}
1674
+
1675
+
call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu);
1676
+
}
1677
+
1678
+
static bool futex_ref_get(struct futex_private_hash *fph)
1679
+
{
1680
+
struct mm_struct *mm = fph->mm;
1681
+
1682
+
guard(rcu)();
1683
+
1684
+
if (smp_load_acquire(&fph->state) == FR_PERCPU) {
1685
+
this_cpu_inc(*mm->futex_ref);
1686
+
return true;
1687
+
}
1688
+
1689
+
return atomic_long_inc_not_zero(&mm->futex_atomic);
1690
+
}
1691
+
1692
+
static bool futex_ref_put(struct futex_private_hash *fph)
1693
+
{
1694
+
struct mm_struct *mm = fph->mm;
1695
+
1696
+
guard(rcu)();
1697
+
1698
+
if (smp_load_acquire(&fph->state) == FR_PERCPU) {
1699
+
this_cpu_dec(*mm->futex_ref);
1700
+
return false;
1701
+
}
1702
+
1703
+
return atomic_long_dec_and_test(&mm->futex_atomic);
1704
+
}
1705
+
1706
+
static bool futex_ref_is_dead(struct futex_private_hash *fph)
1707
+
{
1708
+
struct mm_struct *mm = fph->mm;
1709
+
1710
+
guard(rcu)();
1711
+
1712
+
if (smp_load_acquire(&fph->state) == FR_PERCPU)
1713
+
return false;
1714
+
1715
+
return atomic_long_read(&mm->futex_atomic) == 0;
1716
+
}
1717
+
1718
+
int futex_mm_init(struct mm_struct *mm)
1719
+
{
1720
+
mutex_init(&mm->futex_hash_lock);
1721
+
RCU_INIT_POINTER(mm->futex_phash, NULL);
1722
+
mm->futex_phash_new = NULL;
1723
+
/* futex-ref */
1724
+
atomic_long_set(&mm->futex_atomic, 0);
1725
+
mm->futex_batches = get_state_synchronize_rcu();
1726
+
mm->futex_ref = alloc_percpu(unsigned int);
1727
+
if (!mm->futex_ref)
1728
+
return -ENOMEM;
1729
+
this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */
1730
+
return 0;
1731
+
}
1732
+
1531
1733
void futex_hash_free(struct mm_struct *mm)
1532
1734
{
1533
1735
struct futex_private_hash *fph;
1534
1736
1737
+
free_percpu(mm->futex_ref);
1535
1738
kvfree(mm->futex_phash_new);
1536
1739
fph = rcu_dereference_raw(mm->futex_phash);
1537
-
if (fph) {
1538
-
WARN_ON_ONCE(rcuref_read(&fph->users) > 1);
1740
+
if (fph)
1539
1741
kvfree(fph);
1540
-
}
1541
1742
}
1542
1743
1543
1744
static bool futex_pivot_pending(struct mm_struct *mm)
···
1752
1549
return true;
1753
1550
1754
1551
fph = rcu_dereference(mm->futex_phash);
1755
-
return rcuref_is_dead(&fph->users);
1552
+
return futex_ref_is_dead(fph);
1756
1553
}
1757
1554
1758
1555
static bool futex_hash_less(struct futex_private_hash *a,
···
1794
1591
*/
1795
1592
scoped_guard(rcu) {
1796
1593
fph = rcu_dereference(mm->futex_phash);
1797
-
if (fph && (!fph->hash_mask || fph->immutable)) {
1594
+
if (fph && !fph->hash_mask) {
1798
1595
if (custom)
1799
1596
return -EBUSY;
1800
1597
return 0;
1801
1598
}
1802
1599
}
1803
1600
1804
-
fph = kvzalloc(struct_size(fph, queues, hash_slots), GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
1601
+
fph = kvzalloc(struct_size(fph, queues, hash_slots),
1602
+
GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
1805
1603
if (!fph)
1806
1604
return -ENOMEM;
1807
1605
1808
-
rcuref_init(&fph->users, 1);
1809
1606
fph->hash_mask = hash_slots ? hash_slots - 1 : 0;
1810
1607
fph->custom = custom;
1811
-
fph->immutable = !!(flags & FH_IMMUTABLE);
1812
1608
fph->mm = mm;
1813
1609
1814
1610
for (i = 0; i < hash_slots; i++)
···
1831
1629
mm->futex_phash_new = NULL;
1832
1630
1833
1631
if (fph) {
1834
-
if (cur && (!cur->hash_mask || cur->immutable)) {
1632
+
if (cur && !cur->hash_mask) {
1835
1633
/*
1836
1634
* If two threads simultaneously request the global
1837
1635
* hash then the first one performs the switch,
···
1847
1645
* allocated a replacement hash, drop the initial
1848
1646
* reference on the existing hash.
1849
1647
*/
1850
-
futex_private_hash_put(cur);
1648
+
futex_ref_drop(cur);
1851
1649
}
1852
1650
1853
1651
if (new) {
···
1924
1722
return 0;
1925
1723
}
1926
1724
1927
-
static int futex_hash_get_immutable(void)
1928
-
{
1929
-
struct futex_private_hash *fph;
1930
-
1931
-
guard(rcu)();
1932
-
fph = rcu_dereference(current->mm->futex_phash);
1933
-
if (fph && fph->immutable)
1934
-
return 1;
1935
-
if (fph && !fph->hash_mask)
1936
-
return 1;
1937
-
return 0;
1938
-
}
1939
-
1940
1725
#else
1941
1726
1942
1727
static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
···
1936
1747
return 0;
1937
1748
}
1938
1749
1939
-
static int futex_hash_get_immutable(void)
1940
-
{
1941
-
return 0;
1942
-
}
1943
1750
#endif
1944
1751
1945
1752
int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
···
1945
1760
1946
1761
switch (arg2) {
1947
1762
case PR_FUTEX_HASH_SET_SLOTS:
1948
-
if (arg4 & ~FH_FLAG_IMMUTABLE)
1763
+
if (arg4)
1949
1764
return -EINVAL;
1950
-
if (arg4 & FH_FLAG_IMMUTABLE)
1951
-
flags |= FH_IMMUTABLE;
1952
1765
ret = futex_hash_allocate(arg3, flags);
1953
1766
break;
1954
1767
1955
1768
case PR_FUTEX_HASH_GET_SLOTS:
1956
1769
ret = futex_hash_get_slots();
1957
-
break;
1958
-
1959
-
case PR_FUTEX_HASH_GET_IMMUTABLE:
1960
-
ret = futex_hash_get_immutable();
1961
1770
break;
1962
1771
1963
1772
default:
-2
kernel/futex/futex.h
-2
kernel/futex/futex.h
···
228
228
extern void futex_hash_put(struct futex_hash_bucket *hb);
229
229
230
230
extern struct futex_private_hash *futex_private_hash(void);
231
-
extern bool futex_private_hash_get(struct futex_private_hash *fph);
232
231
extern void futex_private_hash_put(struct futex_private_hash *fph);
233
232
234
233
#else /* !CONFIG_FUTEX_PRIVATE_HASH */
235
234
static inline void futex_hash_get(struct futex_hash_bucket *hb) { }
236
235
static inline void futex_hash_put(struct futex_hash_bucket *hb) { }
237
236
static inline struct futex_private_hash *futex_private_hash(void) { return NULL; }
238
-
static inline bool futex_private_hash_get(void) { return false; }
239
237
static inline void futex_private_hash_put(struct futex_private_hash *fph) { }
240
238
#endif
241
239
-2
tools/include/uapi/linux/prctl.h
-2
tools/include/uapi/linux/prctl.h
-1
tools/perf/bench/futex-hash.c
-1
tools/perf/bench/futex-hash.c
···
56
56
57
57
static const struct option options[] = {
58
58
OPT_INTEGER( 'b', "buckets", ¶ms.nbuckets, "Specify amount of hash buckets"),
59
-
OPT_BOOLEAN( 'I', "immutable", ¶ms.buckets_immutable, "Make the hash buckets immutable"),
60
59
OPT_UINTEGER('t', "threads", ¶ms.nthreads, "Specify amount of threads"),
61
60
OPT_UINTEGER('r', "runtime", ¶ms.runtime, "Specify runtime (in seconds)"),
62
61
OPT_UINTEGER('f', "futexes", ¶ms.nfutexes, "Specify amount of futexes per threads"),
-1
tools/perf/bench/futex-lock-pi.c
-1
tools/perf/bench/futex-lock-pi.c
···
47
47
48
48
static const struct option options[] = {
49
49
OPT_INTEGER( 'b', "buckets", ¶ms.nbuckets, "Specify amount of hash buckets"),
50
-
OPT_BOOLEAN( 'I', "immutable", ¶ms.buckets_immutable, "Make the hash buckets immutable"),
51
50
OPT_UINTEGER('t', "threads", ¶ms.nthreads, "Specify amount of threads"),
52
51
OPT_UINTEGER('r', "runtime", ¶ms.runtime, "Specify runtime (in seconds)"),
53
52
OPT_BOOLEAN( 'M', "multi", ¶ms.multi, "Use multiple futexes"),
-1
tools/perf/bench/futex-requeue.c
-1
tools/perf/bench/futex-requeue.c
···
52
52
53
53
static const struct option options[] = {
54
54
OPT_INTEGER( 'b', "buckets", ¶ms.nbuckets, "Specify amount of hash buckets"),
55
-
OPT_BOOLEAN( 'I', "immutable", ¶ms.buckets_immutable, "Make the hash buckets immutable"),
56
55
OPT_UINTEGER('t', "threads", ¶ms.nthreads, "Specify amount of threads"),
57
56
OPT_UINTEGER('q', "nrequeue", ¶ms.nrequeue, "Specify amount of threads to requeue at once"),
58
57
OPT_BOOLEAN( 's', "silent", ¶ms.silent, "Silent mode: do not display data/details"),
-1
tools/perf/bench/futex-wake-parallel.c
-1
tools/perf/bench/futex-wake-parallel.c
···
63
63
64
64
static const struct option options[] = {
65
65
OPT_INTEGER( 'b', "buckets", ¶ms.nbuckets, "Specify amount of hash buckets"),
66
-
OPT_BOOLEAN( 'I', "immutable", ¶ms.buckets_immutable, "Make the hash buckets immutable"),
67
66
OPT_UINTEGER('t', "threads", ¶ms.nthreads, "Specify amount of threads"),
68
67
OPT_UINTEGER('w', "nwakers", ¶ms.nwakes, "Specify amount of waking threads"),
69
68
OPT_BOOLEAN( 's', "silent", ¶ms.silent, "Silent mode: do not display data/details"),
-1
tools/perf/bench/futex-wake.c
-1
tools/perf/bench/futex-wake.c
···
52
52
53
53
static const struct option options[] = {
54
54
OPT_INTEGER( 'b', "buckets", ¶ms.nbuckets, "Specify amount of hash buckets"),
55
-
OPT_BOOLEAN( 'I', "immutable", ¶ms.buckets_immutable, "Make the hash buckets immutable"),
56
55
OPT_UINTEGER('t', "threads", ¶ms.nthreads, "Specify amount of threads"),
57
56
OPT_UINTEGER('w', "nwakes", ¶ms.nwakes, "Specify amount of threads to wake at once"),
58
57
OPT_BOOLEAN( 's', "silent", ¶ms.silent, "Silent mode: do not display data/details"),
+5
-16
tools/perf/bench/futex.c
+5
-16
tools/perf/bench/futex.c
···
9
9
#ifndef PR_FUTEX_HASH
10
10
#define PR_FUTEX_HASH 78
11
11
# define PR_FUTEX_HASH_SET_SLOTS 1
12
-
# define FH_FLAG_IMMUTABLE (1ULL << 0)
13
12
# define PR_FUTEX_HASH_GET_SLOTS 2
14
-
# define PR_FUTEX_HASH_GET_IMMUTABLE 3
15
13
#endif // PR_FUTEX_HASH
16
14
17
15
void futex_set_nbuckets_param(struct bench_futex_parameters *params)
18
16
{
19
-
unsigned long flags;
20
17
int ret;
21
18
22
19
if (params->nbuckets < 0)
23
20
return;
24
21
25
-
flags = params->buckets_immutable ? FH_FLAG_IMMUTABLE : 0;
26
-
ret = prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_SET_SLOTS, params->nbuckets, flags);
22
+
ret = prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_SET_SLOTS, params->nbuckets, 0);
27
23
if (ret) {
28
24
printf("Requesting %d hash buckets failed: %d/%m\n",
29
25
params->nbuckets, ret);
···
43
47
printf("Requested: %d in usage: %d\n", params->nbuckets, ret);
44
48
err(EXIT_FAILURE, "prctl(PR_FUTEX_HASH)");
45
49
}
46
-
if (params->nbuckets == 0) {
50
+
if (params->nbuckets == 0)
47
51
ret = asprintf(&futex_hash_mode, "Futex hashing: global hash");
48
-
} else {
49
-
ret = prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_IMMUTABLE);
50
-
if (ret < 0) {
51
-
printf("Can't check if the hash is immutable: %m\n");
52
-
err(EXIT_FAILURE, "prctl(PR_FUTEX_HASH)");
53
-
}
54
-
ret = asprintf(&futex_hash_mode, "Futex hashing: %d hash buckets %s",
55
-
params->nbuckets,
56
-
ret == 1 ? "(immutable)" : "");
57
-
}
52
+
else
53
+
ret = asprintf(&futex_hash_mode, "Futex hashing: %d hash buckets",
54
+
params->nbuckets);
58
55
} else {
59
56
if (ret <= 0) {
60
57
ret = asprintf(&futex_hash_mode, "Futex hashing: global hash");
-1
tools/perf/bench/futex.h
-1
tools/perf/bench/futex.h
-2
tools/perf/trace/beauty/include/uapi/linux/prctl.h
-2
tools/perf/trace/beauty/include/uapi/linux/prctl.h
+63
-50
tools/testing/selftests/futex/functional/futex_priv_hash.c
+63
-50
tools/testing/selftests/futex/functional/futex_priv_hash.c
···
26
26
#ifndef PR_FUTEX_HASH
27
27
#define PR_FUTEX_HASH 78
28
28
# define PR_FUTEX_HASH_SET_SLOTS 1
29
-
# define FH_FLAG_IMMUTABLE (1ULL << 0)
30
29
# define PR_FUTEX_HASH_GET_SLOTS 2
31
-
# define PR_FUTEX_HASH_GET_IMMUTABLE 3
32
30
#endif
33
31
34
-
static int futex_hash_slots_set(unsigned int slots, int flags)
32
+
static int futex_hash_slots_set(unsigned int slots)
35
33
{
36
-
return prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_SET_SLOTS, slots, flags);
34
+
return prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_SET_SLOTS, slots, 0);
37
35
}
38
36
39
37
static int futex_hash_slots_get(void)
···
39
41
return prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_SLOTS);
40
42
}
41
43
42
-
static int futex_hash_immutable_get(void)
43
-
{
44
-
return prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_IMMUTABLE);
45
-
}
46
-
47
44
static void futex_hash_slots_set_verify(int slots)
48
45
{
49
46
int ret;
50
47
51
-
ret = futex_hash_slots_set(slots, 0);
48
+
ret = futex_hash_slots_set(slots);
52
49
if (ret != 0) {
53
50
ksft_test_result_fail("Failed to set slots to %d: %m\n", slots);
54
51
ksft_finished();
···
57
64
ksft_test_result_pass("SET and GET slots %d passed\n", slots);
58
65
}
59
66
60
-
static void futex_hash_slots_set_must_fail(int slots, int flags)
67
+
static void futex_hash_slots_set_must_fail(int slots)
61
68
{
62
69
int ret;
63
70
64
-
ret = futex_hash_slots_set(slots, flags);
65
-
ksft_test_result(ret < 0, "futex_hash_slots_set(%d, %d)\n",
66
-
slots, flags);
71
+
ret = futex_hash_slots_set(slots);
72
+
ksft_test_result(ret < 0, "futex_hash_slots_set(%d)\n",
73
+
slots);
67
74
}
68
75
69
76
static void *thread_return_fn(void *arg)
···
104
111
}
105
112
}
106
113
114
+
#define SEC_IN_NSEC 1000000000
115
+
#define MSEC_IN_NSEC 1000000
116
+
117
+
static void futex_dummy_op(void)
118
+
{
119
+
pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
120
+
struct timespec timeout;
121
+
int ret;
122
+
123
+
pthread_mutex_lock(&lock);
124
+
clock_gettime(CLOCK_REALTIME, &timeout);
125
+
timeout.tv_nsec += 100 * MSEC_IN_NSEC;
126
+
if (timeout.tv_nsec >= SEC_IN_NSEC) {
127
+
timeout.tv_nsec -= SEC_IN_NSEC;
128
+
timeout.tv_sec++;
129
+
}
130
+
ret = pthread_mutex_timedlock(&lock, &timeout);
131
+
if (ret == 0)
132
+
ksft_exit_fail_msg("Successfully locked an already locked mutex.\n");
133
+
134
+
if (ret != ETIMEDOUT)
135
+
ksft_exit_fail_msg("pthread_mutex_timedlock() did not timeout: %d.\n", ret);
136
+
}
137
+
107
138
static void usage(char *prog)
108
139
{
109
140
printf("Usage: %s\n", prog);
···
145
128
{
146
129
int futex_slots1, futex_slotsn, online_cpus;
147
130
pthread_mutexattr_t mutex_attr_pi;
148
-
int use_global_hash = 0;
149
-
int ret;
131
+
int ret, retry = 20;
150
132
int c;
151
133
152
-
while ((c = getopt(argc, argv, "cghv:")) != -1) {
134
+
while ((c = getopt(argc, argv, "chv:")) != -1) {
153
135
switch (c) {
154
136
case 'c':
155
137
log_color(1);
156
-
break;
157
-
case 'g':
158
-
use_global_hash = 1;
159
138
break;
160
139
case 'h':
161
140
usage(basename(argv[0]));
···
167
154
}
168
155
169
156
ksft_print_header();
170
-
ksft_set_plan(22);
157
+
ksft_set_plan(21);
171
158
172
159
ret = pthread_mutexattr_init(&mutex_attr_pi);
173
160
ret |= pthread_mutexattr_setprotocol(&mutex_attr_pi, PTHREAD_PRIO_INHERIT);
···
179
166
ret = futex_hash_slots_get();
180
167
if (ret != 0)
181
168
ksft_exit_fail_msg("futex_hash_slots_get() failed: %d, %m\n", ret);
182
-
183
-
ret = futex_hash_immutable_get();
184
-
if (ret != 0)
185
-
ksft_exit_fail_msg("futex_hash_immutable_get() failed: %d, %m\n", ret);
186
169
187
170
ksft_test_result_pass("Basic get slots and immutable status.\n");
188
171
ret = pthread_create(&threads[0], NULL, thread_return_fn, NULL);
···
217
208
*/
218
209
ksft_print_msg("Online CPUs: %d\n", online_cpus);
219
210
if (online_cpus > 16) {
211
+
retry_getslots:
220
212
futex_slotsn = futex_hash_slots_get();
221
213
if (futex_slotsn < 0 || futex_slots1 == futex_slotsn) {
214
+
retry--;
215
+
/*
216
+
* Auto scaling on thread creation can be slightly delayed
217
+
* because it waits for a RCU grace period twice. The new
218
+
* private hash is assigned upon the first futex operation
219
+
* after grace period.
220
+
* To cover all this for testing purposes the function
221
+
* below will acquire a lock and acquire it again with a
222
+
* 100ms timeout which must timeout. This ensures we
223
+
* sleep for 100ms and issue a futex operation.
224
+
*/
225
+
if (retry > 0) {
226
+
futex_dummy_op();
227
+
goto retry_getslots;
228
+
}
222
229
ksft_print_msg("Expected increase of hash buckets but got: %d -> %d\n",
223
230
futex_slots1, futex_slotsn);
224
231
ksft_exit_fail_msg(test_msg_auto_inc);
···
252
227
futex_hash_slots_set_verify(32);
253
228
futex_hash_slots_set_verify(16);
254
229
255
-
ret = futex_hash_slots_set(15, 0);
230
+
ret = futex_hash_slots_set(15);
256
231
ksft_test_result(ret < 0, "Use 15 slots\n");
257
232
258
233
futex_hash_slots_set_verify(2);
···
270
245
ksft_test_result(ret == 2, "No more auto-resize after manaul setting, got %d\n",
271
246
ret);
272
247
273
-
futex_hash_slots_set_must_fail(1 << 29, 0);
248
+
futex_hash_slots_set_must_fail(1 << 29);
249
+
futex_hash_slots_set_verify(4);
274
250
275
251
/*
276
-
* Once the private hash has been made immutable or global hash has been requested,
277
-
* then this requested can not be undone.
252
+
* Once the global hash has been requested, then this requested can not
253
+
* be undone.
278
254
*/
279
-
if (use_global_hash) {
280
-
ret = futex_hash_slots_set(0, 0);
281
-
ksft_test_result(ret == 0, "Global hash request\n");
282
-
} else {
283
-
ret = futex_hash_slots_set(4, FH_FLAG_IMMUTABLE);
284
-
ksft_test_result(ret == 0, "Immutable resize to 4\n");
285
-
}
255
+
ret = futex_hash_slots_set(0);
256
+
ksft_test_result(ret == 0, "Global hash request\n");
286
257
if (ret != 0)
287
258
goto out;
288
259
289
-
futex_hash_slots_set_must_fail(4, 0);
290
-
futex_hash_slots_set_must_fail(4, FH_FLAG_IMMUTABLE);
291
-
futex_hash_slots_set_must_fail(8, 0);
292
-
futex_hash_slots_set_must_fail(8, FH_FLAG_IMMUTABLE);
293
-
futex_hash_slots_set_must_fail(0, FH_FLAG_IMMUTABLE);
294
-
futex_hash_slots_set_must_fail(6, FH_FLAG_IMMUTABLE);
260
+
futex_hash_slots_set_must_fail(4);
261
+
futex_hash_slots_set_must_fail(8);
262
+
futex_hash_slots_set_must_fail(8);
263
+
futex_hash_slots_set_must_fail(0);
264
+
futex_hash_slots_set_must_fail(6);
295
265
296
266
ret = pthread_barrier_init(&barrier_main, NULL, MAX_THREADS);
297
267
if (ret != 0) {
···
297
277
join_max_threads();
298
278
299
279
ret = futex_hash_slots_get();
300
-
if (use_global_hash) {
301
-
ksft_test_result(ret == 0, "Continue to use global hash\n");
302
-
} else {
303
-
ksft_test_result(ret == 4, "Continue to use the 4 hash buckets\n");
304
-
}
305
-
306
-
ret = futex_hash_immutable_get();
307
-
ksft_test_result(ret == 1, "Hash reports to be immutable\n");
280
+
ksft_test_result(ret == 0, "Continue to use global hash\n");
308
281
309
282
out:
310
283
ksft_finished();
+11
tools/testing/selftests/futex/include/futextest.h
+11
tools/testing/selftests/futex/include/futextest.h
···
47
47
FUTEX_PRIVATE_FLAG)
48
48
#endif
49
49
50
+
/*
51
+
* SYS_futex is expected from system C library, in glibc some 32-bit
52
+
* architectures (e.g. RV32) are using 64-bit time_t, therefore it doesn't have
53
+
* SYS_futex defined but just SYS_futex_time64. Define SYS_futex as
54
+
* SYS_futex_time64 in this situation to ensure the compilation and the
55
+
* compatibility.
56
+
*/
57
+
#if !defined(SYS_futex) && defined(SYS_futex_time64)
58
+
#define SYS_futex SYS_futex_time64
59
+
#endif
60
+
50
61
/**
51
62
* futex() - SYS_futex syscall wrapper
52
63
* @uaddr: address of first futex