this repo has no description
Fix locking primitives in xtrace
+125
-31
+124
-30
src/xtrace/lock.c
+124
-30
src/xtrace/lock.c
···
1
1
#include <stdatomic.h>
2
+
#include <limits.h>
3
+
#include <stdbool.h>
2
4
3
5
#include "lock.h"
4
6
#include "simple.h"
5
7
6
8
#ifndef XTRACE_LOCK_DEBUG
7
-
#define XTRACE_LOCK_DEBUG 0
9
+
#define XTRACE_LOCK_DEBUG 1
8
10
#endif
9
11
10
12
#if XTRACE_LOCK_DEBUG
11
-
#define xtrace_lock_debug(x, ...) __simple_printf(x "\n", ## __VA_ARGS__)
13
+
#define xtrace_lock_debug_internal(x, ...) __simple_printf(x "\n", ## __VA_ARGS__)
12
14
#undef XTRACE_INLINE
13
15
#define XTRACE_INLINE
14
16
#else
15
-
#define xtrace_lock_debug(x, ...)
17
+
#define xtrace_lock_debug_internal(x, ...)
16
18
#endif
17
19
20
+
#define xtrace_lock_debug(x, ...) xtrace_lock_debug_internal("lock %p: " x, lock, ## __VA_ARGS__)
21
+
#define xtrace_once_debug(x, ...) xtrace_lock_debug_internal("once %p: " x, once, ## __VA_ARGS__)
22
+
23
+
//
24
+
// xtrace_lock
25
+
//
18
26
// simple lock implementation on top of futexes, based on https://github.com/eliben/code-for-blog/blob/master/2018/futex-basics/mutex-using-futex.cpp
27
+
//
19
28
20
29
// used to hide the use of C11 atomics from external code (e.g. C++ code, which can't use C11 atomics).
21
30
// it's *technically* undefined behavior to cast the external one to this one, but that's okay;
···
24
33
_Atomic uint32_t state;
25
34
} xtrace_lock_internal_t;
26
35
27
-
typedef struct xtrace_once_internal {
28
-
_Atomic uint8_t state;
29
-
} xtrace_once_internal_t;
30
-
31
36
enum xtrace_lock_state {
32
37
// the lock is unlocked (duh)
33
38
xtrace_lock_state_unlocked = 0,
···
39
44
xtrace_lock_state_locked_contended = 2,
40
45
};
41
46
42
-
enum xtrace_once_state {
43
-
// the once hasn't been run yet
44
-
xtrace_once_state_untriggered = 0,
45
-
46
-
// the once has already been run at least once
47
-
xtrace_once_state_triggered = 1,
47
+
XTRACE_INLINE
48
+
uint32_t cmpxchg_wrapper_u32(_Atomic uint32_t* atom, uint32_t expected, uint32_t desired) {
49
+
atomic_compare_exchange_strong_explicit(atom, &expected, desired, memory_order_acq_rel, memory_order_acquire);
50
+
return expected;
48
51
};
49
52
50
53
void xtrace_lock_lock(xtrace_lock_t* _lock) {
51
54
xtrace_lock_internal_t* lock = (xtrace_lock_internal_t*)_lock;
52
55
53
-
xtrace_lock_debug("lock %p: locking", lock);
56
+
xtrace_lock_debug("locking");
54
57
55
58
// we do the initial exchange using `xtrace_lock_state_locked_uncontended` because
56
59
// if it was previously unlocked, it is now locked but nobody is waiting for it
57
-
uint32_t prev = atomic_exchange(&lock->state, xtrace_lock_state_locked_uncontended);
60
+
uint32_t prev = cmpxchg_wrapper_u32(&lock->state, xtrace_lock_state_unlocked, xtrace_lock_state_locked_uncontended);
58
61
59
-
xtrace_lock_debug("lock %p: previous state was %u", lock, prev);
62
+
xtrace_lock_debug("previous state was %u", prev);
60
63
61
64
// if it was not unlocked, we need to do some waiting
62
65
if (prev != xtrace_lock_state_unlocked) {
63
66
do {
64
-
// we can skip the atomic_exchange if the lock was already contended
65
-
// also, when we do the atomic_exchange, we need to make sure it's still locked
67
+
// we can skip the cmpxchg if the lock was already contended
68
+
// also, when we do the cmpxchg, we need to make sure it's still locked
66
69
// we use `xtrace_lock_state_locked_contended` because it was either uncontended (but it is now) or it was already contended
67
-
if (prev == xtrace_lock_state_locked_contended || atomic_exchange(&lock->state, xtrace_lock_state_locked_contended) != xtrace_lock_state_unlocked) {
68
-
xtrace_lock_debug("lock %p: going to wait", lock);
70
+
if (prev == xtrace_lock_state_locked_contended || cmpxchg_wrapper_u32(&lock->state, xtrace_lock_state_locked_uncontended, xtrace_lock_state_locked_contended) != xtrace_lock_state_unlocked) {
71
+
xtrace_lock_debug("going to wait");
69
72
// do the actual sleeping
70
73
// we expect `xtrace_lock_state_locked_contended` because we don't want to sleep if it's not contended
71
74
__linux_futex(&lock->state, FUTEX_WAIT, xtrace_lock_state_locked_contended, NULL, 0, 0);
72
-
xtrace_lock_debug("lock %p: awoken", lock);
75
+
xtrace_lock_debug("awoken");
73
76
}
74
77
75
78
// loop as long as the lock was not unlocked
76
79
// we use `xtrace_lock_state_locked_contended` for the same reason as before
77
-
} while ((prev = atomic_exchange(&lock->state, xtrace_lock_state_locked_contended)) != xtrace_lock_state_unlocked);
78
-
xtrace_lock_debug("lock %p: lock acquired after sleeping", lock);
80
+
} while ((prev = cmpxchg_wrapper_u32(&lock->state, xtrace_lock_state_unlocked, xtrace_lock_state_locked_contended)) != xtrace_lock_state_unlocked);
81
+
xtrace_lock_debug("lock acquired after sleeping");
79
82
}
80
83
81
-
xtrace_lock_debug("lock %p: lock acquired", lock);
84
+
xtrace_lock_debug("lock acquired");
82
85
};
83
86
84
87
void xtrace_lock_unlock(xtrace_lock_t* _lock) {
85
88
xtrace_lock_internal_t* lock = (xtrace_lock_internal_t*)_lock;
86
89
87
-
xtrace_lock_debug("lock %p: unlocking", lock);
90
+
xtrace_lock_debug("unlocking");
88
91
89
-
uint32_t prev = atomic_exchange(&lock->state, xtrace_lock_state_unlocked);
92
+
uint32_t prev = atomic_exchange_explicit(&lock->state, xtrace_lock_state_unlocked, memory_order_acq_rel);
90
93
91
-
xtrace_lock_debug("lock %p: previous state was %u", prev);
94
+
xtrace_lock_debug("previous state was %u", prev);
92
95
93
96
// if it was previously contended, then we need to wake someone up
94
97
if (prev == xtrace_lock_state_locked_contended) {
95
-
xtrace_lock_debug("lock %p: waking someone up", lock);
98
+
xtrace_lock_debug("waking someone up");
96
99
__linux_futex(&lock->state, FUTEX_WAKE, 1, NULL, 0, 0);
97
100
}
98
101
};
99
102
103
+
//
104
+
// xtrace_once
105
+
//
106
+
// see https://github.com/bugaevc/serenity/blob/1fc3f2ba84d9c75c6c30e62014dabe4fcd62aae1/Libraries/LibPthread/pthread_once.cpp
107
+
//
108
+
109
+
typedef struct xtrace_once_internal {
110
+
_Atomic uint32_t state;
111
+
} xtrace_once_internal_t;
112
+
113
+
enum xtrace_once_state {
114
+
// the once hasn't been triggered yet
115
+
xtrace_once_state_untriggered = 0,
116
+
117
+
// the once has been triggered, but no one is waiting for it
118
+
xtrace_once_state_triggered_uncontended = 1,
119
+
120
+
// the once has been triggered and there are waiters
121
+
xtrace_once_state_triggered_contended = 2,
122
+
123
+
// the once has been triggered and has completed execution
124
+
xtrace_once_state_completed = 3,
125
+
};
126
+
100
127
void xtrace_once(xtrace_once_t* _once, xtrace_once_callback callback) {
101
-
xtrace_lock_internal_t* once = (xtrace_lock_internal_t*)_once;
102
-
if (atomic_exchange(&once->state, xtrace_once_state_triggered) == xtrace_once_state_untriggered) {
128
+
xtrace_once_internal_t* once = (xtrace_once_internal_t*)_once;
129
+
130
+
xtrace_once_debug("evaluating...");
131
+
132
+
uint32_t prev = xtrace_once_state_untriggered;
133
+
bool exchanged = atomic_compare_exchange_strong_explicit(&once->state, &prev, xtrace_once_state_triggered_uncontended, memory_order_acq_rel, memory_order_acquire);
134
+
135
+
xtrace_once_debug("previous state was %u", prev);
136
+
137
+
if (exchanged) {
138
+
xtrace_once_debug("performing callback...");
139
+
140
+
// we had `xtrace_once_state_untriggered` and now we have `xtrace_once_state_triggered_uncontended`, so let's do the callback
103
141
callback();
142
+
143
+
xtrace_once_debug("callback done; updating state...");
144
+
145
+
// now let's update the state to tell everyone we're done
146
+
prev = atomic_exchange_explicit(&once->state, xtrace_once_state_completed, memory_order_acq_rel);
147
+
148
+
xtrace_once_debug("previous state was %u", prev);
149
+
150
+
switch (prev) {
151
+
// no one was waiting, so we have no one to wake up
152
+
case xtrace_once_state_triggered_uncontended: break;
153
+
case xtrace_once_state_triggered_contended: {
154
+
xtrace_once_debug("waking up all waiters...");
155
+
// otherwise, we have to wake someone up
156
+
__linux_futex(&once->state, FUTEX_WAKE, INT_MAX, NULL, 0, 0);
157
+
} break;
158
+
}
159
+
160
+
return;
161
+
}
162
+
163
+
xtrace_once_debug("someone else is performing the callback");
164
+
165
+
while (true) {
166
+
switch (prev) {
167
+
case xtrace_once_state_completed: {
168
+
// we're done
169
+
xtrace_once_debug("callback completed by someone else; returning...");
170
+
return;
171
+
}; // not reached
172
+
case xtrace_once_state_triggered_uncontended: {
173
+
// somebody is already perfoming the callback;
174
+
// now we're waiting, so let's update the state for that
175
+
xtrace_once_debug("someone is already performing the callback with no waiters; becoming first waiter...");
176
+
exchanged = atomic_compare_exchange_strong_explicit(&once->state, &prev, xtrace_once_state_triggered_contended, memory_order_acq_rel, memory_order_acquire);
177
+
if (!exchanged) {
178
+
xtrace_once_debug("state changed from uncontended; re-evaluating...");
179
+
// something changed, let's re-evaluate
180
+
continue;
181
+
}
182
+
prev = xtrace_once_state_triggered_contended;
183
+
}; // fall through
184
+
case xtrace_once_state_triggered_contended: {
185
+
// somebody is already performing the callback and there are already waiters;
186
+
// let's wait
187
+
xtrace_once_debug("someone is already performing the callback with waiters; going to wait...");
188
+
__linux_futex(&once->state, FUTEX_WAIT, prev, NULL, 0, 0);
189
+
190
+
xtrace_once_debug("woken up");
191
+
192
+
// we got woken up, but that may have been spurious;
193
+
// let's loop again to re-evaluate
194
+
prev = atomic_load_explicit(&once->state, memory_order_acquire);
195
+
xtrace_once_debug("previous state was %u; going to re-evaluate...", prev);
196
+
} break;
197
+
}
104
198
}
105
199
};