Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
kfence: Enable context analysis
Enable context analysis for the KFENCE subsystem.
Notable, kfence_handle_page_fault() required minor restructure, which
also fixed a subtle race; arguably that function is more readable now.
Signed-off-by: Marco Elver <elver@google.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20251219154418.3592607-29-elver@google.com
authored by
Marco Elver
and committed by
Peter Zijlstra
0f5d7648
48eb4b9a
+25
-15
+2
mm/kfence/Makefile
+2
mm/kfence/Makefile
+13
-7
mm/kfence/core.c
+13
-7
mm/kfence/core.c
···
133
133
static struct kfence_metadata *kfence_metadata_init __read_mostly;
134
134
135
135
/* Freelist with available objects. */
136
-
static struct list_head kfence_freelist = LIST_HEAD_INIT(kfence_freelist);
137
-
static DEFINE_RAW_SPINLOCK(kfence_freelist_lock); /* Lock protecting freelist. */
136
+
DEFINE_RAW_SPINLOCK(kfence_freelist_lock); /* Lock protecting freelist. */
137
+
static struct list_head kfence_freelist __guarded_by(&kfence_freelist_lock) = LIST_HEAD_INIT(kfence_freelist);
138
138
139
139
/*
140
140
* The static key to set up a KFENCE allocation; or if static keys are not used
···
254
254
}
255
255
256
256
static inline unsigned long metadata_to_pageaddr(const struct kfence_metadata *meta)
257
+
__must_hold(&meta->lock)
257
258
{
258
259
unsigned long offset = (meta - kfence_metadata + 1) * PAGE_SIZE * 2;
259
260
unsigned long pageaddr = (unsigned long)&__kfence_pool[offset];
···
290
289
static noinline void
291
290
metadata_update_state(struct kfence_metadata *meta, enum kfence_object_state next,
292
291
unsigned long *stack_entries, size_t num_stack_entries)
292
+
__must_hold(&meta->lock)
293
293
{
294
294
struct kfence_track *track =
295
295
next == KFENCE_OBJECT_ALLOCATED ? &meta->alloc_track : &meta->free_track;
···
488
486
alloc_covered_add(alloc_stack_hash, 1);
489
487
490
488
/* Set required slab fields. */
491
-
slab = virt_to_slab((void *)meta->addr);
489
+
slab = virt_to_slab(addr);
492
490
slab->slab_cache = cache;
493
491
slab->objects = 1;
494
492
···
517
515
static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool zombie)
518
516
{
519
517
struct kcsan_scoped_access assert_page_exclusive;
518
+
u32 alloc_stack_hash;
520
519
unsigned long flags;
521
520
bool init;
522
521
···
550
547
/* Mark the object as freed. */
551
548
metadata_update_state(meta, KFENCE_OBJECT_FREED, NULL, 0);
552
549
init = slab_want_init_on_free(meta->cache);
550
+
alloc_stack_hash = meta->alloc_stack_hash;
553
551
raw_spin_unlock_irqrestore(&meta->lock, flags);
554
552
555
-
alloc_covered_add(meta->alloc_stack_hash, -1);
553
+
alloc_covered_add(alloc_stack_hash, -1);
556
554
557
555
/* Check canary bytes for memory corruption. */
558
556
check_canary(meta);
···
598
594
* which partial initialization succeeded.
599
595
*/
600
596
static unsigned long kfence_init_pool(void)
597
+
__context_unsafe(/* constructor */)
601
598
{
602
599
unsigned long addr, start_pfn;
603
600
int i;
···
1225
1220
{
1226
1221
const int page_index = (addr - (unsigned long)__kfence_pool) / PAGE_SIZE;
1227
1222
struct kfence_metadata *to_report = NULL;
1223
+
unsigned long unprotected_page = 0;
1228
1224
enum kfence_error_type error_type;
1229
1225
unsigned long flags;
1230
1226
···
1259
1253
if (!to_report)
1260
1254
goto out;
1261
1255
1262
-
raw_spin_lock_irqsave(&to_report->lock, flags);
1263
-
to_report->unprotected_page = addr;
1264
1256
error_type = KFENCE_ERROR_OOB;
1257
+
unprotected_page = addr;
1265
1258
1266
1259
/*
1267
1260
* If the object was freed before we took the look we can still
···
1272
1267
if (!to_report)
1273
1268
goto out;
1274
1269
1275
-
raw_spin_lock_irqsave(&to_report->lock, flags);
1276
1270
error_type = KFENCE_ERROR_UAF;
1277
1271
/*
1278
1272
* We may race with __kfence_alloc(), and it is possible that a
···
1283
1279
1284
1280
out:
1285
1281
if (to_report) {
1282
+
raw_spin_lock_irqsave(&to_report->lock, flags);
1283
+
to_report->unprotected_page = unprotected_page;
1286
1284
kfence_report_error(addr, is_write, regs, to_report, error_type);
1287
1285
raw_spin_unlock_irqrestore(&to_report->lock, flags);
1288
1286
} else {
+8
-6
mm/kfence/kfence.h
+8
-6
mm/kfence/kfence.h
···
34
34
/* Maximum stack depth for reports. */
35
35
#define KFENCE_STACK_DEPTH 64
36
36
37
+
extern raw_spinlock_t kfence_freelist_lock;
38
+
37
39
/* KFENCE object states. */
38
40
enum kfence_object_state {
39
41
KFENCE_OBJECT_UNUSED, /* Object is unused. */
···
55
53
56
54
/* KFENCE metadata per guarded allocation. */
57
55
struct kfence_metadata {
58
-
struct list_head list; /* Freelist node; access under kfence_freelist_lock. */
56
+
struct list_head list __guarded_by(&kfence_freelist_lock); /* Freelist node. */
59
57
struct rcu_head rcu_head; /* For delayed freeing. */
60
58
61
59
/*
···
93
91
* In case of an invalid access, the page that was unprotected; we
94
92
* optimistically only store one address.
95
93
*/
96
-
unsigned long unprotected_page;
94
+
unsigned long unprotected_page __guarded_by(&lock);
97
95
98
96
/* Allocation and free stack information. */
99
-
struct kfence_track alloc_track;
100
-
struct kfence_track free_track;
97
+
struct kfence_track alloc_track __guarded_by(&lock);
98
+
struct kfence_track free_track __guarded_by(&lock);
101
99
/* For updating alloc_covered on frees. */
102
-
u32 alloc_stack_hash;
100
+
u32 alloc_stack_hash __guarded_by(&lock);
103
101
#ifdef CONFIG_MEMCG
104
102
struct slabobj_ext obj_exts;
105
103
#endif
···
143
141
void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
144
142
const struct kfence_metadata *meta, enum kfence_error_type type);
145
143
146
-
void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta);
144
+
void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta) __must_hold(&meta->lock);
147
145
148
146
#endif /* MM_KFENCE_KFENCE_H */
+2
-2
mm/kfence/report.c
+2
-2
mm/kfence/report.c
···
106
106
107
107
static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta,
108
108
bool show_alloc)
109
+
__must_hold(&meta->lock)
109
110
{
110
111
const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track;
111
112
u64 ts_sec = track->ts_nsec;
···
208
207
if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta))
209
208
return;
210
209
211
-
if (meta)
212
-
lockdep_assert_held(&meta->lock);
213
210
/*
214
211
* Because we may generate reports in printk-unfriendly parts of the
215
212
* kernel, such as scheduler code, the use of printk() could deadlock.
···
262
263
stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0);
263
264
264
265
if (meta) {
266
+
lockdep_assert_held(&meta->lock);
265
267
pr_err("\n");
266
268
kfence_print_object(NULL, meta);
267
269
}