Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
tracing/user_events: Fixup enable faults asyncly
When events are enabled within the various tracing facilities, such as
ftrace/perf, the event_mutex is held. As events are enabled pages are
accessed. We do not want page faults to occur under this lock. Instead
queue the fault to a workqueue to be handled in a process context safe
way without the lock.
The enable address is marked faulting while the async fault-in occurs.
This ensures that we don't attempt to fault-in more than is necessary.
Once the page has been faulted in, an address write is re-attempted.
If the page couldn't fault-in, then we wait until the next time the
event is enabled to prevent any potential infinite loops.
Link: https://lkml.kernel.org/r/20230328235219.203-5-beaub@linux.microsoft.com
Signed-off-by: Beau Belgrave <beaub@linux.microsoft.com>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
authored by
Beau Belgrave
and committed by
Steven Rostedt (Google)
81f8fb65
72357590
+114
-6
+114
-6
kernel/trace/trace_events_user.c
+114
-6
kernel/trace/trace_events_user.c
···
99
99
/* Bits 0-5 are for the bit to update upon enable/disable (0-63 allowed) */
100
100
#define ENABLE_VAL_BIT_MASK 0x3F
101
101
102
+
/* Bit 6 is for faulting status of enablement */
103
+
#define ENABLE_VAL_FAULTING_BIT 6
104
+
102
105
/* Only duplicate the bit value */
103
106
#define ENABLE_VAL_DUP_MASK ENABLE_VAL_BIT_MASK
107
+
108
+
#define ENABLE_BITOPS(e) ((unsigned long *)&(e)->values)
109
+
110
+
/* Used for asynchronous faulting in of pages */
111
+
struct user_event_enabler_fault {
112
+
struct work_struct work;
113
+
struct user_event_mm *mm;
114
+
struct user_event_enabler *enabler;
115
+
};
116
+
117
+
static struct kmem_cache *fault_cache;
104
118
105
119
/* Global list of memory descriptors using user_events */
106
120
static LIST_HEAD(user_event_mms);
···
277
263
}
278
264
279
265
static int user_event_enabler_write(struct user_event_mm *mm,
280
-
struct user_event_enabler *enabler)
266
+
struct user_event_enabler *enabler,
267
+
bool fixup_fault);
268
+
269
+
static void user_event_enabler_fault_fixup(struct work_struct *work)
270
+
{
271
+
struct user_event_enabler_fault *fault = container_of(
272
+
work, struct user_event_enabler_fault, work);
273
+
struct user_event_enabler *enabler = fault->enabler;
274
+
struct user_event_mm *mm = fault->mm;
275
+
unsigned long uaddr = enabler->addr;
276
+
int ret;
277
+
278
+
ret = user_event_mm_fault_in(mm, uaddr);
279
+
280
+
if (ret && ret != -ENOENT) {
281
+
struct user_event *user = enabler->event;
282
+
283
+
pr_warn("user_events: Fault for mm: 0x%pK @ 0x%llx event: %s\n",
284
+
mm->mm, (unsigned long long)uaddr, EVENT_NAME(user));
285
+
}
286
+
287
+
/* Prevent state changes from racing */
288
+
mutex_lock(&event_mutex);
289
+
290
+
/*
291
+
* If we managed to get the page, re-issue the write. We do not
292
+
* want to get into a possible infinite loop, which is why we only
293
+
* attempt again directly if the page came in. If we couldn't get
294
+
* the page here, then we will try again the next time the event is
295
+
* enabled/disabled.
296
+
*/
297
+
clear_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
298
+
299
+
if (!ret) {
300
+
mmap_read_lock(mm->mm);
301
+
user_event_enabler_write(mm, enabler, true);
302
+
mmap_read_unlock(mm->mm);
303
+
}
304
+
305
+
mutex_unlock(&event_mutex);
306
+
307
+
/* In all cases we no longer need the mm or fault */
308
+
user_event_mm_put(mm);
309
+
kmem_cache_free(fault_cache, fault);
310
+
}
311
+
312
+
static bool user_event_enabler_queue_fault(struct user_event_mm *mm,
313
+
struct user_event_enabler *enabler)
314
+
{
315
+
struct user_event_enabler_fault *fault;
316
+
317
+
fault = kmem_cache_zalloc(fault_cache, GFP_NOWAIT | __GFP_NOWARN);
318
+
319
+
if (!fault)
320
+
return false;
321
+
322
+
INIT_WORK(&fault->work, user_event_enabler_fault_fixup);
323
+
fault->mm = user_event_mm_get(mm);
324
+
fault->enabler = enabler;
325
+
326
+
/* Don't try to queue in again while we have a pending fault */
327
+
set_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
328
+
329
+
if (!schedule_work(&fault->work)) {
330
+
/* Allow another attempt later */
331
+
clear_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
332
+
333
+
user_event_mm_put(mm);
334
+
kmem_cache_free(fault_cache, fault);
335
+
336
+
return false;
337
+
}
338
+
339
+
return true;
340
+
}
341
+
342
+
static int user_event_enabler_write(struct user_event_mm *mm,
343
+
struct user_event_enabler *enabler,
344
+
bool fixup_fault)
281
345
{
282
346
unsigned long uaddr = enabler->addr;
283
347
unsigned long *ptr;
···
370
278
if (refcount_read(&mm->tasks) == 0)
371
279
return -ENOENT;
372
280
281
+
if (unlikely(test_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler))))
282
+
return -EBUSY;
283
+
373
284
ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT,
374
285
&page, NULL, NULL);
375
286
376
-
if (ret <= 0) {
377
-
pr_warn("user_events: Enable write failed\n");
287
+
if (unlikely(ret <= 0)) {
288
+
if (!fixup_fault)
289
+
return -EFAULT;
290
+
291
+
if (!user_event_enabler_queue_fault(mm, enabler))
292
+
pr_warn("user_events: Unable to queue fault handler\n");
293
+
378
294
return -EFAULT;
379
295
}
380
296
···
414
314
415
315
list_for_each_entry_rcu(enabler, &mm->enablers, link)
416
316
if (enabler->event == user)
417
-
user_event_enabler_write(mm, enabler);
317
+
user_event_enabler_write(mm, enabler, true);
418
318
419
319
rcu_read_unlock();
420
320
mmap_read_unlock(mm->mm);
···
662
562
663
563
/* Attempt to reflect the current state within the process */
664
564
mmap_read_lock(user_mm->mm);
665
-
*write_result = user_event_enabler_write(user_mm, enabler);
565
+
*write_result = user_event_enabler_write(user_mm, enabler, false);
666
566
mmap_read_unlock(user_mm->mm);
667
567
668
568
/*
···
2301
2201
{
2302
2202
int ret;
2303
2203
2204
+
fault_cache = KMEM_CACHE(user_event_enabler_fault, 0);
2205
+
2206
+
if (!fault_cache)
2207
+
return -ENOMEM;
2208
+
2304
2209
init_group = user_event_group_create(&init_user_ns);
2305
2210
2306
-
if (!init_group)
2211
+
if (!init_group) {
2212
+
kmem_cache_destroy(fault_cache);
2307
2213
return -ENOMEM;
2214
+
}
2308
2215
2309
2216
ret = create_user_tracefs();
2310
2217
2311
2218
if (ret) {
2312
2219
pr_warn("user_events could not register with tracefs\n");
2313
2220
user_event_group_destroy(init_group);
2221
+
kmem_cache_destroy(fault_cache);
2314
2222
init_group = NULL;
2315
2223
return ret;
2316
2224
}