Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'probes-v7.0' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace
Pull kprobes updates from Masami Hiramatsu:
- Use a dedicated kernel thread to optimize the kprobes instead of
using workqueue thread. Since the kprobe optimizer waits a long time
for synchronize_rcu_task(), it can block other workers in the same
queue if it uses a workqueue.
- kprobe-events: return immediately if no new probe events are
specified on the kernel command line at boot time. This shortens
the kernel boot time.
- When a kprobe is fully removed from the kernel code, retry optimizing
another kprobe which is blocked by that kprobe.
* tag 'probes-v7.0' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace:
kprobes: Use dedicated kthread for kprobe optimizer
tracing: kprobe-event: Return directly when trace kprobes is empty
kprobes: retry blocked optprobe in do_free_cleaned_kprobes
+102
-26
+98
-26
kernel/kprobes.c
+98
-26
kernel/kprobes.c
···
32
32
#include <linux/debugfs.h>
33
33
#include <linux/sysctl.h>
34
34
#include <linux/kdebug.h>
35
+
#include <linux/kthread.h>
35
36
#include <linux/memory.h>
36
37
#include <linux/ftrace.h>
37
38
#include <linux/cpu.h>
···
41
40
#include <linux/perf_event.h>
42
41
#include <linux/execmem.h>
43
42
#include <linux/cleanup.h>
43
+
#include <linux/wait.h>
44
44
45
45
#include <asm/sections.h>
46
46
#include <asm/cacheflush.h>
···
516
514
static LIST_HEAD(unoptimizing_list);
517
515
static LIST_HEAD(freeing_list);
518
516
519
-
static void kprobe_optimizer(struct work_struct *work);
520
-
static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
517
+
static void optimize_kprobe(struct kprobe *p);
518
+
static struct task_struct *kprobe_optimizer_task;
519
+
static wait_queue_head_t kprobe_optimizer_wait;
520
+
static atomic_t optimizer_state;
521
+
enum {
522
+
OPTIMIZER_ST_IDLE = 0,
523
+
OPTIMIZER_ST_KICKED = 1,
524
+
OPTIMIZER_ST_FLUSHING = 2,
525
+
};
526
+
527
+
static DECLARE_COMPLETION(optimizer_completion);
528
+
521
529
#define OPTIMIZE_DELAY 5
522
530
523
531
/*
···
605
593
*/
606
594
continue;
607
595
}
596
+
597
+
/*
598
+
* The aggregator was holding back another probe while it sat on the
599
+
* unoptimizing/freeing lists. Now that the aggregator has been fully
600
+
* reverted we can safely retry the optimization of that sibling.
601
+
*/
602
+
603
+
struct kprobe *_p = get_optimized_kprobe(op->kp.addr);
604
+
if (unlikely(_p))
605
+
optimize_kprobe(_p);
606
+
608
607
free_aggr_kprobe(&op->kp);
609
608
}
610
609
}
611
610
612
-
/* Start optimizer after OPTIMIZE_DELAY passed */
613
-
static void kick_kprobe_optimizer(void)
614
-
{
615
-
schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
616
-
}
611
+
static void kick_kprobe_optimizer(void);
617
612
618
613
/* Kprobe jump optimizer */
619
-
static void kprobe_optimizer(struct work_struct *work)
614
+
static void kprobe_optimizer(void)
620
615
{
621
616
guard(mutex)(&kprobe_mutex);
622
617
···
654
635
do_free_cleaned_kprobes();
655
636
}
656
637
657
-
/* Step 5: Kick optimizer again if needed */
638
+
/* Step 5: Kick optimizer again if needed. But if there is a flush requested, */
639
+
if (completion_done(&optimizer_completion))
640
+
complete(&optimizer_completion);
641
+
658
642
if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
659
-
kick_kprobe_optimizer();
643
+
kick_kprobe_optimizer(); /*normal kick*/
644
+
}
645
+
646
+
static int kprobe_optimizer_thread(void *data)
647
+
{
648
+
while (!kthread_should_stop()) {
649
+
/* To avoid hung_task, wait in interruptible state. */
650
+
wait_event_interruptible(kprobe_optimizer_wait,
651
+
atomic_read(&optimizer_state) != OPTIMIZER_ST_IDLE ||
652
+
kthread_should_stop());
653
+
654
+
if (kthread_should_stop())
655
+
break;
656
+
657
+
/*
658
+
* If it was a normal kick, wait for OPTIMIZE_DELAY.
659
+
* This wait can be interrupted by a flush request.
660
+
*/
661
+
if (atomic_read(&optimizer_state) == 1)
662
+
wait_event_interruptible_timeout(
663
+
kprobe_optimizer_wait,
664
+
atomic_read(&optimizer_state) == OPTIMIZER_ST_FLUSHING ||
665
+
kthread_should_stop(),
666
+
OPTIMIZE_DELAY);
667
+
668
+
if (kthread_should_stop())
669
+
break;
670
+
671
+
atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE);
672
+
673
+
kprobe_optimizer();
674
+
}
675
+
return 0;
676
+
}
677
+
678
+
/* Start optimizer after OPTIMIZE_DELAY passed */
679
+
static void kick_kprobe_optimizer(void)
680
+
{
681
+
lockdep_assert_held(&kprobe_mutex);
682
+
if (atomic_cmpxchg(&optimizer_state,
683
+
OPTIMIZER_ST_IDLE, OPTIMIZER_ST_KICKED) == OPTIMIZER_ST_IDLE)
684
+
wake_up(&kprobe_optimizer_wait);
660
685
}
661
686
662
687
static void wait_for_kprobe_optimizer_locked(void)
···
708
645
lockdep_assert_held(&kprobe_mutex);
709
646
710
647
while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
648
+
init_completion(&optimizer_completion);
649
+
/*
650
+
* Set state to OPTIMIZER_ST_FLUSHING and wake up the thread if it's
651
+
* idle. If it's already kicked, it will see the state change.
652
+
*/
653
+
if (atomic_xchg_acquire(&optimizer_state,
654
+
OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING)
655
+
wake_up(&kprobe_optimizer_wait);
656
+
711
657
mutex_unlock(&kprobe_mutex);
712
-
713
-
/* This will also make 'optimizing_work' execute immmediately */
714
-
flush_delayed_work(&optimizing_work);
715
-
/* 'optimizing_work' might not have been queued yet, relax */
716
-
cpu_relax();
717
-
658
+
wait_for_completion(&optimizer_completion);
718
659
mutex_lock(&kprobe_mutex);
719
660
}
720
661
}
···
1069
1002
if (unlikely(_p) && reopt)
1070
1003
optimize_kprobe(_p);
1071
1004
}
1072
-
/*
1073
-
* TODO: Since unoptimization and real disarming will be done by
1074
-
* the worker thread, we can not check whether another probe are
1075
-
* unoptimized because of this probe here. It should be re-optimized
1076
-
* by the worker thread.
1077
-
*/
1078
1005
}
1079
1006
1007
+
static void __init init_optprobe(void)
1008
+
{
1009
+
#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
1010
+
/* Init 'kprobe_optinsn_slots' for allocation */
1011
+
kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
1012
+
#endif
1013
+
1014
+
init_waitqueue_head(&kprobe_optimizer_wait);
1015
+
atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE);
1016
+
kprobe_optimizer_task = kthread_run(kprobe_optimizer_thread, NULL,
1017
+
"kprobe-optimizer");
1018
+
}
1080
1019
#else /* !CONFIG_OPTPROBES */
1081
1020
1021
+
#define init_optprobe() do {} while (0)
1082
1022
#define optimize_kprobe(p) do {} while (0)
1083
1023
#define unoptimize_kprobe(p, f) do {} while (0)
1084
1024
#define kill_optimized_kprobe(p) do {} while (0)
···
2768
2694
/* By default, kprobes are armed */
2769
2695
kprobes_all_disarmed = false;
2770
2696
2771
-
#if defined(CONFIG_OPTPROBES) && defined(__ARCH_WANT_KPROBES_INSN_SLOT)
2772
-
/* Init 'kprobe_optinsn_slots' for allocation */
2773
-
kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2774
-
#endif
2697
+
/* Initialize the optimization infrastructure */
2698
+
init_optprobe();
2775
2699
2776
2700
err = arch_init_kprobes();
2777
2701
if (!err)
+4
kernel/trace/trace_kprobe.c
+4
kernel/trace/trace_kprobe.c
···
82
82
#define for_each_trace_kprobe(pos, dpos) \
83
83
for_each_dyn_event(dpos) \
84
84
if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos)))
85
+
#define trace_kprobe_list_empty() list_empty(&dyn_event_list)
85
86
86
87
static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
87
88
{
···
1982
1981
struct trace_event_file *file;
1983
1982
struct trace_kprobe *tk;
1984
1983
struct dyn_event *pos;
1984
+
1985
+
if (trace_kprobe_list_empty())
1986
+
return;
1985
1987
1986
1988
guard(mutex)(&event_mutex);
1987
1989
for_each_trace_kprobe(tk, pos) {