Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'perf-core-for-bpf-next' from tip tree
Stable tag for bpf-next's uprobe work.
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
+760
-304
+1
arch/Kconfig
+1
arch/Kconfig
+8
-2
arch/x86/events/amd/core.c
+8
-2
arch/x86/events/amd/core.c
···
943
943
static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
944
944
{
945
945
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
946
+
static atomic64_t status_warned = ATOMIC64_INIT(0);
947
+
u64 reserved, status, mask, new_bits, prev_bits;
946
948
struct perf_sample_data data;
947
949
struct hw_perf_event *hwc;
948
950
struct perf_event *event;
949
951
int handled = 0, idx;
950
-
u64 reserved, status, mask;
951
952
bool pmu_enabled;
952
953
953
954
/*
···
1013
1012
* the corresponding PMCs are expected to be inactive according to the
1014
1013
* active_mask
1015
1014
*/
1016
-
WARN_ON(status > 0);
1015
+
if (status > 0) {
1016
+
prev_bits = atomic64_fetch_or(status, &status_warned);
1017
+
// A new bit was set for the very first time.
1018
+
new_bits = status & ~prev_bits;
1019
+
WARN(new_bits, "New overflows for inactive PMCs: %llx\n", new_bits);
1020
+
}
1017
1021
1018
1022
/* Clear overflow and freeze bits */
1019
1023
amd_pmu_ack_global_status(~status);
+121
-12
arch/x86/events/intel/core.c
+121
-12
arch/x86/events/intel/core.c
···
4599
4599
X86_CONFIG(.event=0xc0, .umask=0x01);
4600
4600
}
4601
4601
4602
+
static struct event_constraint *
4603
+
arl_h_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
4604
+
struct perf_event *event)
4605
+
{
4606
+
struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
4607
+
4608
+
if (pmu->pmu_type == hybrid_tiny)
4609
+
return cmt_get_event_constraints(cpuc, idx, event);
4610
+
4611
+
return mtl_get_event_constraints(cpuc, idx, event);
4612
+
}
4613
+
4614
+
static int arl_h_hw_config(struct perf_event *event)
4615
+
{
4616
+
struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
4617
+
4618
+
if (pmu->pmu_type == hybrid_tiny)
4619
+
return intel_pmu_hw_config(event);
4620
+
4621
+
return adl_hw_config(event);
4622
+
}
4623
+
4602
4624
/*
4603
4625
* The HSW11 requires a period larger than 100 which is the same as the BDM11.
4604
4626
* A minimum period of 128 is enforced as well for the INST_RETIRED.ALL.
···
4946
4924
4947
4925
/*
4948
4926
* This essentially just maps between the 'hybrid_cpu_type'
4949
-
* and 'hybrid_pmu_type' enums:
4927
+
* and 'hybrid_pmu_type' enums except for ARL-H processor
4928
+
* which needs to compare atom uarch native id since ARL-H
4929
+
* contains two different atom uarchs.
4950
4930
*/
4951
4931
for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
4952
4932
enum hybrid_pmu_type pmu_type = x86_pmu.hybrid_pmu[i].pmu_type;
4933
+
u32 native_id;
4953
4934
4954
-
if (cpu_type == HYBRID_INTEL_CORE &&
4955
-
pmu_type == hybrid_big)
4935
+
if (cpu_type == HYBRID_INTEL_CORE && pmu_type == hybrid_big)
4956
4936
return &x86_pmu.hybrid_pmu[i];
4957
-
if (cpu_type == HYBRID_INTEL_ATOM &&
4958
-
pmu_type == hybrid_small)
4959
-
return &x86_pmu.hybrid_pmu[i];
4937
+
if (cpu_type == HYBRID_INTEL_ATOM) {
4938
+
if (x86_pmu.num_hybrid_pmus == 2 && pmu_type == hybrid_small)
4939
+
return &x86_pmu.hybrid_pmu[i];
4940
+
4941
+
native_id = get_this_hybrid_cpu_native_id();
4942
+
if (native_id == skt_native_id && pmu_type == hybrid_small)
4943
+
return &x86_pmu.hybrid_pmu[i];
4944
+
if (native_id == cmt_native_id && pmu_type == hybrid_tiny)
4945
+
return &x86_pmu.hybrid_pmu[i];
4946
+
}
4960
4947
}
4961
4948
4962
4949
return NULL;
···
5996
5965
NULL
5997
5966
};
5998
5967
5968
+
/* The event string must be in PMU IDX order. */
5969
+
EVENT_ATTR_STR_HYBRID(topdown-retiring,
5970
+
td_retiring_arl_h,
5971
+
"event=0xc2,umask=0x02;event=0x00,umask=0x80;event=0xc2,umask=0x0",
5972
+
hybrid_big_small_tiny);
5973
+
EVENT_ATTR_STR_HYBRID(topdown-bad-spec,
5974
+
td_bad_spec_arl_h,
5975
+
"event=0x73,umask=0x0;event=0x00,umask=0x81;event=0x73,umask=0x0",
5976
+
hybrid_big_small_tiny);
5977
+
EVENT_ATTR_STR_HYBRID(topdown-fe-bound,
5978
+
td_fe_bound_arl_h,
5979
+
"event=0x9c,umask=0x01;event=0x00,umask=0x82;event=0x71,umask=0x0",
5980
+
hybrid_big_small_tiny);
5981
+
EVENT_ATTR_STR_HYBRID(topdown-be-bound,
5982
+
td_be_bound_arl_h,
5983
+
"event=0xa4,umask=0x02;event=0x00,umask=0x83;event=0x74,umask=0x0",
5984
+
hybrid_big_small_tiny);
5985
+
5986
+
static struct attribute *arl_h_hybrid_events_attrs[] = {
5987
+
EVENT_PTR(slots_adl),
5988
+
EVENT_PTR(td_retiring_arl_h),
5989
+
EVENT_PTR(td_bad_spec_arl_h),
5990
+
EVENT_PTR(td_fe_bound_arl_h),
5991
+
EVENT_PTR(td_be_bound_arl_h),
5992
+
EVENT_PTR(td_heavy_ops_adl),
5993
+
EVENT_PTR(td_br_mis_adl),
5994
+
EVENT_PTR(td_fetch_lat_adl),
5995
+
EVENT_PTR(td_mem_bound_adl),
5996
+
NULL,
5997
+
};
5998
+
5999
5999
/* Must be in IDX order */
6000
6000
EVENT_ATTR_STR_HYBRID(mem-loads, mem_ld_adl, "event=0xd0,umask=0x5,ldlat=3;event=0xcd,umask=0x1,ldlat=3", hybrid_big_small);
6001
6001
EVENT_ATTR_STR_HYBRID(mem-stores, mem_st_adl, "event=0xd0,umask=0x6;event=0xcd,umask=0x2", hybrid_big_small);
···
6043
5981
EVENT_PTR(mem_ld_adl),
6044
5982
EVENT_PTR(mem_st_adl),
6045
5983
NULL
5984
+
};
5985
+
5986
+
EVENT_ATTR_STR_HYBRID(mem-loads,
5987
+
mem_ld_arl_h,
5988
+
"event=0xd0,umask=0x5,ldlat=3;event=0xcd,umask=0x1,ldlat=3;event=0xd0,umask=0x5,ldlat=3",
5989
+
hybrid_big_small_tiny);
5990
+
EVENT_ATTR_STR_HYBRID(mem-stores,
5991
+
mem_st_arl_h,
5992
+
"event=0xd0,umask=0x6;event=0xcd,umask=0x2;event=0xd0,umask=0x6",
5993
+
hybrid_big_small_tiny);
5994
+
5995
+
static struct attribute *arl_h_hybrid_mem_attrs[] = {
5996
+
EVENT_PTR(mem_ld_arl_h),
5997
+
EVENT_PTR(mem_st_arl_h),
5998
+
NULL,
6046
5999
};
6047
6000
6048
6001
EVENT_ATTR_STR_HYBRID(tx-start, tx_start_adl, "event=0xc9,umask=0x1", hybrid_big);
···
6083
6006
6084
6007
FORMAT_ATTR_HYBRID(in_tx, hybrid_big);
6085
6008
FORMAT_ATTR_HYBRID(in_tx_cp, hybrid_big);
6086
-
FORMAT_ATTR_HYBRID(offcore_rsp, hybrid_big_small);
6087
-
FORMAT_ATTR_HYBRID(ldlat, hybrid_big_small);
6009
+
FORMAT_ATTR_HYBRID(offcore_rsp, hybrid_big_small_tiny);
6010
+
FORMAT_ATTR_HYBRID(ldlat, hybrid_big_small_tiny);
6088
6011
FORMAT_ATTR_HYBRID(frontend, hybrid_big);
6089
6012
6090
6013
#define ADL_HYBRID_RTM_FORMAT_ATTR \
···
6107
6030
NULL
6108
6031
};
6109
6032
6110
-
FORMAT_ATTR_HYBRID(snoop_rsp, hybrid_small);
6033
+
FORMAT_ATTR_HYBRID(snoop_rsp, hybrid_small_tiny);
6111
6034
6112
6035
static struct attribute *mtl_hybrid_extra_attr_rtm[] = {
6113
6036
ADL_HYBRID_RTM_FORMAT_ATTR,
···
6315
6238
}
6316
6239
6317
6240
static const struct { enum hybrid_pmu_type id; char *name; } intel_hybrid_pmu_type_map[] __initconst = {
6318
-
{ hybrid_small, "cpu_atom" },
6319
-
{ hybrid_big, "cpu_core" },
6241
+
{ hybrid_small, "cpu_atom" },
6242
+
{ hybrid_big, "cpu_core" },
6243
+
{ hybrid_tiny, "cpu_lowpower" },
6320
6244
};
6321
6245
6322
6246
static __always_inline int intel_pmu_init_hybrid(enum hybrid_pmu_type pmus)
···
6350
6272
0, x86_pmu_num_counters(&pmu->pmu), 0, 0);
6351
6273
6352
6274
pmu->intel_cap.capabilities = x86_pmu.intel_cap.capabilities;
6353
-
if (pmu->pmu_type & hybrid_small) {
6275
+
if (pmu->pmu_type & hybrid_small_tiny) {
6354
6276
pmu->intel_cap.perf_metrics = 0;
6355
6277
pmu->intel_cap.pebs_output_pt_available = 1;
6356
6278
pmu->mid_ack = true;
···
7187
7109
intel_pmu_pebs_data_source_lnl();
7188
7110
pr_cont("Lunarlake Hybrid events, ");
7189
7111
name = "lunarlake_hybrid";
7112
+
break;
7113
+
7114
+
case INTEL_ARROWLAKE_H:
7115
+
intel_pmu_init_hybrid(hybrid_big_small_tiny);
7116
+
7117
+
x86_pmu.pebs_latency_data = arl_h_latency_data;
7118
+
x86_pmu.get_event_constraints = arl_h_get_event_constraints;
7119
+
x86_pmu.hw_config = arl_h_hw_config;
7120
+
7121
+
td_attr = arl_h_hybrid_events_attrs;
7122
+
mem_attr = arl_h_hybrid_mem_attrs;
7123
+
tsx_attr = adl_hybrid_tsx_attrs;
7124
+
extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
7125
+
mtl_hybrid_extra_attr_rtm : mtl_hybrid_extra_attr;
7126
+
7127
+
/* Initialize big core specific PerfMon capabilities. */
7128
+
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
7129
+
intel_pmu_init_lnc(&pmu->pmu);
7130
+
7131
+
/* Initialize Atom core specific PerfMon capabilities. */
7132
+
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
7133
+
intel_pmu_init_skt(&pmu->pmu);
7134
+
7135
+
/* Initialize Lower Power Atom specific PerfMon capabilities. */
7136
+
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_TINY_IDX];
7137
+
intel_pmu_init_grt(&pmu->pmu);
7138
+
pmu->extra_regs = intel_cmt_extra_regs;
7139
+
7140
+
intel_pmu_pebs_data_source_arl_h();
7141
+
pr_cont("ArrowLake-H Hybrid events, ");
7142
+
name = "arrowlake_h_hybrid";
7190
7143
break;
7191
7144
7192
7145
default:
+21
arch/x86/events/intel/ds.c
+21
arch/x86/events/intel/ds.c
···
177
177
__intel_pmu_pebs_data_source_cmt(data_source);
178
178
}
179
179
180
+
void __init intel_pmu_pebs_data_source_arl_h(void)
181
+
{
182
+
u64 *data_source;
183
+
184
+
intel_pmu_pebs_data_source_lnl();
185
+
186
+
data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_TINY_IDX].pebs_data_source;
187
+
memcpy(data_source, pebs_data_source, sizeof(pebs_data_source));
188
+
__intel_pmu_pebs_data_source_cmt(data_source);
189
+
}
190
+
180
191
void __init intel_pmu_pebs_data_source_cmt(void)
181
192
{
182
193
__intel_pmu_pebs_data_source_cmt(pebs_data_source);
···
397
386
return cmt_latency_data(event, status);
398
387
399
388
return lnc_latency_data(event, status);
389
+
}
390
+
391
+
u64 arl_h_latency_data(struct perf_event *event, u64 status)
392
+
{
393
+
struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
394
+
395
+
if (pmu->pmu_type == hybrid_tiny)
396
+
return cmt_latency_data(event, status);
397
+
398
+
return lnl_latency_data(event, status);
400
399
}
401
400
402
401
static u64 load_latency_data(struct perf_event *event, u64 status)
+27
-9
arch/x86/events/perf_event.h
+27
-9
arch/x86/events/perf_event.h
···
668
668
#define PERF_PEBS_DATA_SOURCE_GRT_MAX 0x10
669
669
#define PERF_PEBS_DATA_SOURCE_GRT_MASK (PERF_PEBS_DATA_SOURCE_GRT_MAX - 1)
670
670
671
+
/*
672
+
* CPUID.1AH.EAX[31:0] uniquely identifies the microarchitecture
673
+
* of the core. Bits 31-24 indicates its core type (Core or Atom)
674
+
* and Bits [23:0] indicates the native model ID of the core.
675
+
* Core type and native model ID are defined in below enumerations.
676
+
*/
671
677
enum hybrid_cpu_type {
672
678
HYBRID_INTEL_NONE,
673
679
HYBRID_INTEL_ATOM = 0x20,
674
680
HYBRID_INTEL_CORE = 0x40,
675
681
};
676
682
677
-
enum hybrid_pmu_type {
678
-
not_hybrid,
679
-
hybrid_small = BIT(0),
680
-
hybrid_big = BIT(1),
681
-
682
-
hybrid_big_small = hybrid_big | hybrid_small, /* only used for matching */
683
-
};
684
-
685
683
#define X86_HYBRID_PMU_ATOM_IDX 0
686
684
#define X86_HYBRID_PMU_CORE_IDX 1
685
+
#define X86_HYBRID_PMU_TINY_IDX 2
687
686
688
-
#define X86_HYBRID_NUM_PMUS 2
687
+
enum hybrid_pmu_type {
688
+
not_hybrid,
689
+
hybrid_small = BIT(X86_HYBRID_PMU_ATOM_IDX),
690
+
hybrid_big = BIT(X86_HYBRID_PMU_CORE_IDX),
691
+
hybrid_tiny = BIT(X86_HYBRID_PMU_TINY_IDX),
692
+
693
+
/* The belows are only used for matching */
694
+
hybrid_big_small = hybrid_big | hybrid_small,
695
+
hybrid_small_tiny = hybrid_small | hybrid_tiny,
696
+
hybrid_big_small_tiny = hybrid_big | hybrid_small_tiny,
697
+
};
698
+
699
+
enum atom_native_id {
700
+
cmt_native_id = 0x2, /* Crestmont */
701
+
skt_native_id = 0x3, /* Skymont */
702
+
};
689
703
690
704
struct x86_hybrid_pmu {
691
705
struct pmu pmu;
···
1592
1578
1593
1579
u64 lnl_latency_data(struct perf_event *event, u64 status);
1594
1580
1581
+
u64 arl_h_latency_data(struct perf_event *event, u64 status);
1582
+
1595
1583
extern struct event_constraint intel_core2_pebs_event_constraints[];
1596
1584
1597
1585
extern struct event_constraint intel_atom_pebs_event_constraints[];
···
1712
1696
void intel_pmu_pebs_data_source_grt(void);
1713
1697
1714
1698
void intel_pmu_pebs_data_source_mtl(void);
1699
+
1700
+
void intel_pmu_pebs_data_source_arl_h(void);
1715
1701
1716
1702
void intel_pmu_pebs_data_source_cmt(void);
1717
1703
+34
-96
arch/x86/events/rapl.c
+34
-96
arch/x86/events/rapl.c
···
148
148
/* 1/2^hw_unit Joule */
149
149
static int rapl_hw_unit[NR_RAPL_DOMAINS] __read_mostly;
150
150
static struct rapl_pmus *rapl_pmus;
151
-
static cpumask_t rapl_cpu_mask;
152
151
static unsigned int rapl_cntr_mask;
153
152
static u64 rapl_timer_ms;
154
153
static struct perf_msr *rapl_msrs;
···
368
369
if (event->cpu < 0)
369
370
return -EINVAL;
370
371
371
-
event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
372
-
373
372
if (!cfg || cfg >= NR_RAPL_DOMAINS + 1)
374
373
return -EINVAL;
375
374
···
386
389
pmu = cpu_to_rapl_pmu(event->cpu);
387
390
if (!pmu)
388
391
return -EINVAL;
389
-
event->cpu = pmu->cpu;
390
392
event->pmu_private = pmu;
391
393
event->hw.event_base = rapl_msrs[bit].msr;
392
394
event->hw.config = cfg;
···
398
402
{
399
403
rapl_event_update(event);
400
404
}
401
-
402
-
static ssize_t rapl_get_attr_cpumask(struct device *dev,
403
-
struct device_attribute *attr, char *buf)
404
-
{
405
-
return cpumap_print_to_pagebuf(true, buf, &rapl_cpu_mask);
406
-
}
407
-
408
-
static DEVICE_ATTR(cpumask, S_IRUGO, rapl_get_attr_cpumask, NULL);
409
-
410
-
static struct attribute *rapl_pmu_attrs[] = {
411
-
&dev_attr_cpumask.attr,
412
-
NULL,
413
-
};
414
-
415
-
static struct attribute_group rapl_pmu_attr_group = {
416
-
.attrs = rapl_pmu_attrs,
417
-
};
418
405
419
406
RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01");
420
407
RAPL_EVENT_ATTR_STR(energy-pkg , rapl_pkg, "event=0x02");
···
446
467
};
447
468
448
469
static const struct attribute_group *rapl_attr_groups[] = {
449
-
&rapl_pmu_attr_group,
450
470
&rapl_pmu_format_group,
451
471
&rapl_pmu_events_group,
452
472
NULL,
···
548
570
[PERF_RAPL_PSYS] = { 0, &rapl_events_psys_group, NULL, false, 0 },
549
571
};
550
572
551
-
static int rapl_cpu_offline(unsigned int cpu)
552
-
{
553
-
struct rapl_pmu *pmu = cpu_to_rapl_pmu(cpu);
554
-
int target;
555
-
556
-
/* Check if exiting cpu is used for collecting rapl events */
557
-
if (!cpumask_test_and_clear_cpu(cpu, &rapl_cpu_mask))
558
-
return 0;
559
-
560
-
pmu->cpu = -1;
561
-
/* Find a new cpu to collect rapl events */
562
-
target = cpumask_any_but(get_rapl_pmu_cpumask(cpu), cpu);
563
-
564
-
/* Migrate rapl events to the new target */
565
-
if (target < nr_cpu_ids) {
566
-
cpumask_set_cpu(target, &rapl_cpu_mask);
567
-
pmu->cpu = target;
568
-
perf_pmu_migrate_context(pmu->pmu, cpu, target);
569
-
}
570
-
return 0;
571
-
}
572
-
573
-
static int rapl_cpu_online(unsigned int cpu)
574
-
{
575
-
s32 rapl_pmu_idx = get_rapl_pmu_idx(cpu);
576
-
if (rapl_pmu_idx < 0) {
577
-
pr_err("topology_logical_(package/die)_id() returned a negative value");
578
-
return -EINVAL;
579
-
}
580
-
struct rapl_pmu *pmu = cpu_to_rapl_pmu(cpu);
581
-
int target;
582
-
583
-
if (!pmu) {
584
-
pmu = kzalloc_node(sizeof(*pmu), GFP_KERNEL, cpu_to_node(cpu));
585
-
if (!pmu)
586
-
return -ENOMEM;
587
-
588
-
raw_spin_lock_init(&pmu->lock);
589
-
INIT_LIST_HEAD(&pmu->active_list);
590
-
pmu->pmu = &rapl_pmus->pmu;
591
-
pmu->timer_interval = ms_to_ktime(rapl_timer_ms);
592
-
rapl_hrtimer_init(pmu);
593
-
594
-
rapl_pmus->pmus[rapl_pmu_idx] = pmu;
595
-
}
596
-
597
-
/*
598
-
* Check if there is an online cpu in the package which collects rapl
599
-
* events already.
600
-
*/
601
-
target = cpumask_any_and(&rapl_cpu_mask, get_rapl_pmu_cpumask(cpu));
602
-
if (target < nr_cpu_ids)
603
-
return 0;
604
-
605
-
cpumask_set_cpu(cpu, &rapl_cpu_mask);
606
-
pmu->cpu = cpu;
607
-
return 0;
608
-
}
609
-
610
573
static int rapl_check_hw_unit(struct rapl_model *rm)
611
574
{
612
575
u64 msr_rapl_power_unit_bits;
···
626
707
NULL,
627
708
};
628
709
710
+
static int __init init_rapl_pmu(void)
711
+
{
712
+
struct rapl_pmu *pmu;
713
+
int idx;
714
+
715
+
for (idx = 0; idx < rapl_pmus->nr_rapl_pmu; idx++) {
716
+
pmu = kzalloc(sizeof(*pmu), GFP_KERNEL);
717
+
if (!pmu)
718
+
goto free;
719
+
720
+
raw_spin_lock_init(&pmu->lock);
721
+
INIT_LIST_HEAD(&pmu->active_list);
722
+
pmu->pmu = &rapl_pmus->pmu;
723
+
pmu->timer_interval = ms_to_ktime(rapl_timer_ms);
724
+
rapl_hrtimer_init(pmu);
725
+
726
+
rapl_pmus->pmus[idx] = pmu;
727
+
}
728
+
729
+
return 0;
730
+
free:
731
+
for (; idx > 0; idx--)
732
+
kfree(rapl_pmus->pmus[idx - 1]);
733
+
return -ENOMEM;
734
+
}
735
+
629
736
static int __init init_rapl_pmus(void)
630
737
{
631
738
int nr_rapl_pmu = topology_max_packages();
739
+
int rapl_pmu_scope = PERF_PMU_SCOPE_PKG;
632
740
633
-
if (!rapl_pmu_is_pkg_scope())
741
+
if (!rapl_pmu_is_pkg_scope()) {
634
742
nr_rapl_pmu *= topology_max_dies_per_package();
743
+
rapl_pmu_scope = PERF_PMU_SCOPE_DIE;
744
+
}
635
745
636
746
rapl_pmus = kzalloc(struct_size(rapl_pmus, pmus, nr_rapl_pmu), GFP_KERNEL);
637
747
if (!rapl_pmus)
···
676
728
rapl_pmus->pmu.start = rapl_pmu_event_start;
677
729
rapl_pmus->pmu.stop = rapl_pmu_event_stop;
678
730
rapl_pmus->pmu.read = rapl_pmu_event_read;
731
+
rapl_pmus->pmu.scope = rapl_pmu_scope;
679
732
rapl_pmus->pmu.module = THIS_MODULE;
680
733
rapl_pmus->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE;
681
-
return 0;
734
+
735
+
return init_rapl_pmu();
682
736
}
683
737
684
738
static struct rapl_model model_snb = {
···
826
876
if (ret)
827
877
return ret;
828
878
829
-
/*
830
-
* Install callbacks. Core will call them for each online cpu.
831
-
*/
832
-
ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_RAPL_ONLINE,
833
-
"perf/x86/rapl:online",
834
-
rapl_cpu_online, rapl_cpu_offline);
835
-
if (ret)
836
-
goto out;
837
-
838
879
ret = perf_pmu_register(&rapl_pmus->pmu, "power", -1);
839
880
if (ret)
840
-
goto out1;
881
+
goto out;
841
882
842
883
rapl_advertise();
843
884
return 0;
844
885
845
-
out1:
846
-
cpuhp_remove_state(CPUHP_AP_PERF_X86_RAPL_ONLINE);
847
886
out:
848
887
pr_warn("Initialization failed (%d), disabled\n", ret);
849
888
cleanup_rapl_pmus();
···
842
903
843
904
static void __exit intel_rapl_exit(void)
844
905
{
845
-
cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_RAPL_ONLINE);
846
906
perf_pmu_unregister(&rapl_pmus->pmu);
847
907
cleanup_rapl_pmus();
848
908
}
+6
arch/x86/include/asm/cpu.h
+6
arch/x86/include/asm/cpu.h
···
32
32
extern bool handle_guest_split_lock(unsigned long ip);
33
33
extern void handle_bus_lock(struct pt_regs *regs);
34
34
u8 get_this_hybrid_cpu_type(void);
35
+
u32 get_this_hybrid_cpu_native_id(void);
35
36
#else
36
37
static inline void __init sld_setup(struct cpuinfo_x86 *c) {}
37
38
static inline bool handle_user_split_lock(struct pt_regs *regs, long error_code)
···
48
47
static inline void handle_bus_lock(struct pt_regs *regs) {}
49
48
50
49
static inline u8 get_this_hybrid_cpu_type(void)
50
+
{
51
+
return 0;
52
+
}
53
+
54
+
static inline u32 get_this_hybrid_cpu_native_id(void)
51
55
{
52
56
return 0;
53
57
}
+15
arch/x86/kernel/cpu/intel.c
+15
arch/x86/kernel/cpu/intel.c
···
1299
1299
1300
1300
return cpuid_eax(0x0000001a) >> X86_HYBRID_CPU_TYPE_ID_SHIFT;
1301
1301
}
1302
+
1303
+
/**
1304
+
* get_this_hybrid_cpu_native_id() - Get the native id of this hybrid CPU
1305
+
*
1306
+
* Returns the uarch native ID [23:0] of a CPU in a hybrid processor.
1307
+
* If the processor is not hybrid, returns 0.
1308
+
*/
1309
+
u32 get_this_hybrid_cpu_native_id(void)
1310
+
{
1311
+
if (!cpu_feature_enabled(X86_FEATURE_HYBRID_CPU))
1312
+
return 0;
1313
+
1314
+
return cpuid_eax(0x0000001a) &
1315
+
(BIT_ULL(X86_HYBRID_CPU_TYPE_ID_SHIFT) - 1);
1316
+
}
-1
include/linux/cpuhotplug.h
-1
include/linux/cpuhotplug.h
+74
-5
include/linux/uprobes.h
+74
-5
include/linux/uprobes.h
···
15
15
#include <linux/rbtree.h>
16
16
#include <linux/types.h>
17
17
#include <linux/wait.h>
18
+
#include <linux/timer.h>
18
19
19
20
struct uprobe;
20
21
struct vm_area_struct;
···
24
23
struct notifier_block;
25
24
struct page;
26
25
26
+
/*
27
+
* Allowed return values from uprobe consumer's handler callback
28
+
* with following meaning:
29
+
*
30
+
* UPROBE_HANDLER_REMOVE
31
+
* - Remove the uprobe breakpoint from current->mm.
32
+
* UPROBE_HANDLER_IGNORE
33
+
* - Ignore ret_handler callback for this consumer.
34
+
*/
27
35
#define UPROBE_HANDLER_REMOVE 1
28
-
#define UPROBE_HANDLER_MASK 1
36
+
#define UPROBE_HANDLER_IGNORE 2
29
37
30
38
#define MAX_URETPROBE_DEPTH 64
31
39
···
47
37
* for the current process. If filter() is omitted or returns true,
48
38
* UPROBE_HANDLER_REMOVE is effectively ignored.
49
39
*/
50
-
int (*handler)(struct uprobe_consumer *self, struct pt_regs *regs);
40
+
int (*handler)(struct uprobe_consumer *self, struct pt_regs *regs, __u64 *data);
51
41
int (*ret_handler)(struct uprobe_consumer *self,
52
42
unsigned long func,
53
-
struct pt_regs *regs);
43
+
struct pt_regs *regs, __u64 *data);
54
44
bool (*filter)(struct uprobe_consumer *self, struct mm_struct *mm);
55
45
56
46
struct list_head cons_node;
47
+
48
+
__u64 id; /* set when uprobe_consumer is registered */
57
49
};
58
50
59
51
#ifdef CONFIG_UPROBES
···
66
54
UTASK_SSTEP,
67
55
UTASK_SSTEP_ACK,
68
56
UTASK_SSTEP_TRAPPED,
57
+
};
58
+
59
+
/* The state of hybrid-lifetime uprobe inside struct return_instance */
60
+
enum hprobe_state {
61
+
HPROBE_LEASED, /* uretprobes_srcu-protected uprobe */
62
+
HPROBE_STABLE, /* refcounted uprobe */
63
+
HPROBE_GONE, /* NULL uprobe, SRCU expired, refcount failed */
64
+
HPROBE_CONSUMED, /* uprobe "consumed" by uretprobe handler */
65
+
};
66
+
67
+
/*
68
+
* Hybrid lifetime uprobe. Represents a uprobe instance that could be either
69
+
* SRCU protected (with SRCU protection eventually potentially timing out),
70
+
* refcounted using uprobe->ref, or there could be no valid uprobe (NULL).
71
+
*
72
+
* hprobe's internal state is setup such that background timer thread can
73
+
* atomically "downgrade" temporarily RCU-protected uprobe into refcounted one
74
+
* (or no uprobe, if refcounting failed).
75
+
*
76
+
* *stable* pointer always point to the uprobe (or could be NULL if there is
77
+
* was no valid underlying uprobe to begin with).
78
+
*
79
+
* *leased* pointer is the key to achieving race-free atomic lifetime state
80
+
* transition and can have three possible states:
81
+
* - either the same non-NULL value as *stable*, in which case uprobe is
82
+
* SRCU-protected;
83
+
* - NULL, in which case uprobe (if there is any) is refcounted;
84
+
* - special __UPROBE_DEAD value, which represents an uprobe that was SRCU
85
+
* protected initially, but SRCU period timed out and we attempted to
86
+
* convert it to refcounted, but refcount_inc_not_zero() failed, because
87
+
* uprobe effectively went away (the last consumer unsubscribed). In this
88
+
* case it's important to know that *stable* pointer (which still has
89
+
* non-NULL uprobe pointer) shouldn't be used, because lifetime of
90
+
* underlying uprobe is not guaranteed anymore. __UPROBE_DEAD is just an
91
+
* internal marker and is handled transparently by hprobe_fetch() helper.
92
+
*
93
+
* When uprobe is SRCU-protected, we also record srcu_idx value, necessary for
94
+
* SRCU unlocking.
95
+
*
96
+
* See hprobe_expire() and hprobe_fetch() for details of race-free uprobe
97
+
* state transitioning details. It all hinges on atomic xchg() over *leaded*
98
+
* pointer. *stable* pointer, once initially set, is not modified concurrently.
99
+
*/
100
+
struct hprobe {
101
+
enum hprobe_state state;
102
+
int srcu_idx;
103
+
struct uprobe *uprobe;
69
104
};
70
105
71
106
/*
···
134
75
};
135
76
136
77
struct uprobe *active_uprobe;
78
+
struct timer_list ri_timer;
137
79
unsigned long xol_vaddr;
138
80
139
81
struct arch_uprobe *auprobe;
···
143
83
unsigned int depth;
144
84
};
145
85
86
+
struct return_consumer {
87
+
__u64 cookie;
88
+
__u64 id;
89
+
};
90
+
146
91
struct return_instance {
147
-
struct uprobe *uprobe;
92
+
struct hprobe hprobe;
148
93
unsigned long func;
149
94
unsigned long stack; /* stack pointer */
150
95
unsigned long orig_ret_vaddr; /* original return address */
151
96
bool chained; /* true, if instance is nested */
97
+
int consumers_cnt;
152
98
153
99
struct return_instance *next; /* keep as stack */
154
-
};
100
+
struct rcu_head rcu;
101
+
102
+
struct return_consumer consumers[] __counted_by(consumers_cnt);
103
+
} ____cacheline_aligned;
155
104
156
105
enum rp_check {
157
106
RP_CHECK_CALL,
+440
-172
kernel/events/uprobes.c
+440
-172
kernel/events/uprobes.c
···
26
26
#include <linux/task_work.h>
27
27
#include <linux/shmem_fs.h>
28
28
#include <linux/khugepaged.h>
29
+
#include <linux/rcupdate_trace.h>
30
+
#include <linux/workqueue.h>
31
+
#include <linux/srcu.h>
29
32
30
33
#include <linux/uprobes.h>
31
34
···
45
42
static DEFINE_RWLOCK(uprobes_treelock); /* serialize rbtree access */
46
43
static seqcount_rwlock_t uprobes_seqcount = SEQCNT_RWLOCK_ZERO(uprobes_seqcount, &uprobes_treelock);
47
44
48
-
DEFINE_STATIC_SRCU(uprobes_srcu);
49
-
50
45
#define UPROBES_HASH_SZ 13
51
46
/* serialize uprobe->pending_list */
52
47
static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
53
48
#define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
54
49
55
50
DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
51
+
52
+
/* Covers return_instance's uprobe lifetime. */
53
+
DEFINE_STATIC_SRCU(uretprobes_srcu);
56
54
57
55
/* Have a copy of original instruction */
58
56
#define UPROBE_COPY_INSN 0
···
66
62
struct list_head pending_list;
67
63
struct list_head consumers;
68
64
struct inode *inode; /* Also hold a ref to inode */
69
-
struct rcu_head rcu;
65
+
union {
66
+
struct rcu_head rcu;
67
+
struct work_struct work;
68
+
};
70
69
loff_t offset;
71
70
loff_t ref_ctr_offset;
72
-
unsigned long flags;
71
+
unsigned long flags; /* "unsigned long" so bitops work */
73
72
74
73
/*
75
74
* The generic code assumes that it has two members of unknown type
···
107
100
*/
108
101
struct xol_area {
109
102
wait_queue_head_t wq; /* if all slots are busy */
110
-
atomic_t slot_count; /* number of in-use slots */
111
103
unsigned long *bitmap; /* 0 = free slot */
112
104
113
105
struct page *page;
···
626
620
return !RB_EMPTY_NODE(&uprobe->rb_node);
627
621
}
628
622
629
-
static void uprobe_free_rcu(struct rcu_head *rcu)
623
+
static void uprobe_free_rcu_tasks_trace(struct rcu_head *rcu)
630
624
{
631
625
struct uprobe *uprobe = container_of(rcu, struct uprobe, rcu);
632
626
633
627
kfree(uprobe);
634
628
}
635
629
636
-
static void put_uprobe(struct uprobe *uprobe)
630
+
static void uprobe_free_srcu(struct rcu_head *rcu)
637
631
{
638
-
if (!refcount_dec_and_test(&uprobe->ref))
639
-
return;
632
+
struct uprobe *uprobe = container_of(rcu, struct uprobe, rcu);
633
+
634
+
call_rcu_tasks_trace(&uprobe->rcu, uprobe_free_rcu_tasks_trace);
635
+
}
636
+
637
+
static void uprobe_free_deferred(struct work_struct *work)
638
+
{
639
+
struct uprobe *uprobe = container_of(work, struct uprobe, work);
640
640
641
641
write_lock(&uprobes_treelock);
642
642
···
663
651
delayed_uprobe_remove(uprobe, NULL);
664
652
mutex_unlock(&delayed_uprobe_lock);
665
653
666
-
call_srcu(&uprobes_srcu, &uprobe->rcu, uprobe_free_rcu);
654
+
/* start srcu -> rcu_tasks_trace -> kfree chain */
655
+
call_srcu(&uretprobes_srcu, &uprobe->rcu, uprobe_free_srcu);
656
+
}
657
+
658
+
static void put_uprobe(struct uprobe *uprobe)
659
+
{
660
+
if (!refcount_dec_and_test(&uprobe->ref))
661
+
return;
662
+
663
+
INIT_WORK(&uprobe->work, uprobe_free_deferred);
664
+
schedule_work(&uprobe->work);
665
+
}
666
+
667
+
/* Initialize hprobe as SRCU-protected "leased" uprobe */
668
+
static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe, int srcu_idx)
669
+
{
670
+
WARN_ON(!uprobe);
671
+
hprobe->state = HPROBE_LEASED;
672
+
hprobe->uprobe = uprobe;
673
+
hprobe->srcu_idx = srcu_idx;
674
+
}
675
+
676
+
/* Initialize hprobe as refcounted ("stable") uprobe (uprobe can be NULL). */
677
+
static void hprobe_init_stable(struct hprobe *hprobe, struct uprobe *uprobe)
678
+
{
679
+
hprobe->state = uprobe ? HPROBE_STABLE : HPROBE_GONE;
680
+
hprobe->uprobe = uprobe;
681
+
hprobe->srcu_idx = -1;
682
+
}
683
+
684
+
/*
685
+
* hprobe_consume() fetches hprobe's underlying uprobe and detects whether
686
+
* uprobe is SRCU protected or is refcounted. hprobe_consume() can be
687
+
* used only once for a given hprobe.
688
+
*
689
+
* Caller has to call hprobe_finalize() and pass previous hprobe_state, so
690
+
* that hprobe_finalize() can perform SRCU unlock or put uprobe, whichever
691
+
* is appropriate.
692
+
*/
693
+
static inline struct uprobe *hprobe_consume(struct hprobe *hprobe, enum hprobe_state *hstate)
694
+
{
695
+
*hstate = xchg(&hprobe->state, HPROBE_CONSUMED);
696
+
switch (*hstate) {
697
+
case HPROBE_LEASED:
698
+
case HPROBE_STABLE:
699
+
return hprobe->uprobe;
700
+
case HPROBE_GONE: /* uprobe is NULL, no SRCU */
701
+
case HPROBE_CONSUMED: /* uprobe was finalized already, do nothing */
702
+
return NULL;
703
+
default:
704
+
WARN(1, "hprobe invalid state %d", *hstate);
705
+
return NULL;
706
+
}
707
+
}
708
+
709
+
/*
710
+
* Reset hprobe state and, if hprobe was LEASED, release SRCU lock.
711
+
* hprobe_finalize() can only be used from current context after
712
+
* hprobe_consume() call (which determines uprobe and hstate value).
713
+
*/
714
+
static void hprobe_finalize(struct hprobe *hprobe, enum hprobe_state hstate)
715
+
{
716
+
switch (hstate) {
717
+
case HPROBE_LEASED:
718
+
__srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx);
719
+
break;
720
+
case HPROBE_STABLE:
721
+
put_uprobe(hprobe->uprobe);
722
+
break;
723
+
case HPROBE_GONE:
724
+
case HPROBE_CONSUMED:
725
+
break;
726
+
default:
727
+
WARN(1, "hprobe invalid state %d", hstate);
728
+
break;
729
+
}
730
+
}
731
+
732
+
/*
733
+
* Attempt to switch (atomically) uprobe from being SRCU protected (LEASED)
734
+
* to refcounted (STABLE) state. Competes with hprobe_consume(); only one of
735
+
* them can win the race to perform SRCU unlocking. Whoever wins must perform
736
+
* SRCU unlock.
737
+
*
738
+
* Returns underlying valid uprobe or NULL, if there was no underlying uprobe
739
+
* to begin with or we failed to bump its refcount and it's going away.
740
+
*
741
+
* Returned non-NULL uprobe can be still safely used within an ongoing SRCU
742
+
* locked region. If `get` is true, it's guaranteed that non-NULL uprobe has
743
+
* an extra refcount for caller to assume and use. Otherwise, it's not
744
+
* guaranteed that returned uprobe has a positive refcount, so caller has to
745
+
* attempt try_get_uprobe(), if it needs to preserve uprobe beyond current
746
+
* SRCU lock region. See dup_utask().
747
+
*/
748
+
static struct uprobe *hprobe_expire(struct hprobe *hprobe, bool get)
749
+
{
750
+
enum hprobe_state hstate;
751
+
752
+
/*
753
+
* return_instance's hprobe is protected by RCU.
754
+
* Underlying uprobe is itself protected from reuse by SRCU.
755
+
*/
756
+
lockdep_assert(rcu_read_lock_held() && srcu_read_lock_held(&uretprobes_srcu));
757
+
758
+
hstate = READ_ONCE(hprobe->state);
759
+
switch (hstate) {
760
+
case HPROBE_STABLE:
761
+
/* uprobe has positive refcount, bump refcount, if necessary */
762
+
return get ? get_uprobe(hprobe->uprobe) : hprobe->uprobe;
763
+
case HPROBE_GONE:
764
+
/*
765
+
* SRCU was unlocked earlier and we didn't manage to take
766
+
* uprobe refcnt, so it's effectively NULL
767
+
*/
768
+
return NULL;
769
+
case HPROBE_CONSUMED:
770
+
/*
771
+
* uprobe was consumed, so it's effectively NULL as far as
772
+
* uretprobe processing logic is concerned
773
+
*/
774
+
return NULL;
775
+
case HPROBE_LEASED: {
776
+
struct uprobe *uprobe = try_get_uprobe(hprobe->uprobe);
777
+
/*
778
+
* Try to switch hprobe state, guarding against
779
+
* hprobe_consume() or another hprobe_expire() racing with us.
780
+
* Note, if we failed to get uprobe refcount, we use special
781
+
* HPROBE_GONE state to signal that hprobe->uprobe shouldn't
782
+
* be used as it will be freed after SRCU is unlocked.
783
+
*/
784
+
if (try_cmpxchg(&hprobe->state, &hstate, uprobe ? HPROBE_STABLE : HPROBE_GONE)) {
785
+
/* We won the race, we are the ones to unlock SRCU */
786
+
__srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx);
787
+
return get ? get_uprobe(uprobe) : uprobe;
788
+
}
789
+
790
+
/*
791
+
* We lost the race, undo refcount bump (if it ever happened),
792
+
* unless caller would like an extra refcount anyways.
793
+
*/
794
+
if (uprobe && !get)
795
+
put_uprobe(uprobe);
796
+
/*
797
+
* Even if hprobe_consume() or another hprobe_expire() wins
798
+
* the state update race and unlocks SRCU from under us, we
799
+
* still have a guarantee that underyling uprobe won't be
800
+
* freed due to ongoing caller's SRCU lock region, so we can
801
+
* return it regardless. Also, if `get` was true, we also have
802
+
* an extra ref for the caller to own. This is used in dup_utask().
803
+
*/
804
+
return uprobe;
805
+
}
806
+
default:
807
+
WARN(1, "unknown hprobe state %d", hstate);
808
+
return NULL;
809
+
}
667
810
}
668
811
669
812
static __always_inline
···
873
706
struct rb_node *node;
874
707
unsigned int seq;
875
708
876
-
lockdep_assert(srcu_read_lock_held(&uprobes_srcu));
709
+
lockdep_assert(rcu_read_lock_trace_held());
877
710
878
711
do {
879
712
seq = read_seqcount_begin(&uprobes_seqcount);
···
992
825
993
826
static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
994
827
{
828
+
static atomic64_t id;
829
+
995
830
down_write(&uprobe->consumer_rwsem);
996
831
list_add_rcu(&uc->cons_node, &uprobe->consumers);
832
+
uc->id = (__u64) atomic64_inc_return(&id);
997
833
up_write(&uprobe->consumer_rwsem);
998
834
}
999
835
···
1104
934
bool ret = false;
1105
935
1106
936
down_read(&uprobe->consumer_rwsem);
1107
-
list_for_each_entry_srcu(uc, &uprobe->consumers, cons_node,
1108
-
srcu_read_lock_held(&uprobes_srcu)) {
937
+
list_for_each_entry_rcu(uc, &uprobe->consumers, cons_node, rcu_read_lock_trace_held()) {
1109
938
ret = consumer_filter(uc, mm);
1110
939
if (ret)
1111
940
break;
···
1325
1156
* unlucky enough caller can free consumer's memory and cause
1326
1157
* handler_chain() or handle_uretprobe_chain() to do an use-after-free.
1327
1158
*/
1328
-
synchronize_srcu(&uprobes_srcu);
1159
+
synchronize_rcu_tasks_trace();
1160
+
synchronize_srcu(&uretprobes_srcu);
1329
1161
}
1330
1162
EXPORT_SYMBOL_GPL(uprobe_unregister_sync);
1331
1163
···
1410
1240
int uprobe_apply(struct uprobe *uprobe, struct uprobe_consumer *uc, bool add)
1411
1241
{
1412
1242
struct uprobe_consumer *con;
1413
-
int ret = -ENOENT, srcu_idx;
1243
+
int ret = -ENOENT;
1414
1244
1415
1245
down_write(&uprobe->register_rwsem);
1416
1246
1417
-
srcu_idx = srcu_read_lock(&uprobes_srcu);
1418
-
list_for_each_entry_srcu(con, &uprobe->consumers, cons_node,
1419
-
srcu_read_lock_held(&uprobes_srcu)) {
1247
+
rcu_read_lock_trace();
1248
+
list_for_each_entry_rcu(con, &uprobe->consumers, cons_node, rcu_read_lock_trace_held()) {
1420
1249
if (con == uc) {
1421
1250
ret = register_for_each_vma(uprobe, add ? uc : NULL);
1422
1251
break;
1423
1252
}
1424
1253
}
1425
-
srcu_read_unlock(&uprobes_srcu, srcu_idx);
1254
+
rcu_read_unlock_trace();
1426
1255
1427
1256
up_write(&uprobe->register_rwsem);
1428
1257
···
1644
1475
return 0;
1645
1476
}
1646
1477
1478
+
static int xol_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma)
1479
+
{
1480
+
return -EPERM;
1481
+
}
1482
+
1647
1483
static const struct vm_special_mapping xol_mapping = {
1648
1484
.name = "[uprobes]",
1649
1485
.fault = xol_fault,
1486
+
.mremap = xol_mremap,
1650
1487
};
1651
1488
1652
1489
/* Slot allocation for XOL */
···
1728
1553
init_waitqueue_head(&area->wq);
1729
1554
/* Reserve the 1st slot for get_trampoline_vaddr() */
1730
1555
set_bit(0, area->bitmap);
1731
-
atomic_set(&area->slot_count, 1);
1732
1556
insns = arch_uprobe_trampoline(&insns_size);
1733
1557
arch_uprobe_copy_ixol(area->page, 0, insns, insns_size);
1734
1558
···
1800
1626
}
1801
1627
}
1802
1628
1803
-
/*
1804
-
* - search for a free slot.
1805
-
*/
1806
-
static unsigned long xol_take_insn_slot(struct xol_area *area)
1629
+
static unsigned long xol_get_slot_nr(struct xol_area *area)
1807
1630
{
1808
-
unsigned long slot_addr;
1809
-
int slot_nr;
1631
+
unsigned long slot_nr;
1810
1632
1811
-
do {
1812
-
slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
1813
-
if (slot_nr < UINSNS_PER_PAGE) {
1814
-
if (!test_and_set_bit(slot_nr, area->bitmap))
1815
-
break;
1633
+
slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
1634
+
if (slot_nr < UINSNS_PER_PAGE) {
1635
+
if (!test_and_set_bit(slot_nr, area->bitmap))
1636
+
return slot_nr;
1637
+
}
1816
1638
1817
-
slot_nr = UINSNS_PER_PAGE;
1818
-
continue;
1819
-
}
1820
-
wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
1821
-
} while (slot_nr >= UINSNS_PER_PAGE);
1822
-
1823
-
slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
1824
-
atomic_inc(&area->slot_count);
1825
-
1826
-
return slot_addr;
1639
+
return UINSNS_PER_PAGE;
1827
1640
}
1828
1641
1829
1642
/*
1830
1643
* xol_get_insn_slot - allocate a slot for xol.
1831
-
* Returns the allocated slot address or 0.
1832
1644
*/
1833
-
static unsigned long xol_get_insn_slot(struct uprobe *uprobe)
1645
+
static bool xol_get_insn_slot(struct uprobe *uprobe, struct uprobe_task *utask)
1834
1646
{
1835
-
struct xol_area *area;
1836
-
unsigned long xol_vaddr;
1647
+
struct xol_area *area = get_xol_area();
1648
+
unsigned long slot_nr;
1837
1649
1838
-
area = get_xol_area();
1839
1650
if (!area)
1840
-
return 0;
1651
+
return false;
1841
1652
1842
-
xol_vaddr = xol_take_insn_slot(area);
1843
-
if (unlikely(!xol_vaddr))
1844
-
return 0;
1653
+
wait_event(area->wq, (slot_nr = xol_get_slot_nr(area)) < UINSNS_PER_PAGE);
1845
1654
1846
-
arch_uprobe_copy_ixol(area->page, xol_vaddr,
1655
+
utask->xol_vaddr = area->vaddr + slot_nr * UPROBE_XOL_SLOT_BYTES;
1656
+
arch_uprobe_copy_ixol(area->page, utask->xol_vaddr,
1847
1657
&uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
1848
-
1849
-
return xol_vaddr;
1658
+
return true;
1850
1659
}
1851
1660
1852
1661
/*
1853
-
* xol_free_insn_slot - If slot was earlier allocated by
1854
-
* @xol_get_insn_slot(), make the slot available for
1855
-
* subsequent requests.
1662
+
* xol_free_insn_slot - free the slot allocated by xol_get_insn_slot()
1856
1663
*/
1857
-
static void xol_free_insn_slot(struct task_struct *tsk)
1664
+
static void xol_free_insn_slot(struct uprobe_task *utask)
1858
1665
{
1859
-
struct xol_area *area;
1860
-
unsigned long vma_end;
1861
-
unsigned long slot_addr;
1666
+
struct xol_area *area = current->mm->uprobes_state.xol_area;
1667
+
unsigned long offset = utask->xol_vaddr - area->vaddr;
1668
+
unsigned int slot_nr;
1862
1669
1863
-
if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)
1670
+
utask->xol_vaddr = 0;
1671
+
/* xol_vaddr must fit into [area->vaddr, area->vaddr + PAGE_SIZE) */
1672
+
if (WARN_ON_ONCE(offset >= PAGE_SIZE))
1864
1673
return;
1865
1674
1866
-
slot_addr = tsk->utask->xol_vaddr;
1867
-
if (unlikely(!slot_addr))
1868
-
return;
1869
-
1870
-
area = tsk->mm->uprobes_state.xol_area;
1871
-
vma_end = area->vaddr + PAGE_SIZE;
1872
-
if (area->vaddr <= slot_addr && slot_addr < vma_end) {
1873
-
unsigned long offset;
1874
-
int slot_nr;
1875
-
1876
-
offset = slot_addr - area->vaddr;
1877
-
slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
1878
-
if (slot_nr >= UINSNS_PER_PAGE)
1879
-
return;
1880
-
1881
-
clear_bit(slot_nr, area->bitmap);
1882
-
atomic_dec(&area->slot_count);
1883
-
smp_mb__after_atomic(); /* pairs with prepare_to_wait() */
1884
-
if (waitqueue_active(&area->wq))
1885
-
wake_up(&area->wq);
1886
-
1887
-
tsk->utask->xol_vaddr = 0;
1888
-
}
1675
+
slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
1676
+
clear_bit(slot_nr, area->bitmap);
1677
+
smp_mb__after_atomic(); /* pairs with prepare_to_wait() */
1678
+
if (waitqueue_active(&area->wq))
1679
+
wake_up(&area->wq);
1889
1680
}
1890
1681
1891
1682
void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
···
1889
1750
return instruction_pointer(regs);
1890
1751
}
1891
1752
1892
-
static struct return_instance *free_ret_instance(struct return_instance *ri)
1753
+
static struct return_instance *free_ret_instance(struct return_instance *ri, bool cleanup_hprobe)
1893
1754
{
1894
1755
struct return_instance *next = ri->next;
1895
-
put_uprobe(ri->uprobe);
1896
-
kfree(ri);
1756
+
1757
+
if (cleanup_hprobe) {
1758
+
enum hprobe_state hstate;
1759
+
1760
+
(void)hprobe_consume(&ri->hprobe, &hstate);
1761
+
hprobe_finalize(&ri->hprobe, hstate);
1762
+
}
1763
+
1764
+
kfree_rcu(ri, rcu);
1897
1765
return next;
1898
1766
}
1899
1767
···
1916
1770
if (!utask)
1917
1771
return;
1918
1772
1919
-
if (utask->active_uprobe)
1920
-
put_uprobe(utask->active_uprobe);
1773
+
WARN_ON_ONCE(utask->active_uprobe || utask->xol_vaddr);
1774
+
1775
+
timer_delete_sync(&utask->ri_timer);
1921
1776
1922
1777
ri = utask->return_instances;
1923
1778
while (ri)
1924
-
ri = free_ret_instance(ri);
1779
+
ri = free_ret_instance(ri, true /* cleanup_hprobe */);
1925
1780
1926
-
xol_free_insn_slot(t);
1927
1781
kfree(utask);
1928
1782
t->utask = NULL;
1783
+
}
1784
+
1785
+
#define RI_TIMER_PERIOD (HZ / 10) /* 100 ms */
1786
+
1787
+
#define for_each_ret_instance_rcu(pos, head) \
1788
+
for (pos = rcu_dereference_raw(head); pos; pos = rcu_dereference_raw(pos->next))
1789
+
1790
+
static void ri_timer(struct timer_list *timer)
1791
+
{
1792
+
struct uprobe_task *utask = container_of(timer, struct uprobe_task, ri_timer);
1793
+
struct return_instance *ri;
1794
+
1795
+
/* SRCU protects uprobe from reuse for the cmpxchg() inside hprobe_expire(). */
1796
+
guard(srcu)(&uretprobes_srcu);
1797
+
/* RCU protects return_instance from freeing. */
1798
+
guard(rcu)();
1799
+
1800
+
for_each_ret_instance_rcu(ri, utask->return_instances)
1801
+
hprobe_expire(&ri->hprobe, false);
1802
+
}
1803
+
1804
+
static struct uprobe_task *alloc_utask(void)
1805
+
{
1806
+
struct uprobe_task *utask;
1807
+
1808
+
utask = kzalloc(sizeof(*utask), GFP_KERNEL);
1809
+
if (!utask)
1810
+
return NULL;
1811
+
1812
+
timer_setup(&utask->ri_timer, ri_timer, 0);
1813
+
1814
+
return utask;
1929
1815
}
1930
1816
1931
1817
/*
···
1971
1793
static struct uprobe_task *get_utask(void)
1972
1794
{
1973
1795
if (!current->utask)
1974
-
current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1796
+
current->utask = alloc_utask();
1975
1797
return current->utask;
1798
+
}
1799
+
1800
+
static size_t ri_size(int consumers_cnt)
1801
+
{
1802
+
struct return_instance *ri;
1803
+
1804
+
return sizeof(*ri) + sizeof(ri->consumers[0]) * consumers_cnt;
1805
+
}
1806
+
1807
+
#define DEF_CNT 4
1808
+
1809
+
static struct return_instance *alloc_return_instance(void)
1810
+
{
1811
+
struct return_instance *ri;
1812
+
1813
+
ri = kzalloc(ri_size(DEF_CNT), GFP_KERNEL);
1814
+
if (!ri)
1815
+
return ZERO_SIZE_PTR;
1816
+
1817
+
ri->consumers_cnt = DEF_CNT;
1818
+
return ri;
1819
+
}
1820
+
1821
+
static struct return_instance *dup_return_instance(struct return_instance *old)
1822
+
{
1823
+
size_t size = ri_size(old->consumers_cnt);
1824
+
1825
+
return kmemdup(old, size, GFP_KERNEL);
1976
1826
}
1977
1827
1978
1828
static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
1979
1829
{
1980
1830
struct uprobe_task *n_utask;
1981
1831
struct return_instance **p, *o, *n;
1832
+
struct uprobe *uprobe;
1982
1833
1983
-
n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1834
+
n_utask = alloc_utask();
1984
1835
if (!n_utask)
1985
1836
return -ENOMEM;
1986
1837
t->utask = n_utask;
1987
1838
1839
+
/* protect uprobes from freeing, we'll need try_get_uprobe() them */
1840
+
guard(srcu)(&uretprobes_srcu);
1841
+
1988
1842
p = &n_utask->return_instances;
1989
1843
for (o = o_utask->return_instances; o; o = o->next) {
1990
-
n = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1844
+
n = dup_return_instance(o);
1991
1845
if (!n)
1992
1846
return -ENOMEM;
1993
1847
1994
-
*n = *o;
1995
-
/*
1996
-
* uprobe's refcnt has to be positive at this point, kept by
1997
-
* utask->return_instances items; return_instances can't be
1998
-
* removed right now, as task is blocked due to duping; so
1999
-
* get_uprobe() is safe to use here.
2000
-
*/
2001
-
get_uprobe(n->uprobe);
2002
-
n->next = NULL;
1848
+
/* if uprobe is non-NULL, we'll have an extra refcount for uprobe */
1849
+
uprobe = hprobe_expire(&o->hprobe, true);
2003
1850
2004
-
*p = n;
1851
+
/*
1852
+
* New utask will have stable properly refcounted uprobe or
1853
+
* NULL. Even if we failed to get refcounted uprobe, we still
1854
+
* need to preserve full set of return_instances for proper
1855
+
* uretprobe handling and nesting in forked task.
1856
+
*/
1857
+
hprobe_init_stable(&n->hprobe, uprobe);
1858
+
1859
+
n->next = NULL;
1860
+
rcu_assign_pointer(*p, n);
2005
1861
p = &n->next;
1862
+
2006
1863
n_utask->depth++;
2007
1864
}
2008
1865
···
2113
1900
enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL;
2114
1901
2115
1902
while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) {
2116
-
ri = free_ret_instance(ri);
1903
+
ri = free_ret_instance(ri, true /* cleanup_hprobe */);
2117
1904
utask->depth--;
2118
1905
}
2119
-
utask->return_instances = ri;
1906
+
rcu_assign_pointer(utask->return_instances, ri);
2120
1907
}
2121
1908
2122
-
static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)
1909
+
static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs,
1910
+
struct return_instance *ri)
2123
1911
{
2124
-
struct return_instance *ri;
2125
-
struct uprobe_task *utask;
1912
+
struct uprobe_task *utask = current->utask;
2126
1913
unsigned long orig_ret_vaddr, trampoline_vaddr;
2127
1914
bool chained;
1915
+
int srcu_idx;
2128
1916
2129
1917
if (!get_xol_area())
2130
-
return;
2131
-
2132
-
utask = get_utask();
2133
-
if (!utask)
2134
-
return;
1918
+
goto free;
2135
1919
2136
1920
if (utask->depth >= MAX_URETPROBE_DEPTH) {
2137
1921
printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to"
2138
1922
" nestedness limit pid/tgid=%d/%d\n",
2139
1923
current->pid, current->tgid);
2140
-
return;
1924
+
goto free;
2141
1925
}
2142
-
2143
-
/* we need to bump refcount to store uprobe in utask */
2144
-
if (!try_get_uprobe(uprobe))
2145
-
return;
2146
-
2147
-
ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
2148
-
if (!ri)
2149
-
goto fail;
2150
1926
2151
1927
trampoline_vaddr = uprobe_get_trampoline_vaddr();
2152
1928
orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
2153
1929
if (orig_ret_vaddr == -1)
2154
-
goto fail;
1930
+
goto free;
2155
1931
2156
1932
/* drop the entries invalidated by longjmp() */
2157
1933
chained = (orig_ret_vaddr == trampoline_vaddr);
···
2158
1956
* attack from user-space.
2159
1957
*/
2160
1958
uprobe_warn(current, "handle tail call");
2161
-
goto fail;
1959
+
goto free;
2162
1960
}
2163
1961
orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
2164
1962
}
2165
-
ri->uprobe = uprobe;
1963
+
1964
+
/* __srcu_read_lock() because SRCU lock survives switch to user space */
1965
+
srcu_idx = __srcu_read_lock(&uretprobes_srcu);
1966
+
2166
1967
ri->func = instruction_pointer(regs);
2167
1968
ri->stack = user_stack_pointer(regs);
2168
1969
ri->orig_ret_vaddr = orig_ret_vaddr;
2169
1970
ri->chained = chained;
2170
1971
2171
1972
utask->depth++;
1973
+
1974
+
hprobe_init_leased(&ri->hprobe, uprobe, srcu_idx);
2172
1975
ri->next = utask->return_instances;
2173
-
utask->return_instances = ri;
1976
+
rcu_assign_pointer(utask->return_instances, ri);
1977
+
1978
+
mod_timer(&utask->ri_timer, jiffies + RI_TIMER_PERIOD);
2174
1979
2175
1980
return;
2176
-
fail:
1981
+
free:
2177
1982
kfree(ri);
2178
-
put_uprobe(uprobe);
2179
1983
}
2180
1984
2181
1985
/* Prepare to single-step probed instruction out of line. */
2182
1986
static int
2183
1987
pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
2184
1988
{
2185
-
struct uprobe_task *utask;
2186
-
unsigned long xol_vaddr;
1989
+
struct uprobe_task *utask = current->utask;
2187
1990
int err;
2188
-
2189
-
utask = get_utask();
2190
-
if (!utask)
2191
-
return -ENOMEM;
2192
1991
2193
1992
if (!try_get_uprobe(uprobe))
2194
1993
return -EINVAL;
2195
1994
2196
-
xol_vaddr = xol_get_insn_slot(uprobe);
2197
-
if (!xol_vaddr) {
1995
+
if (!xol_get_insn_slot(uprobe, utask)) {
2198
1996
err = -ENOMEM;
2199
1997
goto err_out;
2200
1998
}
2201
1999
2202
-
utask->xol_vaddr = xol_vaddr;
2203
2000
utask->vaddr = bp_vaddr;
2204
-
2205
2001
err = arch_uprobe_pre_xol(&uprobe->arch, regs);
2206
2002
if (unlikely(err)) {
2207
-
xol_free_insn_slot(current);
2003
+
xol_free_insn_slot(utask);
2208
2004
goto err_out;
2209
2005
}
2210
2006
···
2325
2125
return uprobe;
2326
2126
}
2327
2127
2128
+
static struct return_instance*
2129
+
push_consumer(struct return_instance *ri, int idx, __u64 id, __u64 cookie)
2130
+
{
2131
+
if (unlikely(ri == ZERO_SIZE_PTR))
2132
+
return ri;
2133
+
2134
+
if (unlikely(idx >= ri->consumers_cnt)) {
2135
+
struct return_instance *old_ri = ri;
2136
+
2137
+
ri->consumers_cnt += DEF_CNT;
2138
+
ri = krealloc(old_ri, ri_size(old_ri->consumers_cnt), GFP_KERNEL);
2139
+
if (!ri) {
2140
+
kfree(old_ri);
2141
+
return ZERO_SIZE_PTR;
2142
+
}
2143
+
}
2144
+
2145
+
ri->consumers[idx].id = id;
2146
+
ri->consumers[idx].cookie = cookie;
2147
+
return ri;
2148
+
}
2149
+
2150
+
static struct return_consumer *
2151
+
return_consumer_find(struct return_instance *ri, int *iter, int id)
2152
+
{
2153
+
struct return_consumer *ric;
2154
+
int idx = *iter;
2155
+
2156
+
for (ric = &ri->consumers[idx]; idx < ri->consumers_cnt; idx++, ric++) {
2157
+
if (ric->id == id) {
2158
+
*iter = idx + 1;
2159
+
return ric;
2160
+
}
2161
+
}
2162
+
return NULL;
2163
+
}
2164
+
2165
+
static bool ignore_ret_handler(int rc)
2166
+
{
2167
+
return rc == UPROBE_HANDLER_REMOVE || rc == UPROBE_HANDLER_IGNORE;
2168
+
}
2169
+
2328
2170
static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
2329
2171
{
2330
2172
struct uprobe_consumer *uc;
2331
-
int remove = UPROBE_HANDLER_REMOVE;
2332
-
bool need_prep = false; /* prepare return uprobe, when needed */
2333
-
bool has_consumers = false;
2173
+
bool has_consumers = false, remove = true;
2174
+
struct return_instance *ri = NULL;
2175
+
int push_idx = 0;
2334
2176
2335
2177
current->utask->auprobe = &uprobe->arch;
2336
2178
2337
-
list_for_each_entry_srcu(uc, &uprobe->consumers, cons_node,
2338
-
srcu_read_lock_held(&uprobes_srcu)) {
2179
+
list_for_each_entry_rcu(uc, &uprobe->consumers, cons_node, rcu_read_lock_trace_held()) {
2180
+
bool session = uc->handler && uc->ret_handler;
2181
+
__u64 cookie = 0;
2339
2182
int rc = 0;
2340
2183
2341
2184
if (uc->handler) {
2342
-
rc = uc->handler(uc, regs);
2343
-
WARN(rc & ~UPROBE_HANDLER_MASK,
2185
+
rc = uc->handler(uc, regs, &cookie);
2186
+
WARN(rc < 0 || rc > 2,
2344
2187
"bad rc=0x%x from %ps()\n", rc, uc->handler);
2345
2188
}
2346
2189
2347
-
if (uc->ret_handler)
2348
-
need_prep = true;
2349
-
2350
-
remove &= rc;
2190
+
remove &= rc == UPROBE_HANDLER_REMOVE;
2351
2191
has_consumers = true;
2192
+
2193
+
if (!uc->ret_handler || ignore_ret_handler(rc))
2194
+
continue;
2195
+
2196
+
if (!ri)
2197
+
ri = alloc_return_instance();
2198
+
2199
+
if (session)
2200
+
ri = push_consumer(ri, push_idx++, uc->id, cookie);
2352
2201
}
2353
2202
current->utask->auprobe = NULL;
2354
2203
2355
-
if (need_prep && !remove)
2356
-
prepare_uretprobe(uprobe, regs); /* put bp at return */
2204
+
if (!ZERO_OR_NULL_PTR(ri)) {
2205
+
/*
2206
+
* The push_idx value has the final number of return consumers,
2207
+
* and ri->consumers_cnt has number of allocated consumers.
2208
+
*/
2209
+
ri->consumers_cnt = push_idx;
2210
+
prepare_uretprobe(uprobe, regs, ri);
2211
+
}
2357
2212
2358
2213
if (remove && has_consumers) {
2359
2214
down_read(&uprobe->register_rwsem);
···
2424
2169
}
2425
2170
2426
2171
static void
2427
-
handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs)
2172
+
handle_uretprobe_chain(struct return_instance *ri, struct uprobe *uprobe, struct pt_regs *regs)
2428
2173
{
2429
-
struct uprobe *uprobe = ri->uprobe;
2174
+
struct return_consumer *ric;
2430
2175
struct uprobe_consumer *uc;
2431
-
int srcu_idx;
2176
+
int ric_idx = 0;
2432
2177
2433
-
srcu_idx = srcu_read_lock(&uprobes_srcu);
2434
-
list_for_each_entry_srcu(uc, &uprobe->consumers, cons_node,
2435
-
srcu_read_lock_held(&uprobes_srcu)) {
2436
-
if (uc->ret_handler)
2437
-
uc->ret_handler(uc, ri->func, regs);
2178
+
/* all consumers unsubscribed meanwhile */
2179
+
if (unlikely(!uprobe))
2180
+
return;
2181
+
2182
+
rcu_read_lock_trace();
2183
+
list_for_each_entry_rcu(uc, &uprobe->consumers, cons_node, rcu_read_lock_trace_held()) {
2184
+
bool session = uc->handler && uc->ret_handler;
2185
+
2186
+
if (uc->ret_handler) {
2187
+
ric = return_consumer_find(ri, &ric_idx, uc->id);
2188
+
if (!session || ric)
2189
+
uc->ret_handler(uc, ri->func, regs, ric ? &ric->cookie : NULL);
2190
+
}
2438
2191
}
2439
-
srcu_read_unlock(&uprobes_srcu, srcu_idx);
2192
+
rcu_read_unlock_trace();
2440
2193
}
2441
2194
2442
2195
static struct return_instance *find_next_ret_chain(struct return_instance *ri)
···
2463
2200
{
2464
2201
struct uprobe_task *utask;
2465
2202
struct return_instance *ri, *next;
2203
+
struct uprobe *uprobe;
2204
+
enum hprobe_state hstate;
2466
2205
bool valid;
2467
2206
2468
2207
utask = current->utask;
···
2495
2230
* trampoline addresses on the stack are replaced with correct
2496
2231
* original return addresses
2497
2232
*/
2498
-
utask->return_instances = ri->next;
2233
+
rcu_assign_pointer(utask->return_instances, ri->next);
2234
+
2235
+
uprobe = hprobe_consume(&ri->hprobe, &hstate);
2499
2236
if (valid)
2500
-
handle_uretprobe_chain(ri, regs);
2501
-
ri = free_ret_instance(ri);
2237
+
handle_uretprobe_chain(ri, uprobe, regs);
2238
+
hprobe_finalize(&ri->hprobe, hstate);
2239
+
2240
+
/* We already took care of hprobe, no need to waste more time on that. */
2241
+
ri = free_ret_instance(ri, false /* !cleanup_hprobe */);
2502
2242
utask->depth--;
2503
2243
} while (ri != next);
2504
2244
} while (!valid);
2505
2245
2506
-
utask->return_instances = ri;
2507
2246
return;
2508
2247
2509
-
sigill:
2248
+
sigill:
2510
2249
uprobe_warn(current, "handle uretprobe, sending SIGILL.");
2511
2250
force_sig(SIGILL);
2512
-
2513
2251
}
2514
2252
2515
2253
bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
···
2534
2266
{
2535
2267
struct uprobe *uprobe;
2536
2268
unsigned long bp_vaddr;
2537
-
int is_swbp, srcu_idx;
2269
+
int is_swbp;
2538
2270
2539
2271
bp_vaddr = uprobe_get_swbp_addr(regs);
2540
2272
if (bp_vaddr == uprobe_get_trampoline_vaddr())
2541
2273
return uprobe_handle_trampoline(regs);
2542
2274
2543
-
srcu_idx = srcu_read_lock(&uprobes_srcu);
2275
+
rcu_read_lock_trace();
2544
2276
2545
2277
uprobe = find_active_uprobe_rcu(bp_vaddr, &is_swbp);
2546
2278
if (!uprobe) {
···
2598
2330
2599
2331
out:
2600
2332
/* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
2601
-
srcu_read_unlock(&uprobes_srcu, srcu_idx);
2333
+
rcu_read_unlock_trace();
2602
2334
}
2603
2335
2604
2336
/*
···
2621
2353
put_uprobe(uprobe);
2622
2354
utask->active_uprobe = NULL;
2623
2355
utask->state = UTASK_RUNNING;
2624
-
xol_free_insn_slot(current);
2356
+
xol_free_insn_slot(utask);
2625
2357
2626
2358
spin_lock_irq(¤t->sighand->siglock);
2627
2359
recalc_sigpending(); /* see uprobe_deny_signal() */
+4
-2
kernel/trace/bpf_trace.c
+4
-2
kernel/trace/bpf_trace.c
···
3264
3264
}
3265
3265
3266
3266
static int
3267
-
uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs)
3267
+
uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs,
3268
+
__u64 *data)
3268
3269
{
3269
3270
struct bpf_uprobe *uprobe;
3270
3271
···
3274
3273
}
3275
3274
3276
3275
static int
3277
-
uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, struct pt_regs *regs)
3276
+
uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, struct pt_regs *regs,
3277
+
__u64 *data)
3278
3278
{
3279
3279
struct bpf_uprobe *uprobe;
3280
3280
+8
-4
kernel/trace/trace_uprobe.c
+8
-4
kernel/trace/trace_uprobe.c
···
89
89
static int register_uprobe_event(struct trace_uprobe *tu);
90
90
static int unregister_uprobe_event(struct trace_uprobe *tu);
91
91
92
-
static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs);
92
+
static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs,
93
+
__u64 *data);
93
94
static int uretprobe_dispatcher(struct uprobe_consumer *con,
94
-
unsigned long func, struct pt_regs *regs);
95
+
unsigned long func, struct pt_regs *regs,
96
+
__u64 *data);
95
97
96
98
#ifdef CONFIG_STACK_GROWSUP
97
99
static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
···
1524
1522
}
1525
1523
}
1526
1524
1527
-
static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
1525
+
static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs,
1526
+
__u64 *data)
1528
1527
{
1529
1528
struct trace_uprobe *tu;
1530
1529
struct uprobe_dispatch_data udd;
···
1556
1553
}
1557
1554
1558
1555
static int uretprobe_dispatcher(struct uprobe_consumer *con,
1559
-
unsigned long func, struct pt_regs *regs)
1556
+
unsigned long func, struct pt_regs *regs,
1557
+
__u64 *data)
1560
1558
{
1561
1559
struct trace_uprobe *tu;
1562
1560
struct uprobe_dispatch_data udd;