Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'perf-urgent-2020-11-15' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull perf fixes from Thomas Gleixner:
"A set of fixes for perf:
- A set of commits which reduce the stack usage of various perf
event handling functions which allocated large data structs on
stack causing stack overflows in the worst case
- Use the proper mechanism for detecting soft interrupts in the
recursion protection
- Make the resursion protection simpler and more robust
- Simplify the scheduling of event groups to make the code more
robust and prepare for fixing the issues vs. scheduling of
exclusive event groups
- Prevent event multiplexing and rotation for exclusive event groups
- Correct the perf event attribute exclusive semantics to take
pinned events, e.g. the PMU watchdog, into account
- Make the anythread filtering conditional for Intel's generic PMU
counters as it is not longer guaranteed to be supported on newer
CPUs. Check the corresponding CPUID leaf to make sure
- Fixup a duplicate initialization in an array which was probably
caused by the usual 'copy & paste - forgot to edit' mishap"
* tag 'perf-urgent-2020-11-15' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
perf/x86/intel/uncore: Fix Add BW copypasta
perf/x86/intel: Make anythread filter support conditional
perf: Tweak perf_event_attr::exclusive semantics
perf: Fix event multiplexing for exclusive groups
perf: Simplify group_sched_in()
perf: Simplify group_sched_out()
perf/x86: Make dummy_iregs static
perf/arch: Remove perf_sample_data::regs_user_copy
perf: Optimize get_recursion_context()
perf: Fix get_recursion_context()
perf/x86: Reduce stack usage for x86_pmu::drain_pebs()
perf: Reduce stack usage of perf_output_begin()
+115
-110
+1
-2
arch/arm/kernel/perf_regs.c
+1
-2
arch/arm/kernel/perf_regs.c
+1
-2
arch/arm64/kernel/perf_regs.c
+1
-2
arch/arm64/kernel/perf_regs.c
+1
-2
arch/csky/kernel/perf_regs.c
+1
-2
arch/csky/kernel/perf_regs.c
+1
-1
arch/powerpc/perf/imc-pmu.c
+1
-1
arch/powerpc/perf/imc-pmu.c
···
1336
1336
/* If this is a valid record, create the sample */
1337
1337
struct perf_output_handle handle;
1338
1338
1339
-
if (perf_output_begin(&handle, event, header.size))
1339
+
if (perf_output_begin(&handle, &data, event, header.size))
1340
1340
return;
1341
1341
1342
1342
perf_output_sample(&handle, &header, &data, event);
+1
-2
arch/powerpc/perf/perf_regs.c
+1
-2
arch/powerpc/perf/perf_regs.c
···
144
144
}
145
145
146
146
void perf_get_regs_user(struct perf_regs *regs_user,
147
-
struct pt_regs *regs,
148
-
struct pt_regs *regs_user_copy)
147
+
struct pt_regs *regs)
149
148
{
150
149
regs_user->regs = task_pt_regs(current);
151
150
regs_user->abi = (regs_user->regs) ? perf_reg_abi(current) :
+1
-2
arch/riscv/kernel/perf_regs.c
+1
-2
arch/riscv/kernel/perf_regs.c
+1
-1
arch/s390/kernel/perf_cpum_sf.c
+1
-1
arch/s390/kernel/perf_cpum_sf.c
···
672
672
rcu_read_lock();
673
673
674
674
perf_prepare_sample(&header, data, event, regs);
675
-
if (perf_output_begin(&handle, event, header.size))
675
+
if (perf_output_begin(&handle, data, event, header.size))
676
676
goto out;
677
677
678
678
/* Update the process ID (see also kernel/events/core.c) */
+1
-2
arch/s390/kernel/perf_regs.c
+1
-2
arch/s390/kernel/perf_regs.c
+11
-1
arch/x86/events/intel/core.c
+11
-1
arch/x86/events/intel/core.c
···
2630
2630
u64 pebs_enabled = cpuc->pebs_enabled;
2631
2631
2632
2632
handled++;
2633
-
x86_pmu.drain_pebs(regs);
2633
+
x86_pmu.drain_pebs(regs, &data);
2634
2634
status &= x86_pmu.intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI;
2635
2635
2636
2636
/*
···
4987
4987
4988
4988
x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
4989
4989
4990
+
if (version >= 5) {
4991
+
x86_pmu.intel_cap.anythread_deprecated = edx.split.anythread_deprecated;
4992
+
if (x86_pmu.intel_cap.anythread_deprecated)
4993
+
pr_cont(" AnyThread deprecated, ");
4994
+
}
4995
+
4990
4996
/*
4991
4997
* Install the hw-cache-events table:
4992
4998
*/
···
5517
5511
5518
5512
x86_pmu.intel_ctrl |=
5519
5513
((1LL << x86_pmu.num_counters_fixed)-1) << INTEL_PMC_IDX_FIXED;
5514
+
5515
+
/* AnyThread may be deprecated on arch perfmon v5 or later */
5516
+
if (x86_pmu.intel_cap.anythread_deprecated)
5517
+
x86_pmu.format_attrs = intel_arch_formats_attr;
5520
5518
5521
5519
if (x86_pmu.event_constraints) {
5522
5520
/*
+28
-25
arch/x86/events/intel/ds.c
+28
-25
arch/x86/events/intel/ds.c
···
642
642
rcu_read_lock();
643
643
perf_prepare_sample(&header, &data, event, ®s);
644
644
645
-
if (perf_output_begin(&handle, event, header.size *
646
-
(top - base - skip)))
645
+
if (perf_output_begin(&handle, &data, event,
646
+
header.size * (top - base - skip)))
647
647
goto unlock;
648
648
649
649
for (at = base; at < top; at++) {
···
670
670
671
671
static inline void intel_pmu_drain_pebs_buffer(void)
672
672
{
673
-
x86_pmu.drain_pebs(NULL);
673
+
struct perf_sample_data data;
674
+
675
+
x86_pmu.drain_pebs(NULL, &data);
674
676
}
675
677
676
678
/*
···
1721
1719
return 0;
1722
1720
}
1723
1721
1724
-
static void __intel_pmu_pebs_event(struct perf_event *event,
1725
-
struct pt_regs *iregs,
1726
-
void *base, void *top,
1727
-
int bit, int count,
1728
-
void (*setup_sample)(struct perf_event *,
1729
-
struct pt_regs *,
1730
-
void *,
1731
-
struct perf_sample_data *,
1732
-
struct pt_regs *))
1722
+
static __always_inline void
1723
+
__intel_pmu_pebs_event(struct perf_event *event,
1724
+
struct pt_regs *iregs,
1725
+
struct perf_sample_data *data,
1726
+
void *base, void *top,
1727
+
int bit, int count,
1728
+
void (*setup_sample)(struct perf_event *,
1729
+
struct pt_regs *,
1730
+
void *,
1731
+
struct perf_sample_data *,
1732
+
struct pt_regs *))
1733
1733
{
1734
1734
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1735
1735
struct hw_perf_event *hwc = &event->hw;
1736
-
struct perf_sample_data data;
1737
1736
struct x86_perf_regs perf_regs;
1738
1737
struct pt_regs *regs = &perf_regs.regs;
1739
1738
void *at = get_next_pebs_record_by_bit(base, top, bit);
1740
-
struct pt_regs dummy_iregs;
1739
+
static struct pt_regs dummy_iregs;
1741
1740
1742
1741
if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
1743
1742
/*
···
1755
1752
iregs = &dummy_iregs;
1756
1753
1757
1754
while (count > 1) {
1758
-
setup_sample(event, iregs, at, &data, regs);
1759
-
perf_event_output(event, &data, regs);
1755
+
setup_sample(event, iregs, at, data, regs);
1756
+
perf_event_output(event, data, regs);
1760
1757
at += cpuc->pebs_record_size;
1761
1758
at = get_next_pebs_record_by_bit(at, top, bit);
1762
1759
count--;
1763
1760
}
1764
1761
1765
-
setup_sample(event, iregs, at, &data, regs);
1762
+
setup_sample(event, iregs, at, data, regs);
1766
1763
if (iregs == &dummy_iregs) {
1767
1764
/*
1768
1765
* The PEBS records may be drained in the non-overflow context,
···
1770
1767
* last record the same as other PEBS records, and doesn't
1771
1768
* invoke the generic overflow handler.
1772
1769
*/
1773
-
perf_event_output(event, &data, regs);
1770
+
perf_event_output(event, data, regs);
1774
1771
} else {
1775
1772
/*
1776
1773
* All but the last records are processed.
1777
1774
* The last one is left to be able to call the overflow handler.
1778
1775
*/
1779
-
if (perf_event_overflow(event, &data, regs))
1776
+
if (perf_event_overflow(event, data, regs))
1780
1777
x86_pmu_stop(event, 0);
1781
1778
}
1782
1779
}
1783
1780
1784
-
static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
1781
+
static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data)
1785
1782
{
1786
1783
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1787
1784
struct debug_store *ds = cpuc->ds;
···
1815
1812
return;
1816
1813
}
1817
1814
1818
-
__intel_pmu_pebs_event(event, iregs, at, top, 0, n,
1815
+
__intel_pmu_pebs_event(event, iregs, data, at, top, 0, n,
1819
1816
setup_pebs_fixed_sample_data);
1820
1817
}
1821
1818
···
1838
1835
}
1839
1836
}
1840
1837
1841
-
static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
1838
+
static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_data *data)
1842
1839
{
1843
1840
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1844
1841
struct debug_store *ds = cpuc->ds;
···
1945
1942
}
1946
1943
1947
1944
if (counts[bit]) {
1948
-
__intel_pmu_pebs_event(event, iregs, base,
1945
+
__intel_pmu_pebs_event(event, iregs, data, base,
1949
1946
top, bit, counts[bit],
1950
1947
setup_pebs_fixed_sample_data);
1951
1948
}
1952
1949
}
1953
1950
}
1954
1951
1955
-
static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs)
1952
+
static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data)
1956
1953
{
1957
1954
short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
1958
1955
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
···
2000
1997
if (WARN_ON_ONCE(!event->attr.precise_ip))
2001
1998
continue;
2002
1999
2003
-
__intel_pmu_pebs_event(event, iregs, base,
2000
+
__intel_pmu_pebs_event(event, iregs, data, base,
2004
2001
top, bit, counts[bit],
2005
2002
setup_pebs_adaptive_sample_data);
2006
2003
}
+1
-1
arch/x86/events/intel/uncore_snb.c
+1
-1
arch/x86/events/intel/uncore_snb.c
···
475
475
static struct freerunning_counters snb_uncore_imc_freerunning[] = {
476
476
[SNB_PCI_UNCORE_IMC_DATA_READS] = { SNB_UNCORE_PCI_IMC_DATA_READS_BASE,
477
477
0x0, 0x0, 1, 32 },
478
-
[SNB_PCI_UNCORE_IMC_DATA_READS] = { SNB_UNCORE_PCI_IMC_DATA_WRITES_BASE,
478
+
[SNB_PCI_UNCORE_IMC_DATA_WRITES] = { SNB_UNCORE_PCI_IMC_DATA_WRITES_BASE,
479
479
0x0, 0x0, 1, 32 },
480
480
[SNB_PCI_UNCORE_IMC_GT_REQUESTS] = { SNB_UNCORE_PCI_IMC_GT_REQUESTS_BASE,
481
481
0x0, 0x0, 1, 32 },
+2
-1
arch/x86/events/perf_event.h
+2
-1
arch/x86/events/perf_event.h
···
585
585
u64 pebs_baseline:1;
586
586
u64 perf_metrics:1;
587
587
u64 pebs_output_pt_available:1;
588
+
u64 anythread_deprecated:1;
588
589
};
589
590
u64 capabilities;
590
591
};
···
728
727
int pebs_record_size;
729
728
int pebs_buffer_size;
730
729
int max_pebs_events;
731
-
void (*drain_pebs)(struct pt_regs *regs);
730
+
void (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
732
731
struct event_constraint *pebs_constraints;
733
732
void (*pebs_aliases)(struct perf_event *event);
734
733
unsigned long large_pebs_flags;
+3
-1
arch/x86/include/asm/perf_event.h
+3
-1
arch/x86/include/asm/perf_event.h
+11
-4
arch/x86/kernel/perf_regs.c
+11
-4
arch/x86/kernel/perf_regs.c
···
101
101
}
102
102
103
103
void perf_get_regs_user(struct perf_regs *regs_user,
104
-
struct pt_regs *regs,
105
-
struct pt_regs *regs_user_copy)
104
+
struct pt_regs *regs)
106
105
{
107
106
regs_user->regs = task_pt_regs(current);
108
107
regs_user->abi = perf_reg_abi(current);
···
128
129
return PERF_SAMPLE_REGS_ABI_64;
129
130
}
130
131
132
+
static DEFINE_PER_CPU(struct pt_regs, nmi_user_regs);
133
+
131
134
void perf_get_regs_user(struct perf_regs *regs_user,
132
-
struct pt_regs *regs,
133
-
struct pt_regs *regs_user_copy)
135
+
struct pt_regs *regs)
134
136
{
137
+
struct pt_regs *regs_user_copy = this_cpu_ptr(&nmi_user_regs);
135
138
struct pt_regs *user_regs = task_pt_regs(current);
139
+
140
+
if (!in_nmi()) {
141
+
regs_user->regs = user_regs;
142
+
regs_user->abi = perf_reg_abi(current);
143
+
return;
144
+
}
136
145
137
146
/*
138
147
* If we're in an NMI that interrupted task_pt_regs setup, then
+3
-1
arch/x86/kvm/cpuid.c
+3
-1
arch/x86/kvm/cpuid.c
···
681
681
682
682
edx.split.num_counters_fixed = min(cap.num_counters_fixed, MAX_FIXED_COUNTERS);
683
683
edx.split.bit_width_fixed = cap.bit_width_fixed;
684
-
edx.split.reserved = 0;
684
+
edx.split.anythread_deprecated = 1;
685
+
edx.split.reserved1 = 0;
686
+
edx.split.reserved2 = 0;
685
687
686
688
entry->eax = eax.full;
687
689
entry->ebx = cap.events_mask;
+5
-8
include/linux/perf_event.h
+5
-8
include/linux/perf_event.h
···
1022
1022
struct perf_callchain_entry *callchain;
1023
1023
u64 aux_size;
1024
1024
1025
-
/*
1026
-
* regs_user may point to task_pt_regs or to regs_user_copy, depending
1027
-
* on arch details.
1028
-
*/
1029
1025
struct perf_regs regs_user;
1030
-
struct pt_regs regs_user_copy;
1031
-
1032
1026
struct perf_regs regs_intr;
1033
1027
u64 stack_user_size;
1034
1028
···
1394
1400
extern void perf_event_addr_filters_sync(struct perf_event *event);
1395
1401
1396
1402
extern int perf_output_begin(struct perf_output_handle *handle,
1403
+
struct perf_sample_data *data,
1397
1404
struct perf_event *event, unsigned int size);
1398
1405
extern int perf_output_begin_forward(struct perf_output_handle *handle,
1399
-
struct perf_event *event,
1400
-
unsigned int size);
1406
+
struct perf_sample_data *data,
1407
+
struct perf_event *event,
1408
+
unsigned int size);
1401
1409
extern int perf_output_begin_backward(struct perf_output_handle *handle,
1410
+
struct perf_sample_data *data,
1402
1411
struct perf_event *event,
1403
1412
unsigned int size);
1404
1413
+2
-4
include/linux/perf_regs.h
+2
-4
include/linux/perf_regs.h
···
20
20
int perf_reg_validate(u64 mask);
21
21
u64 perf_reg_abi(struct task_struct *task);
22
22
void perf_get_regs_user(struct perf_regs *regs_user,
23
-
struct pt_regs *regs,
24
-
struct pt_regs *regs_user_copy);
23
+
struct pt_regs *regs);
25
24
#else
26
25
27
26
#define PERF_REG_EXTENDED_MASK 0
···
41
42
}
42
43
43
44
static inline void perf_get_regs_user(struct perf_regs *regs_user,
44
-
struct pt_regs *regs,
45
-
struct pt_regs *regs_user_copy)
45
+
struct pt_regs *regs)
46
46
{
47
47
regs_user->regs = task_pt_regs(current);
48
48
regs_user->abi = perf_reg_abi(current);
+25
-32
kernel/events/core.c
+25
-32
kernel/events/core.c
···
2312
2312
event_sched_out(event, cpuctx, ctx);
2313
2313
2314
2314
perf_pmu_enable(ctx->pmu);
2315
-
2316
-
if (group_event->attr.exclusive)
2317
-
cpuctx->exclusive = 0;
2318
2315
}
2319
2316
2320
2317
#define DETACH_GROUP 0x01UL
···
2580
2583
2581
2584
pmu->start_txn(pmu, PERF_PMU_TXN_ADD);
2582
2585
2583
-
if (event_sched_in(group_event, cpuctx, ctx)) {
2584
-
pmu->cancel_txn(pmu);
2585
-
perf_mux_hrtimer_restart(cpuctx);
2586
-
return -EAGAIN;
2587
-
}
2586
+
if (event_sched_in(group_event, cpuctx, ctx))
2587
+
goto error;
2588
2588
2589
2589
/*
2590
2590
* Schedule in siblings as one group (if any):
···
2610
2616
}
2611
2617
event_sched_out(group_event, cpuctx, ctx);
2612
2618
2619
+
error:
2613
2620
pmu->cancel_txn(pmu);
2614
-
2615
-
perf_mux_hrtimer_restart(cpuctx);
2616
-
2617
2621
return -EAGAIN;
2618
2622
}
2619
2623
···
2637
2645
* If this group is exclusive and there are already
2638
2646
* events on the CPU, it can't go on.
2639
2647
*/
2640
-
if (event->attr.exclusive && cpuctx->active_oncpu)
2648
+
if (event->attr.exclusive && !list_empty(get_event_list(event)))
2641
2649
return 0;
2642
2650
/*
2643
2651
* Otherwise, try to add it if all previous groups were able
···
3671
3679
3672
3680
*can_add_hw = 0;
3673
3681
ctx->rotate_necessary = 1;
3682
+
perf_mux_hrtimer_restart(cpuctx);
3674
3683
}
3675
3684
3676
3685
return 0;
···
6367
6374
}
6368
6375
6369
6376
static void perf_sample_regs_user(struct perf_regs *regs_user,
6370
-
struct pt_regs *regs,
6371
-
struct pt_regs *regs_user_copy)
6377
+
struct pt_regs *regs)
6372
6378
{
6373
6379
if (user_mode(regs)) {
6374
6380
regs_user->abi = perf_reg_abi(current);
6375
6381
regs_user->regs = regs;
6376
6382
} else if (!(current->flags & PF_KTHREAD)) {
6377
-
perf_get_regs_user(regs_user, regs, regs_user_copy);
6383
+
perf_get_regs_user(regs_user, regs);
6378
6384
} else {
6379
6385
regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
6380
6386
regs_user->regs = NULL;
···
7075
7083
}
7076
7084
7077
7085
if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER))
7078
-
perf_sample_regs_user(&data->regs_user, regs,
7079
-
&data->regs_user_copy);
7086
+
perf_sample_regs_user(&data->regs_user, regs);
7080
7087
7081
7088
if (sample_type & PERF_SAMPLE_REGS_USER) {
7082
7089
/* regs dump ABI info */
···
7177
7186
struct perf_sample_data *data,
7178
7187
struct pt_regs *regs,
7179
7188
int (*output_begin)(struct perf_output_handle *,
7189
+
struct perf_sample_data *,
7180
7190
struct perf_event *,
7181
7191
unsigned int))
7182
7192
{
···
7190
7198
7191
7199
perf_prepare_sample(&header, data, event, regs);
7192
7200
7193
-
err = output_begin(&handle, event, header.size);
7201
+
err = output_begin(&handle, data, event, header.size);
7194
7202
if (err)
7195
7203
goto exit;
7196
7204
···
7256
7264
int ret;
7257
7265
7258
7266
perf_event_header__init_id(&read_event.header, &sample, event);
7259
-
ret = perf_output_begin(&handle, event, read_event.header.size);
7267
+
ret = perf_output_begin(&handle, &sample, event, read_event.header.size);
7260
7268
if (ret)
7261
7269
return;
7262
7270
···
7525
7533
7526
7534
perf_event_header__init_id(&task_event->event_id.header, &sample, event);
7527
7535
7528
-
ret = perf_output_begin(&handle, event,
7536
+
ret = perf_output_begin(&handle, &sample, event,
7529
7537
task_event->event_id.header.size);
7530
7538
if (ret)
7531
7539
goto out;
···
7628
7636
return;
7629
7637
7630
7638
perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
7631
-
ret = perf_output_begin(&handle, event,
7639
+
ret = perf_output_begin(&handle, &sample, event,
7632
7640
comm_event->event_id.header.size);
7633
7641
7634
7642
if (ret)
···
7728
7736
7729
7737
perf_event_header__init_id(&namespaces_event->event_id.header,
7730
7738
&sample, event);
7731
-
ret = perf_output_begin(&handle, event,
7739
+
ret = perf_output_begin(&handle, &sample, event,
7732
7740
namespaces_event->event_id.header.size);
7733
7741
if (ret)
7734
7742
goto out;
···
7855
7863
7856
7864
perf_event_header__init_id(&cgroup_event->event_id.header,
7857
7865
&sample, event);
7858
-
ret = perf_output_begin(&handle, event,
7866
+
ret = perf_output_begin(&handle, &sample, event,
7859
7867
cgroup_event->event_id.header.size);
7860
7868
if (ret)
7861
7869
goto out;
···
7981
7989
}
7982
7990
7983
7991
perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
7984
-
ret = perf_output_begin(&handle, event,
7992
+
ret = perf_output_begin(&handle, &sample, event,
7985
7993
mmap_event->event_id.header.size);
7986
7994
if (ret)
7987
7995
goto out;
···
8291
8299
int ret;
8292
8300
8293
8301
perf_event_header__init_id(&rec.header, &sample, event);
8294
-
ret = perf_output_begin(&handle, event, rec.header.size);
8302
+
ret = perf_output_begin(&handle, &sample, event, rec.header.size);
8295
8303
8296
8304
if (ret)
8297
8305
return;
···
8325
8333
8326
8334
perf_event_header__init_id(&lost_samples_event.header, &sample, event);
8327
8335
8328
-
ret = perf_output_begin(&handle, event,
8336
+
ret = perf_output_begin(&handle, &sample, event,
8329
8337
lost_samples_event.header.size);
8330
8338
if (ret)
8331
8339
return;
···
8380
8388
8381
8389
perf_event_header__init_id(&se->event_id.header, &sample, event);
8382
8390
8383
-
ret = perf_output_begin(&handle, event, se->event_id.header.size);
8391
+
ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);
8384
8392
if (ret)
8385
8393
return;
8386
8394
···
8455
8463
8456
8464
perf_event_header__init_id(&throttle_event.header, &sample, event);
8457
8465
8458
-
ret = perf_output_begin(&handle, event,
8466
+
ret = perf_output_begin(&handle, &sample, event,
8459
8467
throttle_event.header.size);
8460
8468
if (ret)
8461
8469
return;
···
8498
8506
8499
8507
perf_event_header__init_id(&ksymbol_event->event_id.header,
8500
8508
&sample, event);
8501
-
ret = perf_output_begin(&handle, event,
8509
+
ret = perf_output_begin(&handle, &sample, event,
8502
8510
ksymbol_event->event_id.header.size);
8503
8511
if (ret)
8504
8512
return;
···
8588
8596
8589
8597
perf_event_header__init_id(&bpf_event->event_id.header,
8590
8598
&sample, event);
8591
-
ret = perf_output_begin(&handle, event,
8599
+
ret = perf_output_begin(&handle, data, event,
8592
8600
bpf_event->event_id.header.size);
8593
8601
if (ret)
8594
8602
return;
···
8697
8705
8698
8706
perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);
8699
8707
8700
-
ret = perf_output_begin(&handle, event, text_poke_event->event_id.header.size);
8708
+
ret = perf_output_begin(&handle, &sample, event,
8709
+
text_poke_event->event_id.header.size);
8701
8710
if (ret)
8702
8711
return;
8703
8712
···
8779
8786
rec.tid = perf_event_tid(event, current);
8780
8787
8781
8788
perf_event_header__init_id(&rec.header, &sample, event);
8782
-
ret = perf_output_begin(&handle, event, rec.header.size);
8789
+
ret = perf_output_begin(&handle, &sample, event, rec.header.size);
8783
8790
8784
8791
if (ret)
8785
8792
return;
+5
-9
kernel/events/internal.h
+5
-9
kernel/events/internal.h
···
205
205
206
206
static inline int get_recursion_context(int *recursion)
207
207
{
208
-
int rctx;
208
+
unsigned int pc = preempt_count();
209
+
unsigned char rctx = 0;
209
210
210
-
if (unlikely(in_nmi()))
211
-
rctx = 3;
212
-
else if (in_irq())
213
-
rctx = 2;
214
-
else if (in_softirq())
215
-
rctx = 1;
216
-
else
217
-
rctx = 0;
211
+
rctx += !!(pc & (NMI_MASK));
212
+
rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK));
213
+
rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
218
214
219
215
if (recursion[rctx])
220
216
return -1;
+11
-9
kernel/events/ring_buffer.c
+11
-9
kernel/events/ring_buffer.c
···
147
147
148
148
static __always_inline int
149
149
__perf_output_begin(struct perf_output_handle *handle,
150
+
struct perf_sample_data *data,
150
151
struct perf_event *event, unsigned int size,
151
152
bool backward)
152
153
{
···
238
237
handle->size = (1UL << page_shift) - offset;
239
238
240
239
if (unlikely(have_lost)) {
241
-
struct perf_sample_data sample_data;
242
-
243
240
lost_event.header.size = sizeof(lost_event);
244
241
lost_event.header.type = PERF_RECORD_LOST;
245
242
lost_event.header.misc = 0;
246
243
lost_event.id = event->id;
247
244
lost_event.lost = local_xchg(&rb->lost, 0);
248
245
249
-
perf_event_header__init_id(&lost_event.header,
250
-
&sample_data, event);
246
+
/* XXX mostly redundant; @data is already fully initializes */
247
+
perf_event_header__init_id(&lost_event.header, data, event);
251
248
perf_output_put(handle, lost_event);
252
-
perf_event__output_id_sample(event, handle, &sample_data);
249
+
perf_event__output_id_sample(event, handle, data);
253
250
}
254
251
255
252
return 0;
···
262
263
}
263
264
264
265
int perf_output_begin_forward(struct perf_output_handle *handle,
265
-
struct perf_event *event, unsigned int size)
266
+
struct perf_sample_data *data,
267
+
struct perf_event *event, unsigned int size)
266
268
{
267
-
return __perf_output_begin(handle, event, size, false);
269
+
return __perf_output_begin(handle, data, event, size, false);
268
270
}
269
271
270
272
int perf_output_begin_backward(struct perf_output_handle *handle,
273
+
struct perf_sample_data *data,
271
274
struct perf_event *event, unsigned int size)
272
275
{
273
-
return __perf_output_begin(handle, event, size, true);
276
+
return __perf_output_begin(handle, data, event, size, true);
274
277
}
275
278
276
279
int perf_output_begin(struct perf_output_handle *handle,
280
+
struct perf_sample_data *data,
277
281
struct perf_event *event, unsigned int size)
278
282
{
279
283
280
-
return __perf_output_begin(handle, event, size,
284
+
return __perf_output_begin(handle, data, event, size,
281
285
unlikely(is_write_backward(event)));
282
286
}
283
287