Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
powerpc/ftrace: Add support for DYNAMIC_FTRACE_WITH_DIRECT_CALLS
Add support for DYNAMIC_FTRACE_WITH_DIRECT_CALLS similar to the arm64
implementation.
ftrace direct calls allow custom trampolines to be called into directly
from function ftrace call sites, bypassing the ftrace trampoline
completely. This functionality is currently utilized by BPF trampolines
to hook into kernel function entries.
Since we have limited relative branch range, we support ftrace direct
calls through support for DYNAMIC_FTRACE_WITH_CALL_OPS. In this
approach, ftrace trampoline is not entirely bypassed. Rather, it is
re-purposed into a stub that reads direct_call field from the associated
ftrace_ops structure and branches into that, if it is not NULL. For
this, it is sufficient if we can ensure that the ftrace trampoline is
reachable from all traceable functions.
When multiple ftrace_ops are associated with a call site, we utilize a
call back to set pt_regs->orig_gpr3 that can then be tested on the
return path from the ftrace trampoline to branch into the direct caller.
Signed-off-by: Naveen N Rao <naveen@kernel.org>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://patch.msgid.link/20241030070850.1361304-16-hbathini@linux.ibm.com
authored by
Naveen N Rao
and committed by
Michael Ellerman
a52f6043
e717754f
+116
-29
+1
arch/powerpc/Kconfig
+1
arch/powerpc/Kconfig
···
235
235
select HAVE_DYNAMIC_FTRACE
236
236
select HAVE_DYNAMIC_FTRACE_WITH_ARGS if ARCH_USING_PATCHABLE_FUNCTION_ENTRY || MPROFILE_KERNEL || PPC32
237
237
select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if PPC_FTRACE_OUT_OF_LINE || (PPC32 && ARCH_USING_PATCHABLE_FUNCTION_ENTRY)
238
+
select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
238
239
select HAVE_DYNAMIC_FTRACE_WITH_REGS if ARCH_USING_PATCHABLE_FUNCTION_ENTRY || MPROFILE_KERNEL || PPC32
239
240
select HAVE_EBPF_JIT
240
241
select HAVE_EFFICIENT_UNALIGNED_ACCESS
+16
arch/powerpc/include/asm/ftrace.h
+16
arch/powerpc/include/asm/ftrace.h
···
148
148
#endif
149
149
void ftrace_free_init_tramp(void);
150
150
unsigned long ftrace_call_adjust(unsigned long addr);
151
+
152
+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
153
+
/*
154
+
* When an ftrace registered caller is tracing a function that is also set by a
155
+
* register_ftrace_direct() call, it needs to be differentiated in the
156
+
* ftrace_caller trampoline so that the direct call can be invoked after the
157
+
* other ftrace ops. To do this, place the direct caller in the orig_gpr3 field
158
+
* of pt_regs. This tells ftrace_caller that there's a direct caller.
159
+
*/
160
+
static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs, unsigned long addr)
161
+
{
162
+
struct pt_regs *regs = &fregs->regs;
163
+
164
+
regs->orig_gpr3 = addr;
165
+
}
166
+
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
151
167
#else
152
168
static inline void ftrace_free_init_tramp(void) { }
153
169
static inline unsigned long ftrace_call_adjust(unsigned long addr) { return addr; }
+3
arch/powerpc/kernel/asm-offsets.c
+3
arch/powerpc/kernel/asm-offsets.c
+11
arch/powerpc/kernel/trace/ftrace.c
+11
arch/powerpc/kernel/trace/ftrace.c
···
150
150
else
151
151
ip = rec->ip;
152
152
153
+
if (!is_offset_in_branch_range(addr - ip) && addr != FTRACE_ADDR &&
154
+
addr != FTRACE_REGS_ADDR) {
155
+
/* This can only happen with ftrace direct */
156
+
if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS)) {
157
+
pr_err("0x%lx (0x%lx): Unexpected target address 0x%lx\n",
158
+
ip, rec->ip, addr);
159
+
return -EINVAL;
160
+
}
161
+
addr = FTRACE_ADDR;
162
+
}
163
+
153
164
if (is_offset_in_branch_range(addr - ip))
154
165
/* Within range */
155
166
stub = addr;
+85
-29
arch/powerpc/kernel/trace/ftrace_entry.S
+85
-29
arch/powerpc/kernel/trace/ftrace_entry.S
···
33
33
* and then arrange for the ftrace function to be called.
34
34
*/
35
35
.macro ftrace_regs_entry allregs
36
-
/* Save the original return address in A's stack frame */
37
-
PPC_STL r0, LRSAVE(r1)
38
36
/* Create a minimal stack frame for representing B */
39
37
PPC_STLU r1, -STACK_FRAME_MIN_SIZE(r1)
40
38
41
39
/* Create our stack frame + pt_regs */
42
40
PPC_STLU r1,-SWITCH_FRAME_SIZE(r1)
41
+
42
+
.if \allregs == 1
43
+
SAVE_GPRS(11, 12, r1)
44
+
.endif
45
+
46
+
/* Get the _mcount() call site out of LR */
47
+
mflr r11
48
+
49
+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
50
+
/* Load the ftrace_op */
51
+
PPC_LL r12, -(MCOUNT_INSN_SIZE*2 + SZL)(r11)
52
+
53
+
/* Load direct_call from the ftrace_op */
54
+
PPC_LL r12, FTRACE_OPS_DIRECT_CALL(r12)
55
+
PPC_LCMPI r12, 0
56
+
.if \allregs == 1
57
+
bne .Lftrace_direct_call_regs
58
+
.else
59
+
bne .Lftrace_direct_call
60
+
.endif
61
+
#endif
62
+
63
+
/* Save the previous LR in pt_regs->link */
64
+
PPC_STL r0, _LINK(r1)
65
+
/* Also save it in A's stack frame */
66
+
PPC_STL r0, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE+LRSAVE(r1)
43
67
44
68
/* Save all gprs to pt_regs */
45
69
SAVE_GPR(0, r1)
···
78
54
79
55
.if \allregs == 1
80
56
SAVE_GPR(2, r1)
81
-
SAVE_GPRS(11, 31, r1)
57
+
SAVE_GPRS(13, 31, r1)
82
58
.else
83
59
#if defined(CONFIG_LIVEPATCH_64) || defined(CONFIG_PPC_FTRACE_OUT_OF_LINE)
84
60
SAVE_GPR(14, r1)
···
91
67
92
68
.if \allregs == 1
93
69
/* Load special regs for save below */
70
+
mfcr r7
94
71
mfmsr r8
95
72
mfctr r9
96
73
mfxer r10
97
-
mfcr r11
98
74
.else
99
75
/* Clear MSR to flag as ftrace_caller versus frace_regs_caller */
100
76
li r8, 0
101
77
.endif
102
-
103
-
/* Get the _mcount() call site out of LR */
104
-
mflr r7
105
-
/* Save the read LR in pt_regs->link */
106
-
PPC_STL r0, _LINK(r1)
107
78
108
79
#ifdef CONFIG_PPC64
109
80
/* Save callee's TOC in the ABI compliant location */
···
107
88
#endif
108
89
109
90
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
110
-
/* r7 points to the instruction following the call to ftrace */
111
-
PPC_LL r5, -(MCOUNT_INSN_SIZE*2 + SZL)(r7)
91
+
/* r11 points to the instruction following the call to ftrace */
92
+
PPC_LL r5, -(MCOUNT_INSN_SIZE*2 + SZL)(r11)
112
93
PPC_LL r12, FTRACE_OPS_FUNC(r5)
113
94
mtctr r12
114
95
#else /* !CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS */
···
124
105
/* Save special regs */
125
106
PPC_STL r8, _MSR(r1)
126
107
.if \allregs == 1
108
+
PPC_STL r7, _CCR(r1)
127
109
PPC_STL r9, _CTR(r1)
128
110
PPC_STL r10, _XER(r1)
129
-
PPC_STL r11, _CCR(r1)
130
111
.endif
112
+
113
+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
114
+
/* Clear orig_gpr3 to later detect ftrace_direct call */
115
+
li r7, 0
116
+
PPC_STL r7, ORIG_GPR3(r1)
117
+
#endif
131
118
132
119
#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
133
120
/* Save our real return address in nvr for return */
134
121
.if \allregs == 0
135
122
SAVE_GPR(15, r1)
136
123
.endif
137
-
mr r15, r7
124
+
mr r15, r11
138
125
/*
139
-
* We want the ftrace location in the function, but our lr (in r7)
126
+
* We want the ftrace location in the function, but our lr (in r11)
140
127
* points at the 'mtlr r0' instruction in the out of line stub. To
141
128
* recover the ftrace location, we read the branch instruction in the
142
129
* stub, and adjust our lr by the branch offset.
143
130
*
144
131
* See ftrace_init_ool_stub() for the profile sequence.
145
132
*/
146
-
lwz r8, MCOUNT_INSN_SIZE(r7)
133
+
lwz r8, MCOUNT_INSN_SIZE(r11)
147
134
slwi r8, r8, 6
148
135
srawi r8, r8, 6
149
-
add r3, r7, r8
136
+
add r3, r11, r8
150
137
/*
151
138
* Override our nip to point past the branch in the original function.
152
139
* This allows reliable stack trace and the ftrace stack tracer to work as-is.
153
140
*/
154
-
addi r7, r3, MCOUNT_INSN_SIZE
141
+
addi r11, r3, MCOUNT_INSN_SIZE
155
142
#else
156
143
/* Calculate ip from nip-4 into r3 for call below */
157
-
subi r3, r7, MCOUNT_INSN_SIZE
144
+
subi r3, r11, MCOUNT_INSN_SIZE
158
145
#endif
159
146
160
147
/* Save NIP as pt_regs->nip */
161
-
PPC_STL r7, _NIP(r1)
148
+
PPC_STL r11, _NIP(r1)
162
149
/* Also save it in B's stackframe header for proper unwind */
163
-
PPC_STL r7, LRSAVE+SWITCH_FRAME_SIZE(r1)
150
+
PPC_STL r11, LRSAVE+SWITCH_FRAME_SIZE(r1)
164
151
#if defined(CONFIG_LIVEPATCH_64) || defined(CONFIG_PPC_FTRACE_OUT_OF_LINE)
165
-
mr r14, r7 /* remember old NIP */
152
+
mr r14, r11 /* remember old NIP */
166
153
#endif
167
154
168
155
/* Put the original return address in r4 as parent_ip */
···
179
154
.endm
180
155
181
156
.macro ftrace_regs_exit allregs
157
+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
158
+
/* Check orig_gpr3 to detect ftrace_direct call */
159
+
PPC_LL r3, ORIG_GPR3(r1)
160
+
PPC_LCMPI cr1, r3, 0
161
+
mtctr r3
162
+
#endif
163
+
164
+
/* Restore possibly modified LR */
165
+
PPC_LL r0, _LINK(r1)
166
+
182
167
#ifndef CONFIG_PPC_FTRACE_OUT_OF_LINE
183
168
/* Load ctr with the possibly modified NIP */
184
169
PPC_LL r3, _NIP(r1)
185
-
mtctr r3
186
-
187
170
#ifdef CONFIG_LIVEPATCH_64
188
171
cmpd r14, r3 /* has NIP been altered? */
189
172
#endif
173
+
174
+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
175
+
beq cr1,2f
176
+
mtlr r3
177
+
b 3f
178
+
#endif
179
+
2: mtctr r3
180
+
mtlr r0
181
+
3:
182
+
190
183
#else /* !CONFIG_PPC_FTRACE_OUT_OF_LINE */
191
184
/* Load LR with the possibly modified NIP */
192
185
PPC_LL r3, _NIP(r1)
···
228
185
#endif
229
186
.endif
230
187
231
-
/* Restore possibly modified LR */
232
-
PPC_LL r0, _LINK(r1)
233
-
#ifndef CONFIG_PPC_FTRACE_OUT_OF_LINE
234
-
mtlr r0
235
-
#endif
236
-
237
188
#ifdef CONFIG_PPC64
238
189
/* Restore callee's TOC */
239
190
ld r2, STK_GOT(r1)
···
240
203
/* Based on the cmpd above, if the NIP was altered handle livepatch */
241
204
bne- livepatch_handler
242
205
#endif
206
+
243
207
/* jump after _mcount site */
244
208
#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
209
+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
210
+
bnectr cr1
211
+
#endif
245
212
/*
246
213
* Return with blr to keep the link stack balanced. The function profiling sequence
247
214
* uses 'mtlr r0' to restore LR.
···
299
258
mtlr r0
300
259
bctr
301
260
#endif
261
+
#endif
262
+
263
+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
264
+
.Lftrace_direct_call_regs:
265
+
mtctr r12
266
+
REST_GPRS(11, 12, r1)
267
+
addi r1, r1, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE
268
+
bctr
269
+
.Lftrace_direct_call:
270
+
mtctr r12
271
+
addi r1, r1, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE
272
+
bctr
273
+
SYM_FUNC_START(ftrace_stub_direct_tramp)
274
+
blr
275
+
SYM_FUNC_END(ftrace_stub_direct_tramp)
302
276
#endif
303
277
304
278
#ifdef CONFIG_LIVEPATCH_64