Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

powerpc/ftrace: Add support for DYNAMIC_FTRACE_WITH_CALL_OPS

Implement support for DYNAMIC_FTRACE_WITH_CALL_OPS similar to the
arm64 implementation.

This works by patching-in a pointer to an associated ftrace_ops
structure before each traceable function. If multiple ftrace_ops are
associated with a call site, then a special ftrace_list_ops is used to
enable iterating over all the registered ftrace_ops. If no ftrace_ops
are associated with a call site, then a special ftrace_nop_ops structure
is used to render the ftrace call as a no-op. ftrace trampoline can then
read the associated ftrace_ops for a call site by loading from an offset
from the LR, and branch directly to the associated function.

The primary advantage with this approach is that we don't have to
iterate over all the registered ftrace_ops for call sites that have a
single ftrace_ops registered. This is the equivalent of implementing
support for dynamic ftrace trampolines, which set up a special ftrace
trampoline for each registered ftrace_ops and have individual call sites
branch into those directly.

A secondary advantage is that this gives us a way to add support for
direct ftrace callers without having to resort to using stubs. The
address of the direct call trampoline can be loaded from the ftrace_ops
structure.

To support this, we reserve a nop before each function on 32-bit
powerpc. For 64-bit powerpc, two nops are reserved before each
out-of-line stub. During ftrace activation, we update this location with
the associated ftrace_ops pointer. Then, on ftrace entry, we load from
this location and call into ftrace_ops->func().

For 64-bit powerpc, we ensure that the out-of-line stub area is
doubleword aligned so that ftrace_ops address can be updated atomically.

Signed-off-by: Naveen N Rao <naveen@kernel.org>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://patch.msgid.link/20241030070850.1361304-15-hbathini@linux.ibm.com

authored by

Naveen N Rao and committed by
Michael Ellerman
e717754f cf9bc0ef

+102 -12
+1
arch/powerpc/Kconfig
··· 234 234 select HAVE_DEBUG_STACKOVERFLOW 235 235 select HAVE_DYNAMIC_FTRACE 236 236 select HAVE_DYNAMIC_FTRACE_WITH_ARGS if ARCH_USING_PATCHABLE_FUNCTION_ENTRY || MPROFILE_KERNEL || PPC32 237 + select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if PPC_FTRACE_OUT_OF_LINE || (PPC32 && ARCH_USING_PATCHABLE_FUNCTION_ENTRY) 237 238 select HAVE_DYNAMIC_FTRACE_WITH_REGS if ARCH_USING_PATCHABLE_FUNCTION_ENTRY || MPROFILE_KERNEL || PPC32 238 239 select HAVE_EBPF_JIT 239 240 select HAVE_EFFICIENT_UNALIGNED_ACCESS
+4
arch/powerpc/Makefile
··· 151 151 ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE 152 152 CC_FLAGS_FTRACE := -fpatchable-function-entry=1 153 153 else 154 + ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS # PPC32 only 155 + CC_FLAGS_FTRACE := -fpatchable-function-entry=3,1 156 + else 154 157 CC_FLAGS_FTRACE := -fpatchable-function-entry=2 158 + endif 155 159 endif 156 160 else 157 161 CC_FLAGS_FTRACE := -pg
+4 -1
arch/powerpc/include/asm/ftrace.h
··· 136 136 extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; 137 137 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE 138 138 struct ftrace_ool_stub { 139 + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS 140 + struct ftrace_ops *ftrace_op; 141 + #endif 139 142 u32 insn[4]; 140 - }; 143 + } __aligned(sizeof(unsigned long)); 141 144 extern struct ftrace_ool_stub ftrace_ool_stub_text_end[], ftrace_ool_stub_text[], 142 145 ftrace_ool_stub_inittext[]; 143 146 extern unsigned int ftrace_ool_stub_text_end_count, ftrace_ool_stub_text_count,
+4
arch/powerpc/kernel/asm-offsets.c
··· 679 679 DEFINE(FTRACE_OOL_STUB_SIZE, sizeof(struct ftrace_ool_stub)); 680 680 #endif 681 681 682 + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS 683 + OFFSET(FTRACE_OPS_FUNC, ftrace_ops, func); 684 + #endif 685 + 682 686 return 0; 683 687 }
+58 -1
arch/powerpc/kernel/trace/ftrace.c
··· 38 38 return 0; 39 39 40 40 if (IS_ENABLED(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY) && 41 - !IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 41 + !IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 42 42 addr += MCOUNT_INSN_SIZE; 43 + if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS)) 44 + addr += MCOUNT_INSN_SIZE; 45 + } 43 46 44 47 return addr; 45 48 } ··· 267 264 #endif 268 265 } 269 266 267 + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS 268 + static const struct ftrace_ops *powerpc_rec_get_ops(struct dyn_ftrace *rec) 269 + { 270 + const struct ftrace_ops *ops = NULL; 271 + 272 + if (rec->flags & FTRACE_FL_CALL_OPS_EN) { 273 + ops = ftrace_find_unique_ops(rec); 274 + WARN_ON_ONCE(!ops); 275 + } 276 + 277 + if (!ops) 278 + ops = &ftrace_list_ops; 279 + 280 + return ops; 281 + } 282 + 283 + static int ftrace_rec_set_ops(struct dyn_ftrace *rec, const struct ftrace_ops *ops) 284 + { 285 + if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 286 + return patch_ulong((void *)(ftrace_get_ool_stub(rec) - sizeof(unsigned long)), 287 + (unsigned long)ops); 288 + else 289 + return patch_ulong((void *)(rec->ip - MCOUNT_INSN_SIZE - sizeof(unsigned long)), 290 + (unsigned long)ops); 291 + } 292 + 293 + static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) 294 + { 295 + return ftrace_rec_set_ops(rec, &ftrace_nop_ops); 296 + } 297 + 298 + static int ftrace_rec_update_ops(struct dyn_ftrace *rec) 299 + { 300 + return ftrace_rec_set_ops(rec, powerpc_rec_get_ops(rec)); 301 + } 302 + #else 303 + static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) { return 0; } 304 + static int ftrace_rec_update_ops(struct dyn_ftrace *rec) { return 0; } 305 + #endif 306 + 270 307 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 271 308 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) 272 309 { ··· 336 293 337 294 if (!ret) 338 295 ret = ftrace_modify_code(ip, old, new); 296 + 297 + ret = ftrace_rec_update_ops(rec); 298 + if (ret) 299 + return ret; 339 300 340 301 if (!ret && IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 341 302 ret = ftrace_modify_code(rec->ip, ppc_inst(PPC_RAW_NOP()), ··· 392 345 case FTRACE_UPDATE_MODIFY_CALL: 393 346 ret = ftrace_get_call_inst(rec, new_addr, &new_call_inst); 394 347 ret |= ftrace_get_call_inst(rec, addr, &call_inst); 348 + ret |= ftrace_rec_update_ops(rec); 395 349 old = call_inst; 396 350 new = new_call_inst; 397 351 break; 398 352 case FTRACE_UPDATE_MAKE_NOP: 399 353 ret = ftrace_get_call_inst(rec, addr, &call_inst); 354 + ret |= ftrace_rec_set_nop_ops(rec); 400 355 old = call_inst; 401 356 new = nop_inst; 402 357 break; 403 358 case FTRACE_UPDATE_MAKE_CALL: 404 359 ret = ftrace_get_call_inst(rec, new_addr, &call_inst); 360 + ret |= ftrace_rec_update_ops(rec); 405 361 old = nop_inst; 406 362 new = call_inst; 407 363 break; ··· 519 469 unsigned long ip = (unsigned long)(&ftrace_call); 520 470 ppc_inst_t old, new; 521 471 int ret; 472 + 473 + /* 474 + * When using CALL_OPS, the function to call is associated with the 475 + * call site, and we don't have a global function pointer to update. 476 + */ 477 + if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS)) 478 + return 0; 522 479 523 480 old = ppc_inst_read((u32 *)&ftrace_call); 524 481 new = ftrace_create_branch_inst(ip, ppc_function_entry(func), 1);
+28 -8
arch/powerpc/kernel/trace/ftrace_entry.S
··· 85 85 /* Save callee's TOC in the ABI compliant location */ 86 86 std r2, STK_GOT(r1) 87 87 LOAD_PACA_TOC() /* get kernel TOC in r2 */ 88 + #endif 89 + 90 + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS 91 + /* r7 points to the instruction following the call to ftrace */ 92 + PPC_LL r5, -(MCOUNT_INSN_SIZE*2 + SZL)(r7) 93 + PPC_LL r12, FTRACE_OPS_FUNC(r5) 94 + mtctr r12 95 + #else /* !CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS */ 96 + #ifdef CONFIG_PPC64 88 97 LOAD_REG_ADDR(r3, function_trace_op) 89 98 ld r5,0(r3) 90 99 #else 91 100 lis r3,function_trace_op@ha 92 101 lwz r5,function_trace_op@l(r3) 102 + #endif 93 103 #endif 94 104 95 105 /* Save special regs */ ··· 215 205 #endif 216 206 .endm 217 207 218 - _GLOBAL(ftrace_regs_caller) 219 - ftrace_regs_entry 1 220 - /* ftrace_call(r3, r4, r5, r6) */ 208 + .macro ftrace_regs_func allregs 209 + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS 210 + bctrl 211 + #else 212 + .if \allregs == 1 221 213 .globl ftrace_regs_call 222 214 ftrace_regs_call: 215 + .else 216 + .globl ftrace_call 217 + ftrace_call: 218 + .endif 219 + /* ftrace_call(r3, r4, r5, r6) */ 223 220 bl ftrace_stub 221 + #endif 222 + .endm 223 + 224 + _GLOBAL(ftrace_regs_caller) 225 + ftrace_regs_entry 1 226 + ftrace_regs_func 1 224 227 ftrace_regs_exit 1 225 228 226 229 _GLOBAL(ftrace_caller) 227 230 ftrace_regs_entry 0 228 - /* ftrace_call(r3, r4, r5, r6) */ 229 - .globl ftrace_call 230 - ftrace_call: 231 - bl ftrace_stub 231 + ftrace_regs_func 0 232 232 ftrace_regs_exit 0 233 233 234 234 _GLOBAL(ftrace_stub) ··· 397 377 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE 398 378 SYM_DATA(ftrace_ool_stub_text_count, .long CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE) 399 379 400 - SYM_CODE_START(ftrace_ool_stub_text) 380 + SYM_START(ftrace_ool_stub_text, SYM_L_GLOBAL, .balign SZL) 401 381 .space CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE * FTRACE_OOL_STUB_SIZE 402 382 SYM_CODE_END(ftrace_ool_stub_text) 403 383 #endif
+3 -2
arch/powerpc/tools/ftrace-gen-ool-stubs.sh
··· 28 28 29 29 cat > "$arch_vmlinux_S" <<EOF 30 30 #include <asm/asm-offsets.h> 31 + #include <asm/ppc_asm.h> 31 32 #include <linux/linkage.h> 32 33 33 34 .pushsection .tramp.ftrace.text,"aw" 34 35 SYM_DATA(ftrace_ool_stub_text_end_count, .long $num_ool_stubs_text_end) 35 36 36 - SYM_CODE_START(ftrace_ool_stub_text_end) 37 + SYM_START(ftrace_ool_stub_text_end, SYM_L_GLOBAL, .balign SZL) 37 38 #if $num_ool_stubs_text_end 38 39 .space $num_ool_stubs_text_end * FTRACE_OOL_STUB_SIZE 39 40 #endif ··· 44 43 .pushsection .tramp.ftrace.init,"aw" 45 44 SYM_DATA(ftrace_ool_stub_inittext_count, .long $num_ool_stubs_inittext) 46 45 47 - SYM_CODE_START(ftrace_ool_stub_inittext) 46 + SYM_START(ftrace_ool_stub_inittext, SYM_L_GLOBAL, .balign SZL) 48 47 .space $num_ool_stubs_inittext * FTRACE_OOL_STUB_SIZE 49 48 SYM_CODE_END(ftrace_ool_stub_inittext) 50 49 .popsection