Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

ftrace,bpf: Remove FTRACE_OPS_FL_JMP ftrace_ops flag

At the moment the we allow the jmp attach only for ftrace_ops that
has FTRACE_OPS_FL_JMP set. This conflicts with following changes
where we use single ftrace_ops object for all direct call sites,
so all could be be attached via just call or jmp.

We already limit the jmp attach support with config option and bit
(LSB) set on the trampoline address. It turns out that's actually
enough to limit the jmp attach for architecture and only for chosen
addresses (with LSB bit set).

Each user of register_ftrace_direct or modify_ftrace_direct can set
the trampoline bit (LSB) to indicate it has to be attached by jmp.

The bpf trampoline generation code uses trampoline flags to generate
jmp-attach specific code and ftrace inner code uses the trampoline
bit (LSB) to handle return from jmp attachment, so there's no harm
to remove the FTRACE_OPS_FL_JMP bit.

The fexit/fmodret performance stays the same (did not drop),
current code:

fentry : 77.904 ± 0.546M/s
fexit : 62.430 ± 0.554M/s
fmodret : 66.503 ± 0.902M/s

with this change:

fentry : 80.472 ± 0.061M/s
fexit : 63.995 ± 0.127M/s
fmodret : 67.362 ± 0.175M/s

Fixes: 25e4e3565d45 ("ftrace: Introduce FTRACE_OPS_FL_JMP")
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/bpf/20251230145010.103439-2-jolsa@kernel.org

authored by

Jiri Olsa and committed by
Andrii Nakryiko
4be42c92 ae23bc81

+14 -33
-1
include/linux/ftrace.h
··· 359 359 FTRACE_OPS_FL_DIRECT = BIT(17), 360 360 FTRACE_OPS_FL_SUBOP = BIT(18), 361 361 FTRACE_OPS_FL_GRAPH = BIT(19), 362 - FTRACE_OPS_FL_JMP = BIT(20), 363 362 }; 364 363 365 364 #ifndef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
+14 -18
kernel/bpf/trampoline.c
··· 221 221 int ret; 222 222 223 223 if (tr->func.ftrace_managed) { 224 + unsigned long addr = (unsigned long) new_addr; 225 + 226 + if (bpf_trampoline_use_jmp(tr->flags)) 227 + addr = ftrace_jmp_set(addr); 228 + 224 229 if (lock_direct_mutex) 225 - ret = modify_ftrace_direct(tr->fops, (long)new_addr); 230 + ret = modify_ftrace_direct(tr->fops, addr); 226 231 else 227 - ret = modify_ftrace_direct_nolock(tr->fops, (long)new_addr); 232 + ret = modify_ftrace_direct_nolock(tr->fops, addr); 228 233 } else { 229 234 ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, 230 235 new_addr); ··· 252 247 } 253 248 254 249 if (tr->func.ftrace_managed) { 250 + unsigned long addr = (unsigned long) new_addr; 251 + 252 + if (bpf_trampoline_use_jmp(tr->flags)) 253 + addr = ftrace_jmp_set(addr); 254 + 255 255 ret = ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1); 256 256 if (ret) 257 257 return ret; 258 - ret = register_ftrace_direct(tr->fops, (long)new_addr); 258 + ret = register_ftrace_direct(tr->fops, addr); 259 259 } else { 260 260 ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); 261 261 } ··· 516 506 if (err) 517 507 goto out_free; 518 508 519 - #ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP 520 - if (bpf_trampoline_use_jmp(tr->flags)) 521 - tr->fops->flags |= FTRACE_OPS_FL_JMP; 522 - else 523 - tr->fops->flags &= ~FTRACE_OPS_FL_JMP; 524 - #endif 525 - 526 509 WARN_ON(tr->cur_image && total == 0); 527 510 if (tr->cur_image) 528 511 /* progs already running at this address */ ··· 543 540 tr->cur_image = im; 544 541 out: 545 542 /* If any error happens, restore previous flags */ 546 - if (err) { 543 + if (err) 547 544 tr->flags = orig_flags; 548 - #ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP 549 - if (bpf_trampoline_use_jmp(tr->flags)) 550 - tr->fops->flags |= FTRACE_OPS_FL_JMP; 551 - else 552 - tr->fops->flags &= ~FTRACE_OPS_FL_JMP; 553 - #endif 554 - } 555 545 kfree(tlinks); 556 546 return err; 557 547
-14
kernel/trace/ftrace.c
··· 6046 6046 if (ftrace_hash_empty(hash)) 6047 6047 return -EINVAL; 6048 6048 6049 - /* This is a "raw" address, and this should never happen. */ 6050 - if (WARN_ON_ONCE(ftrace_is_jmp(addr))) 6051 - return -EINVAL; 6052 - 6053 6049 mutex_lock(&direct_mutex); 6054 - 6055 - if (ops->flags & FTRACE_OPS_FL_JMP) 6056 - addr = ftrace_jmp_set(addr); 6057 6050 6058 6051 /* Make sure requested entries are not already registered.. */ 6059 6052 size = 1 << hash->size_bits; ··· 6167 6174 int err; 6168 6175 6169 6176 lockdep_assert_held_once(&direct_mutex); 6170 - 6171 - /* This is a "raw" address, and this should never happen. */ 6172 - if (WARN_ON_ONCE(ftrace_is_jmp(addr))) 6173 - return -EINVAL; 6174 - 6175 - if (ops->flags & FTRACE_OPS_FL_JMP) 6176 - addr = ftrace_jmp_set(addr); 6177 6177 6178 6178 /* Enable the tmp_ops to have the same functions as the direct ops */ 6179 6179 ftrace_ops_init(&tmp_ops);