ftrace: Add update_ftrace_direct_mod function

Adding update_ftrace_direct_mod function that modifies all entries
(ip -> direct) provided in hash argument to direct ftrace ops and
updates its attachments.

The difference to current modify_ftrace_direct is:
- hash argument that allows to modify multiple ip -> direct
entries at once

This change will allow us to have simple ftrace_ops for all bpf
direct interface users in following changes.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/bpf/20251230145010.103439-7-jolsa@kernel.org

authored by

Jiri Olsa and committed by

Andrii Nakryiko 4 months ago e93672f7 8d2c1233

+100

2 changed files

expand all

include

linux

ftrace.h

kernel

trace

ftrace.c

include/linux/ftrace.h

··· 545 545 546 546 int update_ftrace_direct_add(struct ftrace_ops *ops, struct ftrace_hash *hash); 547 547 int update_ftrace_direct_del(struct ftrace_ops *ops, struct ftrace_hash *hash); 548 + int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, bool do_direct_lock); 548 549 549 550 void ftrace_stub_direct_tramp(void); 550 551 ··· 579 578 } 580 579 581 580 static inline int update_ftrace_direct_del(struct ftrace_ops *ops, struct ftrace_hash *hash) 581 + { 582 + return -ENODEV; 583 + } 584 + 585 + static inline int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, bool do_direct_lock) 582 586 { 583 587 return -ENODEV; 584 588 }

+94

kernel/trace/ftrace.c

··· 6545 6545 return err; 6546 6546 } 6547 6547 6548 + /** 6549 + * update_ftrace_direct_mod - Updates @ops by modifing its direct 6550 + * callers provided in @hash 6551 + * @ops: The address of the struct ftrace_ops object 6552 + * @hash: The address of the struct ftrace_hash object 6553 + * @do_direct_lock: If true lock the direct_mutex 6554 + * 6555 + * This is used to modify custom direct callers (ip -> addr) in 6556 + * @ops specified via @hash. 6557 + * 6558 + * This can be called from within ftrace ops_func callback with 6559 + * direct_mutex already locked, in which case @do_direct_lock 6560 + * needs to be false. 6561 + * 6562 + * Returns: zero on success. Non zero on error, which includes: 6563 + * -EINVAL - The @hash is empty 6564 + * -EINVAL - The @ops is not registered 6565 + */ 6566 + int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, bool do_direct_lock) 6567 + { 6568 + struct ftrace_func_entry *entry, *tmp; 6569 + static struct ftrace_ops tmp_ops = { 6570 + .func = ftrace_stub, 6571 + .flags = FTRACE_OPS_FL_STUB, 6572 + }; 6573 + struct ftrace_hash *orig_hash; 6574 + unsigned long size, i; 6575 + int err = -EINVAL; 6576 + 6577 + if (!hash_count(hash)) 6578 + return -EINVAL; 6579 + if (check_direct_multi(ops)) 6580 + return -EINVAL; 6581 + if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 6582 + return -EINVAL; 6583 + if (direct_functions == EMPTY_HASH) 6584 + return -EINVAL; 6585 + 6586 + /* 6587 + * We can be called from within ops_func callback with direct_mutex 6588 + * already taken. 6589 + */ 6590 + if (do_direct_lock) 6591 + mutex_lock(&direct_mutex); 6592 + 6593 + orig_hash = ops->func_hash ? ops->func_hash->filter_hash : NULL; 6594 + if (!orig_hash) 6595 + goto unlock; 6596 + 6597 + /* Enable the tmp_ops to have the same functions as the direct ops */ 6598 + ftrace_ops_init(&tmp_ops); 6599 + tmp_ops.func_hash = ops->func_hash; 6600 + 6601 + err = register_ftrace_function_nolock(&tmp_ops); 6602 + if (err) 6603 + goto unlock; 6604 + 6605 + /* 6606 + * Call __ftrace_hash_update_ipmodify() here, so that we can call 6607 + * ops->ops_func for the ops. This is needed because the above 6608 + * register_ftrace_function_nolock() worked on tmp_ops. 6609 + */ 6610 + err = __ftrace_hash_update_ipmodify(ops, orig_hash, orig_hash, true); 6611 + if (err) 6612 + goto out; 6613 + 6614 + /* 6615 + * Now the ftrace_ops_list_func() is called to do the direct callers. 6616 + * We can safely change the direct functions attached to each entry. 6617 + */ 6618 + mutex_lock(&ftrace_lock); 6619 + 6620 + size = 1 << hash->size_bits; 6621 + for (i = 0; i < size; i++) { 6622 + hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 6623 + tmp = __ftrace_lookup_ip(direct_functions, entry->ip); 6624 + if (!tmp) 6625 + continue; 6626 + tmp->direct = entry->direct; 6627 + } 6628 + } 6629 + 6630 + mutex_unlock(&ftrace_lock); 6631 + 6632 + out: 6633 + /* Removing the tmp_ops will add the updated direct callers to the functions */ 6634 + unregister_ftrace_function(&tmp_ops); 6635 + 6636 + unlock: 6637 + if (do_direct_lock) 6638 + mutex_unlock(&direct_mutex); 6639 + return err; 6640 + } 6641 + 6548 6642 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ 6549 6643 6550 6644 /**

Configure Feed

Configure Feed