Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
ftrace: Add update_ftrace_direct_add function
Adding update_ftrace_direct_add function that adds all entries
(ip -> addr) provided in hash argument to direct ftrace ops
and updates its attachments.
The difference to current register_ftrace_direct is
- hash argument that allows to register multiple ip -> direct
entries at once
- we can call update_ftrace_direct_add multiple times on the
same ftrace_ops object, becase after first registration with
register_ftrace_function_nolock, it uses ftrace_update_ops to
update the ftrace_ops object
This change will allow us to have simple ftrace_ops for all bpf
direct interface users in following changes.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/bpf/20251230145010.103439-5-jolsa@kernel.org
authored by
Jiri Olsa
and committed by
Andrii Nakryiko
05dc5e9c
0e860d07
+147
+7
include/linux/ftrace.h
+7
include/linux/ftrace.h
···
543
543
int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr);
544
544
int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr);
545
545
546
+
int update_ftrace_direct_add(struct ftrace_ops *ops, struct ftrace_hash *hash);
547
+
546
548
void ftrace_stub_direct_tramp(void);
547
549
548
550
#else
···
567
565
return -ENODEV;
568
566
}
569
567
static inline int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr)
568
+
{
569
+
return -ENODEV;
570
+
}
571
+
572
+
static inline int update_ftrace_direct_add(struct ftrace_ops *ops, struct ftrace_hash *hash)
570
573
{
571
574
return -ENODEV;
572
575
}
+140
kernel/trace/ftrace.c
+140
kernel/trace/ftrace.c
···
6278
6278
return err;
6279
6279
}
6280
6280
EXPORT_SYMBOL_GPL(modify_ftrace_direct);
6281
+
6282
+
static unsigned long hash_count(struct ftrace_hash *hash)
6283
+
{
6284
+
return hash ? hash->count : 0;
6285
+
}
6286
+
6287
+
/**
6288
+
* hash_add - adds two struct ftrace_hash and returns the result
6289
+
* @a: struct ftrace_hash object
6290
+
* @b: struct ftrace_hash object
6291
+
*
6292
+
* Returns struct ftrace_hash object on success, NULL on error.
6293
+
*/
6294
+
static struct ftrace_hash *hash_add(struct ftrace_hash *a, struct ftrace_hash *b)
6295
+
{
6296
+
struct ftrace_func_entry *entry;
6297
+
struct ftrace_hash *add;
6298
+
int size;
6299
+
6300
+
size = hash_count(a) + hash_count(b);
6301
+
if (size > 32)
6302
+
size = 32;
6303
+
6304
+
add = alloc_and_copy_ftrace_hash(fls(size), a);
6305
+
if (!add)
6306
+
return NULL;
6307
+
6308
+
size = 1 << b->size_bits;
6309
+
for (int i = 0; i < size; i++) {
6310
+
hlist_for_each_entry(entry, &b->buckets[i], hlist) {
6311
+
if (add_ftrace_hash_entry_direct(add, entry->ip, entry->direct) == NULL) {
6312
+
free_ftrace_hash(add);
6313
+
return NULL;
6314
+
}
6315
+
}
6316
+
}
6317
+
return add;
6318
+
}
6319
+
6320
+
/**
6321
+
* update_ftrace_direct_add - Updates @ops by adding direct
6322
+
* callers provided in @hash
6323
+
* @ops: The address of the struct ftrace_ops object
6324
+
* @hash: The address of the struct ftrace_hash object
6325
+
*
6326
+
* This is used to add custom direct callers (ip -> addr) to @ops,
6327
+
* specified in @hash. The @ops will be either registered or updated.
6328
+
*
6329
+
* Returns: zero on success. Non zero on error, which includes:
6330
+
* -EINVAL - The @hash is empty
6331
+
*/
6332
+
int update_ftrace_direct_add(struct ftrace_ops *ops, struct ftrace_hash *hash)
6333
+
{
6334
+
struct ftrace_hash *old_direct_functions = NULL;
6335
+
struct ftrace_hash *new_direct_functions;
6336
+
struct ftrace_hash *old_filter_hash;
6337
+
struct ftrace_hash *new_filter_hash = NULL;
6338
+
struct ftrace_func_entry *entry;
6339
+
int err = -EINVAL;
6340
+
int size;
6341
+
bool reg;
6342
+
6343
+
if (!hash_count(hash))
6344
+
return -EINVAL;
6345
+
6346
+
mutex_lock(&direct_mutex);
6347
+
6348
+
/* Make sure requested entries are not already registered. */
6349
+
size = 1 << hash->size_bits;
6350
+
for (int i = 0; i < size; i++) {
6351
+
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
6352
+
if (__ftrace_lookup_ip(direct_functions, entry->ip))
6353
+
goto out_unlock;
6354
+
}
6355
+
}
6356
+
6357
+
old_filter_hash = ops->func_hash ? ops->func_hash->filter_hash : NULL;
6358
+
6359
+
/* If there's nothing in filter_hash we need to register the ops. */
6360
+
reg = hash_count(old_filter_hash) == 0;
6361
+
if (reg) {
6362
+
if (ops->func || ops->trampoline)
6363
+
goto out_unlock;
6364
+
if (ops->flags & FTRACE_OPS_FL_ENABLED)
6365
+
goto out_unlock;
6366
+
}
6367
+
6368
+
err = -ENOMEM;
6369
+
new_filter_hash = hash_add(old_filter_hash, hash);
6370
+
if (!new_filter_hash)
6371
+
goto out_unlock;
6372
+
6373
+
new_direct_functions = hash_add(direct_functions, hash);
6374
+
if (!new_direct_functions)
6375
+
goto out_unlock;
6376
+
6377
+
old_direct_functions = direct_functions;
6378
+
rcu_assign_pointer(direct_functions, new_direct_functions);
6379
+
6380
+
if (reg) {
6381
+
ops->func = call_direct_funcs;
6382
+
ops->flags |= MULTI_FLAGS;
6383
+
ops->trampoline = FTRACE_REGS_ADDR;
6384
+
ops->local_hash.filter_hash = new_filter_hash;
6385
+
6386
+
err = register_ftrace_function_nolock(ops);
6387
+
if (err) {
6388
+
/* restore old filter on error */
6389
+
ops->local_hash.filter_hash = old_filter_hash;
6390
+
6391
+
/* cleanup for possible another register call */
6392
+
ops->func = NULL;
6393
+
ops->trampoline = 0;
6394
+
} else {
6395
+
new_filter_hash = old_filter_hash;
6396
+
}
6397
+
} else {
6398
+
err = ftrace_update_ops(ops, new_filter_hash, EMPTY_HASH);
6399
+
/*
6400
+
* new_filter_hash is dup-ed, so we need to release it anyway,
6401
+
* old_filter_hash either stays on error or is already released
6402
+
*/
6403
+
}
6404
+
6405
+
if (err) {
6406
+
/* reset direct_functions and free the new one */
6407
+
rcu_assign_pointer(direct_functions, old_direct_functions);
6408
+
old_direct_functions = new_direct_functions;
6409
+
}
6410
+
6411
+
out_unlock:
6412
+
mutex_unlock(&direct_mutex);
6413
+
6414
+
if (old_direct_functions && old_direct_functions != EMPTY_HASH)
6415
+
call_rcu_tasks(&old_direct_functions->rcu, register_ftrace_direct_cb);
6416
+
free_ftrace_hash(new_filter_hash);
6417
+
6418
+
return err;
6419
+
}
6420
+
6281
6421
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
6282
6422
6283
6423
/**