Merge tag 'probes-v6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

+2 -1

include/linux/fprobe.h

··· 7 7 #include <linux/ftrace.h> 8 8 #include <linux/rcupdate.h> 9 9 #include <linux/refcount.h> 10 + #include <linux/rhashtable.h> 10 11 #include <linux/slab.h> 11 12 12 13 struct fprobe; ··· 27 26 * @fp: The fprobe which owns this. 28 27 */ 29 28 struct fprobe_hlist_node { 30 - struct hlist_node hlist; 29 + struct rhlist_head hlist; 31 30 unsigned long addr; 32 31 struct fprobe *fp; 33 32 };

+224 -79

kernel/trace/fprobe.c

··· 10 10 #include <linux/kprobes.h> 11 11 #include <linux/list.h> 12 12 #include <linux/mutex.h> 13 + #include <linux/rhashtable.h> 13 14 #include <linux/slab.h> 14 15 #include <linux/sort.h> 15 16 ··· 42 41 * - RCU hlist traversal under disabling preempt 43 42 */ 44 43 static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE]; 45 - static struct hlist_head fprobe_ip_table[FPROBE_IP_TABLE_SIZE]; 44 + static struct rhltable fprobe_ip_table; 46 45 static DEFINE_MUTEX(fprobe_mutex); 46 + static struct fgraph_ops fprobe_graph_ops; 47 47 48 - /* 49 - * Find first fprobe in the hlist. It will be iterated twice in the entry 50 - * probe, once for correcting the total required size, the second time is 51 - * calling back the user handlers. 52 - * Thus the hlist in the fprobe_table must be sorted and new probe needs to 53 - * be added *before* the first fprobe. 54 - */ 55 - static struct fprobe_hlist_node *find_first_fprobe_node(unsigned long ip) 48 + static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed) 56 49 { 57 - struct fprobe_hlist_node *node; 58 - struct hlist_head *head; 59 - 60 - head = &fprobe_ip_table[hash_ptr((void *)ip, FPROBE_IP_HASH_BITS)]; 61 - hlist_for_each_entry_rcu(node, head, hlist, 62 - lockdep_is_held(&fprobe_mutex)) { 63 - if (node->addr == ip) 64 - return node; 65 - } 66 - return NULL; 50 + return hash_ptr(*(unsigned long **)data, 32); 67 51 } 68 - NOKPROBE_SYMBOL(find_first_fprobe_node); 52 + 53 + static int fprobe_node_cmp(struct rhashtable_compare_arg *arg, 54 + const void *ptr) 55 + { 56 + unsigned long key = *(unsigned long *)arg->key; 57 + const struct fprobe_hlist_node *n = ptr; 58 + 59 + return n->addr != key; 60 + } 61 + 62 + static u32 fprobe_node_obj_hashfn(const void *data, u32 len, u32 seed) 63 + { 64 + const struct fprobe_hlist_node *n = data; 65 + 66 + return hash_ptr((void *)n->addr, 32); 67 + } 68 + 69 + static const struct rhashtable_params fprobe_rht_params = { 70 + .head_offset = offsetof(struct fprobe_hlist_node, hlist), 71 + .key_offset = offsetof(struct fprobe_hlist_node, addr), 72 + .key_len = sizeof_field(struct fprobe_hlist_node, addr), 73 + .hashfn = fprobe_node_hashfn, 74 + .obj_hashfn = fprobe_node_obj_hashfn, 75 + .obj_cmpfn = fprobe_node_cmp, 76 + .automatic_shrinking = true, 77 + }; 69 78 70 79 /* Node insertion and deletion requires the fprobe_mutex */ 71 - static void insert_fprobe_node(struct fprobe_hlist_node *node) 80 + static int insert_fprobe_node(struct fprobe_hlist_node *node) 72 81 { 73 - unsigned long ip = node->addr; 74 - struct fprobe_hlist_node *next; 75 - struct hlist_head *head; 76 - 77 82 lockdep_assert_held(&fprobe_mutex); 78 83 79 - next = find_first_fprobe_node(ip); 80 - if (next) { 81 - hlist_add_before_rcu(&node->hlist, &next->hlist); 82 - return; 83 - } 84 - head = &fprobe_ip_table[hash_ptr((void *)ip, FPROBE_IP_HASH_BITS)]; 85 - hlist_add_head_rcu(&node->hlist, head); 84 + return rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params); 86 85 } 87 86 88 87 /* Return true if there are synonims */ 89 88 static bool delete_fprobe_node(struct fprobe_hlist_node *node) 90 89 { 91 90 lockdep_assert_held(&fprobe_mutex); 91 + bool ret; 92 92 93 93 /* Avoid double deleting */ 94 94 if (READ_ONCE(node->fp) != NULL) { 95 95 WRITE_ONCE(node->fp, NULL); 96 - hlist_del_rcu(&node->hlist); 96 + rhltable_remove(&fprobe_ip_table, &node->hlist, 97 + fprobe_rht_params); 97 98 } 98 - return !!find_first_fprobe_node(node->addr); 99 + 100 + rcu_read_lock(); 101 + ret = !!rhltable_lookup(&fprobe_ip_table, &node->addr, 102 + fprobe_rht_params); 103 + rcu_read_unlock(); 104 + 105 + return ret; 99 106 } 100 107 101 108 /* Check existence of the fprobe */ ··· 255 246 return ret; 256 247 } 257 248 258 - static int fprobe_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops, 259 - struct ftrace_regs *fregs) 249 + #if defined(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) || defined(CONFIG_DYNAMIC_FTRACE_WITH_REGS) 250 + /* ftrace_ops callback, this processes fprobes which have only entry_handler. */ 251 + static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, 252 + struct ftrace_ops *ops, struct ftrace_regs *fregs) 260 253 { 261 - struct fprobe_hlist_node *node, *first; 254 + struct fprobe_hlist_node *node; 255 + struct rhlist_head *head, *pos; 256 + struct fprobe *fp; 257 + int bit; 258 + 259 + bit = ftrace_test_recursion_trylock(ip, parent_ip); 260 + if (bit < 0) 261 + return; 262 + 263 + /* 264 + * ftrace_test_recursion_trylock() disables preemption, but 265 + * rhltable_lookup() checks whether rcu_read_lcok is held. 266 + * So we take rcu_read_lock() here. 267 + */ 268 + rcu_read_lock(); 269 + head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params); 270 + 271 + rhl_for_each_entry_rcu(node, pos, head, hlist) { 272 + if (node->addr != ip) 273 + break; 274 + fp = READ_ONCE(node->fp); 275 + if (unlikely(!fp || fprobe_disabled(fp) || fp->exit_handler)) 276 + continue; 277 + 278 + if (fprobe_shared_with_kprobes(fp)) 279 + __fprobe_kprobe_handler(ip, parent_ip, fp, fregs, NULL); 280 + else 281 + __fprobe_handler(ip, parent_ip, fp, fregs, NULL); 282 + } 283 + rcu_read_unlock(); 284 + ftrace_test_recursion_unlock(bit); 285 + } 286 + NOKPROBE_SYMBOL(fprobe_ftrace_entry); 287 + 288 + static struct ftrace_ops fprobe_ftrace_ops = { 289 + .func = fprobe_ftrace_entry, 290 + .flags = FTRACE_OPS_FL_SAVE_ARGS, 291 + }; 292 + static int fprobe_ftrace_active; 293 + 294 + static int fprobe_ftrace_add_ips(unsigned long *addrs, int num) 295 + { 296 + int ret; 297 + 298 + lockdep_assert_held(&fprobe_mutex); 299 + 300 + ret = ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 0, 0); 301 + if (ret) 302 + return ret; 303 + 304 + if (!fprobe_ftrace_active) { 305 + ret = register_ftrace_function(&fprobe_ftrace_ops); 306 + if (ret) { 307 + ftrace_free_filter(&fprobe_ftrace_ops); 308 + return ret; 309 + } 310 + } 311 + fprobe_ftrace_active++; 312 + return 0; 313 + } 314 + 315 + static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num) 316 + { 317 + lockdep_assert_held(&fprobe_mutex); 318 + 319 + fprobe_ftrace_active--; 320 + if (!fprobe_ftrace_active) 321 + unregister_ftrace_function(&fprobe_ftrace_ops); 322 + if (num) 323 + ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 1, 0); 324 + } 325 + 326 + static bool fprobe_is_ftrace(struct fprobe *fp) 327 + { 328 + return !fp->exit_handler; 329 + } 330 + 331 + #ifdef CONFIG_MODULES 332 + static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove, 333 + int reset) 334 + { 335 + ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset); 336 + ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, remove, reset); 337 + } 338 + #endif 339 + #else 340 + static int fprobe_ftrace_add_ips(unsigned long *addrs, int num) 341 + { 342 + return -ENOENT; 343 + } 344 + 345 + static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num) 346 + { 347 + } 348 + 349 + static bool fprobe_is_ftrace(struct fprobe *fp) 350 + { 351 + return false; 352 + } 353 + 354 + #ifdef CONFIG_MODULES 355 + static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove, 356 + int reset) 357 + { 358 + ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset); 359 + } 360 + #endif 361 + #endif /* !CONFIG_DYNAMIC_FTRACE_WITH_ARGS && !CONFIG_DYNAMIC_FTRACE_WITH_REGS */ 362 + 363 + /* fgraph_ops callback, this processes fprobes which have exit_handler. */ 364 + static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops, 365 + struct ftrace_regs *fregs) 366 + { 262 367 unsigned long *fgraph_data = NULL; 263 368 unsigned long func = trace->func; 369 + struct fprobe_hlist_node *node; 370 + struct rhlist_head *head, *pos; 264 371 unsigned long ret_ip; 265 372 int reserved_words; 266 373 struct fprobe *fp; ··· 385 260 if (WARN_ON_ONCE(!fregs)) 386 261 return 0; 387 262 388 - first = node = find_first_fprobe_node(func); 389 - if (unlikely(!first)) 390 - return 0; 391 - 263 + guard(rcu)(); 264 + head = rhltable_lookup(&fprobe_ip_table, &func, fprobe_rht_params); 392 265 reserved_words = 0; 393 - hlist_for_each_entry_from_rcu(node, hlist) { 266 + rhl_for_each_entry_rcu(node, pos, head, hlist) { 394 267 if (node->addr != func) 395 - break; 268 + continue; 396 269 fp = READ_ONCE(node->fp); 397 270 if (!fp || !fp->exit_handler) 398 271 continue; ··· 401 278 reserved_words += 402 279 FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(fp->entry_data_size); 403 280 } 404 - node = first; 405 281 if (reserved_words) { 406 282 fgraph_data = fgraph_reserve_data(gops->idx, reserved_words * sizeof(long)); 407 283 if (unlikely(!fgraph_data)) { 408 - hlist_for_each_entry_from_rcu(node, hlist) { 284 + rhl_for_each_entry_rcu(node, pos, head, hlist) { 409 285 if (node->addr != func) 410 - break; 286 + continue; 411 287 fp = READ_ONCE(node->fp); 412 - if (fp && !fprobe_disabled(fp)) 288 + if (fp && !fprobe_disabled(fp) && !fprobe_is_ftrace(fp)) 413 289 fp->nmissed++; 414 290 } 415 291 return 0; ··· 421 299 */ 422 300 ret_ip = ftrace_regs_get_return_address(fregs); 423 301 used = 0; 424 - hlist_for_each_entry_from_rcu(node, hlist) { 302 + rhl_for_each_entry_rcu(node, pos, head, hlist) { 425 303 int data_size; 426 304 void *data; 427 305 428 306 if (node->addr != func) 429 - break; 307 + continue; 430 308 fp = READ_ONCE(node->fp); 431 - if (!fp || fprobe_disabled(fp)) 309 + if (unlikely(!fp || fprobe_disabled(fp) || fprobe_is_ftrace(fp))) 432 310 continue; 433 311 434 312 data_size = fp->entry_data_size; ··· 456 334 /* If any exit_handler is set, data must be used. */ 457 335 return used != 0; 458 336 } 459 - NOKPROBE_SYMBOL(fprobe_entry); 337 + NOKPROBE_SYMBOL(fprobe_fgraph_entry); 460 338 461 339 static void fprobe_return(struct ftrace_graph_ret *trace, 462 340 struct fgraph_ops *gops, ··· 495 373 NOKPROBE_SYMBOL(fprobe_return); 496 374 497 375 static struct fgraph_ops fprobe_graph_ops = { 498 - .entryfunc = fprobe_entry, 376 + .entryfunc = fprobe_fgraph_entry, 499 377 .retfunc = fprobe_return, 500 378 }; 501 379 static int fprobe_graph_active; ··· 571 449 return 0; 572 450 } 573 451 574 - static void fprobe_remove_node_in_module(struct module *mod, struct hlist_head *head, 575 - struct fprobe_addr_list *alist) 452 + static void fprobe_remove_node_in_module(struct module *mod, struct fprobe_hlist_node *node, 453 + struct fprobe_addr_list *alist) 576 454 { 577 - struct fprobe_hlist_node *node; 578 - int ret = 0; 579 - 580 - hlist_for_each_entry_rcu(node, head, hlist, 581 - lockdep_is_held(&fprobe_mutex)) { 582 - if (!within_module(node->addr, mod)) 583 - continue; 584 - if (delete_fprobe_node(node)) 585 - continue; 586 - /* 587 - * If failed to update alist, just continue to update hlist. 588 - * Therefore, at list user handler will not hit anymore. 589 - */ 590 - if (!ret) 591 - ret = fprobe_addr_list_add(alist, node->addr); 592 - } 455 + if (!within_module(node->addr, mod)) 456 + return; 457 + if (delete_fprobe_node(node)) 458 + return; 459 + /* 460 + * If failed to update alist, just continue to update hlist. 461 + * Therefore, at list user handler will not hit anymore. 462 + */ 463 + fprobe_addr_list_add(alist, node->addr); 593 464 } 594 465 595 466 /* Handle module unloading to manage fprobe_ip_table. */ ··· 590 475 unsigned long val, void *data) 591 476 { 592 477 struct fprobe_addr_list alist = {.size = FPROBE_IPS_BATCH_INIT}; 478 + struct fprobe_hlist_node *node; 479 + struct rhashtable_iter iter; 593 480 struct module *mod = data; 594 - int i; 595 481 596 482 if (val != MODULE_STATE_GOING) 597 483 return NOTIFY_DONE; ··· 603 487 return NOTIFY_DONE; 604 488 605 489 mutex_lock(&fprobe_mutex); 606 - for (i = 0; i < FPROBE_IP_TABLE_SIZE; i++) 607 - fprobe_remove_node_in_module(mod, &fprobe_ip_table[i], &alist); 490 + rhltable_walk_enter(&fprobe_ip_table, &iter); 491 + do { 492 + rhashtable_walk_start(&iter); 493 + 494 + while ((node = rhashtable_walk_next(&iter)) && !IS_ERR(node)) 495 + fprobe_remove_node_in_module(mod, node, &alist); 496 + 497 + rhashtable_walk_stop(&iter); 498 + } while (node == ERR_PTR(-EAGAIN)); 499 + rhashtable_walk_exit(&iter); 608 500 609 501 if (alist.index > 0) 610 - ftrace_set_filter_ips(&fprobe_graph_ops.ops, 611 - alist.addrs, alist.index, 1, 0); 502 + fprobe_set_ips(alist.addrs, alist.index, 1, 0); 612 503 mutex_unlock(&fprobe_mutex); 613 504 614 505 kfree(alist.addrs); ··· 848 725 mutex_lock(&fprobe_mutex); 849 726 850 727 hlist_array = fp->hlist_array; 851 - ret = fprobe_graph_add_ips(addrs, num); 728 + if (fprobe_is_ftrace(fp)) 729 + ret = fprobe_ftrace_add_ips(addrs, num); 730 + else 731 + ret = fprobe_graph_add_ips(addrs, num); 732 + 852 733 if (!ret) { 853 734 add_fprobe_hash(fp); 854 - for (i = 0; i < hlist_array->size; i++) 855 - insert_fprobe_node(&hlist_array->array[i]); 735 + for (i = 0; i < hlist_array->size; i++) { 736 + ret = insert_fprobe_node(&hlist_array->array[i]); 737 + if (ret) 738 + break; 739 + } 740 + /* fallback on insert error */ 741 + if (ret) { 742 + for (i--; i >= 0; i--) 743 + delete_fprobe_node(&hlist_array->array[i]); 744 + } 856 745 } 857 746 mutex_unlock(&fprobe_mutex); 858 747 ··· 948 813 } 949 814 del_fprobe_hash(fp); 950 815 951 - fprobe_graph_remove_ips(addrs, count); 816 + if (fprobe_is_ftrace(fp)) 817 + fprobe_ftrace_remove_ips(addrs, count); 818 + else 819 + fprobe_graph_remove_ips(addrs, count); 952 820 953 821 kfree_rcu(hlist_array, rcu); 954 822 fp->hlist_array = NULL; ··· 963 825 return ret; 964 826 } 965 827 EXPORT_SYMBOL_GPL(unregister_fprobe); 828 + 829 + static int __init fprobe_initcall(void) 830 + { 831 + rhltable_init(&fprobe_ip_table, &fprobe_rht_params); 832 + return 0; 833 + } 834 + core_initcall(fprobe_initcall);

+44 -64

kernel/trace/trace_eprobe.c

··· 61 61 kfree(ep); 62 62 } 63 63 64 + DEFINE_FREE(trace_event_probe_cleanup, struct trace_eprobe *, 65 + if (!IS_ERR_OR_NULL(_T)) trace_event_probe_cleanup(_T)) 66 + 64 67 static struct trace_eprobe *to_trace_eprobe(struct dyn_event *ev) 65 68 { 66 69 return container_of(ev, struct trace_eprobe, devent); ··· 200 197 struct trace_event_call *event, 201 198 int nargs) 202 199 { 203 - struct trace_eprobe *ep; 200 + struct trace_eprobe *ep __free(trace_event_probe_cleanup) = NULL; 204 201 const char *event_name; 205 202 const char *sys_name; 206 - int ret = -ENOMEM; 203 + int ret; 207 204 208 205 if (!event) 209 206 return ERR_PTR(-ENODEV); ··· 214 211 ep = kzalloc(struct_size(ep, tp.args, nargs), GFP_KERNEL); 215 212 if (!ep) { 216 213 trace_event_put_ref(event); 217 - goto error; 214 + return ERR_PTR(-ENOMEM); 218 215 } 219 216 ep->event = event; 220 217 ep->event_name = kstrdup(event_name, GFP_KERNEL); 221 218 if (!ep->event_name) 222 - goto error; 219 + return ERR_PTR(-ENOMEM); 223 220 ep->event_system = kstrdup(sys_name, GFP_KERNEL); 224 221 if (!ep->event_system) 225 - goto error; 222 + return ERR_PTR(-ENOMEM); 226 223 227 224 ret = trace_probe_init(&ep->tp, this_event, group, false, nargs); 228 225 if (ret < 0) 229 - goto error; 226 + return ERR_PTR(ret); 230 227 231 228 dyn_event_init(&ep->devent, &eprobe_dyn_event_ops); 232 - return ep; 233 - error: 234 - trace_event_probe_cleanup(ep); 235 - return ERR_PTR(ret); 229 + return_ptr(ep); 236 230 } 237 231 238 232 static int eprobe_event_define_fields(struct trace_event_call *event_call) ··· 790 790 return NULL; 791 791 } 792 792 793 - static int trace_eprobe_tp_update_arg(struct trace_eprobe *ep, const char *argv[], int i) 794 - { 795 - struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL; 796 - int ret; 797 - 798 - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 799 - if (!ctx) 800 - return -ENOMEM; 801 - ctx->event = ep->event; 802 - ctx->flags = TPARG_FL_KERNEL | TPARG_FL_TEVENT; 803 - 804 - ret = traceprobe_parse_probe_arg(&ep->tp, i, argv[i], ctx); 805 - /* Handle symbols "@" */ 806 - if (!ret) 807 - ret = traceprobe_update_arg(&ep->tp.args[i]); 808 - 809 - return ret; 810 - } 811 - 812 793 static int trace_eprobe_parse_filter(struct trace_eprobe *ep, int argc, const char *argv[]) 813 794 { 814 795 struct event_filter *dummy = NULL; ··· 826 845 ret = create_event_filter(top_trace_array(), ep->event, ep->filter_str, 827 846 true, &dummy); 828 847 free_event_filter(dummy); 829 - if (ret) 830 - goto error; 831 - 832 - return 0; 833 - error: 834 - kfree(ep->filter_str); 835 - ep->filter_str = NULL; 848 + if (ret) { 849 + kfree(ep->filter_str); 850 + ep->filter_str = NULL; 851 + } 836 852 return ret; 837 853 } 838 854 ··· 841 863 * Fetch args (no space): 842 864 * <name>=$<field>[:TYPE] 843 865 */ 866 + struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL; 867 + struct trace_eprobe *ep __free(trace_event_probe_cleanup) = NULL; 868 + const char *trlog __free(trace_probe_log_clear) = NULL; 844 869 const char *event = NULL, *group = EPROBE_EVENT_SYSTEM; 845 870 const char *sys_event = NULL, *sys_name = NULL; 846 871 struct trace_event_call *event_call; 847 872 char *buf1 __free(kfree) = NULL; 848 873 char *buf2 __free(kfree) = NULL; 849 874 char *gbuf __free(kfree) = NULL; 850 - struct trace_eprobe *ep = NULL; 851 875 int ret = 0, filter_idx = 0; 852 876 int i, filter_cnt; 853 877 854 878 if (argc < 2 || argv[0][0] != 'e') 855 879 return -ECANCELED; 856 880 857 - trace_probe_log_init("event_probe", argc, argv); 881 + trlog = trace_probe_log_init("event_probe", argc, argv); 858 882 859 883 event = strchr(&argv[0][1], ':'); 860 884 if (event) { 861 885 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 862 886 if (!gbuf) 863 - goto mem_error; 887 + return -ENOMEM; 864 888 event++; 865 889 ret = traceprobe_parse_event_name(&event, &group, gbuf, 866 890 event - argv[0]); 867 891 if (ret) 868 - goto parse_error; 892 + return -EINVAL; 869 893 } 870 894 871 895 trace_probe_log_set_index(1); ··· 875 895 876 896 buf2 = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 877 897 if (!buf2) 878 - goto mem_error; 898 + return -ENOMEM; 879 899 880 900 ret = traceprobe_parse_event_name(&sys_event, &sys_name, buf2, 0); 881 901 if (ret || !sys_event || !sys_name) { 882 902 trace_probe_log_err(0, NO_EVENT_INFO); 883 - goto parse_error; 903 + return -EINVAL; 884 904 } 885 905 886 906 if (!event) { 887 907 buf1 = kstrdup(sys_event, GFP_KERNEL); 888 908 if (!buf1) 889 - goto mem_error; 909 + return -ENOMEM; 890 910 event = buf1; 891 911 } 892 912 ··· 902 922 if (argc - 2 > MAX_TRACE_ARGS) { 903 923 trace_probe_log_set_index(2); 904 924 trace_probe_log_err(0, TOO_MANY_ARGS); 905 - ret = -E2BIG; 906 - goto error; 925 + return -E2BIG; 907 926 } 908 927 909 928 scoped_guard(mutex, &event_mutex) { ··· 916 937 trace_probe_log_err(0, BAD_ATTACH_EVENT); 917 938 /* This must return -ENOMEM or missing event, else there is a bug */ 918 939 WARN_ON_ONCE(ret != -ENOMEM && ret != -ENODEV); 919 - ep = NULL; 920 - goto error; 940 + return ret; 921 941 } 922 942 923 943 if (filter_idx) { 924 944 trace_probe_log_set_index(filter_idx); 925 945 ret = trace_eprobe_parse_filter(ep, filter_cnt, argv + filter_idx); 926 946 if (ret) 927 - goto parse_error; 947 + return -EINVAL; 928 948 } else 929 949 ep->filter_str = NULL; 950 + 951 + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 952 + if (!ctx) 953 + return -ENOMEM; 954 + ctx->event = ep->event; 955 + ctx->flags = TPARG_FL_KERNEL | TPARG_FL_TEVENT; 930 956 931 957 argc -= 2; argv += 2; 932 958 /* parse arguments */ 933 959 for (i = 0; i < argc; i++) { 934 960 trace_probe_log_set_index(i + 2); 935 - ret = trace_eprobe_tp_update_arg(ep, argv, i); 961 + 962 + ret = traceprobe_parse_probe_arg(&ep->tp, i, argv[i], ctx); 963 + /* Handle symbols "@" */ 964 + if (!ret) 965 + ret = traceprobe_update_arg(&ep->tp.args[i]); 936 966 if (ret) 937 - goto error; 967 + return ret; 938 968 } 939 969 ret = traceprobe_set_print_fmt(&ep->tp, PROBE_PRINT_EVENT); 940 970 if (ret < 0) 941 - goto error; 971 + return ret; 972 + 942 973 init_trace_eprobe_call(ep); 943 974 scoped_guard(mutex, &event_mutex) { 944 975 ret = trace_probe_register_event_call(&ep->tp); ··· 957 968 trace_probe_log_set_index(0); 958 969 trace_probe_log_err(0, EVENT_EXIST); 959 970 } 960 - goto error; 971 + return ret; 961 972 } 962 973 ret = dyn_event_add(&ep->devent, &ep->tp.event->call); 963 974 if (ret < 0) { 964 975 trace_probe_unregister_event_call(&ep->tp); 965 - goto error; 976 + return ret; 966 977 } 978 + /* To avoid freeing registered eprobe event, clear ep. */ 979 + ep = NULL; 967 980 } 968 - trace_probe_log_clear(); 969 - return ret; 970 - 971 - mem_error: 972 - ret = -ENOMEM; 973 - goto error; 974 - parse_error: 975 - ret = -EINVAL; 976 - error: 977 - trace_probe_log_clear(); 978 - trace_event_probe_cleanup(ep); 979 981 return ret; 980 982 } 981 983

+3 -2

kernel/trace/trace_probe.c

··· 156 156 static struct trace_probe_log trace_probe_log; 157 157 extern struct mutex dyn_event_ops_mutex; 158 158 159 - void trace_probe_log_init(const char *subsystem, int argc, const char **argv) 159 + const char *trace_probe_log_init(const char *subsystem, int argc, const char **argv) 160 160 { 161 161 lockdep_assert_held(&dyn_event_ops_mutex); 162 162 ··· 164 164 trace_probe_log.argc = argc; 165 165 trace_probe_log.argv = argv; 166 166 trace_probe_log.index = 0; 167 + return subsystem; 167 168 } 168 169 169 170 void trace_probe_log_clear(void) ··· 215 214 p = command; 216 215 for (i = 0; i < trace_probe_log.argc; i++) { 217 216 len = strlen(trace_probe_log.argv[i]); 218 - strcpy(p, trace_probe_log.argv[i]); 217 + memcpy(p, trace_probe_log.argv[i], len); 219 218 p[len] = ' '; 220 219 p += len + 1; 221 220 }

+3 -1

kernel/trace/trace_probe.h

··· 578 578 int index; 579 579 }; 580 580 581 - void trace_probe_log_init(const char *subsystem, int argc, const char **argv); 581 + const char *trace_probe_log_init(const char *subsystem, int argc, const char **argv); 582 582 void trace_probe_log_set_index(int index); 583 583 void trace_probe_log_clear(void); 584 584 void __trace_probe_log_err(int offset, int err); 585 + 586 + DEFINE_FREE(trace_probe_log_clear, const char *, if (_T) trace_probe_log_clear()) 585 587 586 588 #define trace_probe_log_err(offs, err) \ 587 589 __trace_probe_log_err(offs, TP_ERR_##err)

+32 -50

kernel/trace/trace_uprobe.c

··· 533 533 return ret; 534 534 } 535 535 536 + DEFINE_FREE(free_trace_uprobe, struct trace_uprobe *, if (_T) free_trace_uprobe(_T)) 537 + 536 538 /* 537 539 * Argument syntax: 538 540 * - Add uprobe: p|r[:[GRP/][EVENT]] PATH:OFFSET[%return][(REF)] [FETCHARGS] 539 541 */ 540 542 static int __trace_uprobe_create(int argc, const char **argv) 541 543 { 544 + struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL; 545 + struct trace_uprobe *tu __free(free_trace_uprobe) = NULL; 546 + const char *trlog __free(trace_probe_log_clear) = NULL; 542 547 const char *event = NULL, *group = UPROBE_EVENT_SYSTEM; 543 - char *arg, *filename, *rctr, *rctr_end, *tmp; 548 + struct path path __free(path_put) = {}; 544 549 unsigned long offset, ref_ctr_offset; 550 + char *filename __free(kfree) = NULL; 551 + char *arg, *rctr, *rctr_end, *tmp; 545 552 char *gbuf __free(kfree) = NULL; 546 553 char *buf __free(kfree) = NULL; 547 554 enum probe_print_type ptype; 548 - struct trace_uprobe *tu; 549 555 bool is_return = false; 550 - struct path path; 551 556 int i, ret; 552 557 553 558 ref_ctr_offset = 0; ··· 570 565 if (argc < 2) 571 566 return -ECANCELED; 572 567 573 - trace_probe_log_init("trace_uprobe", argc, argv); 568 + trlog = trace_probe_log_init("trace_uprobe", argc, argv); 574 569 575 570 if (argc - 2 > MAX_TRACE_ARGS) { 576 571 trace_probe_log_set_index(2); ··· 590 585 591 586 /* Find the last occurrence, in case the path contains ':' too. */ 592 587 arg = strrchr(filename, ':'); 593 - if (!arg || !isdigit(arg[1])) { 594 - kfree(filename); 588 + if (!arg || !isdigit(arg[1])) 595 589 return -ECANCELED; 596 - } 597 590 598 591 trace_probe_log_set_index(1); /* filename is the 2nd argument */ 599 592 ··· 599 596 ret = kern_path(filename, LOOKUP_FOLLOW, &path); 600 597 if (ret) { 601 598 trace_probe_log_err(0, FILE_NOT_FOUND); 602 - kfree(filename); 603 - trace_probe_log_clear(); 604 599 return ret; 605 600 } 606 601 if (!d_is_reg(path.dentry)) { 607 602 trace_probe_log_err(0, NO_REGULAR_FILE); 608 - ret = -EINVAL; 609 - goto fail_address_parse; 603 + return -EINVAL; 610 604 } 611 605 612 606 /* Parse reference counter offset if specified. */ ··· 611 611 if (rctr) { 612 612 rctr_end = strchr(rctr, ')'); 613 613 if (!rctr_end) { 614 - ret = -EINVAL; 615 614 rctr_end = rctr + strlen(rctr); 616 615 trace_probe_log_err(rctr_end - filename, 617 616 REFCNT_OPEN_BRACE); 618 - goto fail_address_parse; 617 + return -EINVAL; 619 618 } else if (rctr_end[1] != '\0') { 620 - ret = -EINVAL; 621 619 trace_probe_log_err(rctr_end + 1 - filename, 622 620 BAD_REFCNT_SUFFIX); 623 - goto fail_address_parse; 621 + return -EINVAL; 624 622 } 625 623 626 624 *rctr++ = '\0'; ··· 626 628 ret = kstrtoul(rctr, 0, &ref_ctr_offset); 627 629 if (ret) { 628 630 trace_probe_log_err(rctr - filename, BAD_REFCNT); 629 - goto fail_address_parse; 631 + return ret; 630 632 } 631 633 } 632 634 ··· 638 640 is_return = true; 639 641 } else { 640 642 trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX); 641 - ret = -EINVAL; 642 - goto fail_address_parse; 643 + return -EINVAL; 643 644 } 644 645 } 645 646 ··· 646 649 ret = kstrtoul(arg, 0, &offset); 647 650 if (ret) { 648 651 trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS); 649 - goto fail_address_parse; 652 + return ret; 650 653 } 651 654 652 655 /* setup a probe */ ··· 654 657 if (event) { 655 658 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 656 659 if (!gbuf) 657 - goto fail_mem; 660 + return -ENOMEM; 658 661 659 662 ret = traceprobe_parse_event_name(&event, &group, gbuf, 660 663 event - argv[0]); 661 664 if (ret) 662 - goto fail_address_parse; 665 + return ret; 663 666 } 664 667 665 668 if (!event) { ··· 668 671 669 672 tail = kstrdup(kbasename(filename), GFP_KERNEL); 670 673 if (!tail) 671 - goto fail_mem; 674 + return -ENOMEM; 672 675 673 676 ptr = strpbrk(tail, ".-_"); 674 677 if (ptr) ··· 676 679 677 680 buf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 678 681 if (!buf) 679 - goto fail_mem; 682 + return -ENOMEM; 680 683 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset); 681 684 event = buf; 682 685 kfree(tail); ··· 690 693 ret = PTR_ERR(tu); 691 694 /* This must return -ENOMEM otherwise there is a bug */ 692 695 WARN_ON_ONCE(ret != -ENOMEM); 693 - goto fail_address_parse; 696 + return ret; 694 697 } 695 698 tu->offset = offset; 696 699 tu->ref_ctr_offset = ref_ctr_offset; 697 700 tu->path = path; 698 - tu->filename = filename; 701 + /* Clear @path so that it will not freed by path_put() */ 702 + memset(&path, 0, sizeof(path)); 703 + tu->filename = no_free_ptr(filename); 704 + 705 + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 706 + if (!ctx) 707 + return -ENOMEM; 708 + ctx->flags = (is_return ? TPARG_FL_RETURN : 0) | TPARG_FL_USER; 699 709 700 710 /* parse arguments */ 701 711 for (i = 0; i < argc; i++) { 702 - struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) 703 - = kzalloc(sizeof(*ctx), GFP_KERNEL); 704 - 705 - if (!ctx) { 706 - ret = -ENOMEM; 707 - goto error; 708 - } 709 - ctx->flags = (is_return ? TPARG_FL_RETURN : 0) | TPARG_FL_USER; 710 712 trace_probe_log_set_index(i + 2); 711 713 ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], ctx); 712 714 if (ret) 713 - goto error; 715 + return ret; 714 716 } 715 717 716 718 ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 717 719 ret = traceprobe_set_print_fmt(&tu->tp, ptype); 718 720 if (ret < 0) 719 - goto error; 721 + return ret; 720 722 721 723 ret = register_trace_uprobe(tu); 722 724 if (!ret) 723 - goto out; 724 - 725 - error: 726 - free_trace_uprobe(tu); 727 - out: 728 - trace_probe_log_clear(); 729 - return ret; 730 - 731 - fail_mem: 732 - ret = -ENOMEM; 733 - 734 - fail_address_parse: 735 - trace_probe_log_clear(); 736 - path_put(&path); 737 - kfree(filename); 725 + tu = NULL; 738 726 739 727 return ret; 740 728 }

+98 -1

lib/tests/test_fprobe.c

··· 12 12 13 13 static struct kunit *current_test; 14 14 15 - static u32 rand1, entry_val, exit_val; 15 + static u32 rand1, entry_only_val, entry_val, exit_val; 16 + static u32 entry_only_count, entry_count, exit_count; 16 17 17 18 /* Use indirect calls to avoid inlining the target functions */ 18 19 static u32 (*target)(u32 value); ··· 191 190 KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp)); 192 191 } 193 192 193 + /* Handler for fprobe entry only case */ 194 + static notrace int entry_only_handler(struct fprobe *fp, unsigned long ip, 195 + unsigned long ret_ip, 196 + struct ftrace_regs *fregs, void *data) 197 + { 198 + KUNIT_EXPECT_FALSE(current_test, preemptible()); 199 + KUNIT_EXPECT_EQ(current_test, ip, target_ip); 200 + 201 + entry_only_count++; 202 + entry_only_val = (rand1 / div_factor); 203 + 204 + return 0; 205 + } 206 + 207 + static notrace int fprobe_entry_multi_handler(struct fprobe *fp, unsigned long ip, 208 + unsigned long ret_ip, 209 + struct ftrace_regs *fregs, 210 + void *data) 211 + { 212 + KUNIT_EXPECT_FALSE(current_test, preemptible()); 213 + KUNIT_EXPECT_EQ(current_test, ip, target_ip); 214 + 215 + entry_count++; 216 + entry_val = (rand1 / div_factor); 217 + 218 + return 0; 219 + } 220 + 221 + static notrace void fprobe_exit_multi_handler(struct fprobe *fp, unsigned long ip, 222 + unsigned long ret_ip, 223 + struct ftrace_regs *fregs, 224 + void *data) 225 + { 226 + unsigned long ret = ftrace_regs_get_return_value(fregs); 227 + 228 + KUNIT_EXPECT_FALSE(current_test, preemptible()); 229 + KUNIT_EXPECT_EQ(current_test, ip, target_ip); 230 + KUNIT_EXPECT_EQ(current_test, ret, (rand1 / div_factor)); 231 + 232 + exit_count++; 233 + exit_val = ret; 234 + } 235 + 236 + static void check_fprobe_multi(struct kunit *test) 237 + { 238 + entry_only_count = entry_count = exit_count = 0; 239 + entry_only_val = entry_val = exit_val = 0; 240 + 241 + target(rand1); 242 + 243 + /* Verify all handlers were called */ 244 + KUNIT_EXPECT_EQ(test, 1, entry_only_count); 245 + KUNIT_EXPECT_EQ(test, 1, entry_count); 246 + KUNIT_EXPECT_EQ(test, 1, exit_count); 247 + 248 + /* Verify values are correct */ 249 + KUNIT_EXPECT_EQ(test, (rand1 / div_factor), entry_only_val); 250 + KUNIT_EXPECT_EQ(test, (rand1 / div_factor), entry_val); 251 + KUNIT_EXPECT_EQ(test, (rand1 / div_factor), exit_val); 252 + } 253 + 254 + /* Test multiple fprobes hooking the same target function */ 255 + static void test_fprobe_multi(struct kunit *test) 256 + { 257 + struct fprobe fp1 = { 258 + .entry_handler = fprobe_entry_multi_handler, 259 + .exit_handler = fprobe_exit_multi_handler, 260 + }; 261 + struct fprobe fp2 = { 262 + .entry_handler = entry_only_handler, 263 + }; 264 + 265 + current_test = test; 266 + 267 + /* Test Case 1: Register in order 1 -> 2 */ 268 + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp1, "fprobe_selftest_target", NULL)); 269 + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp2, "fprobe_selftest_target", NULL)); 270 + 271 + check_fprobe_multi(test); 272 + 273 + /* Unregister all */ 274 + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp1)); 275 + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp2)); 276 + 277 + /* Test Case 2: Register in order 2 -> 1 */ 278 + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp2, "fprobe_selftest_target", NULL)); 279 + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp1, "fprobe_selftest_target", NULL)); 280 + 281 + check_fprobe_multi(test); 282 + 283 + /* Unregister all */ 284 + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp1)); 285 + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp2)); 286 + } 287 + 194 288 static unsigned long get_ftrace_location(void *func) 195 289 { 196 290 unsigned long size, addr = (unsigned long)func; ··· 313 217 KUNIT_CASE(test_fprobe_syms), 314 218 KUNIT_CASE(test_fprobe_data), 315 219 KUNIT_CASE(test_fprobe_skip), 220 + KUNIT_CASE(test_fprobe_multi), 316 221 {} 317 222 }; 318 223

Configure Feed

Configure Feed