Merge tag 'probes-v6.14' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

+3

include/linux/jump_label.h

··· 75 75 76 76 #include <linux/types.h> 77 77 #include <linux/compiler.h> 78 + #include <linux/cleanup.h> 78 79 79 80 extern bool static_key_initialized; 80 81 ··· 347 346 #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) } 348 347 349 348 #endif /* CONFIG_JUMP_LABEL */ 349 + 350 + DEFINE_LOCK_GUARD_0(jump_label_lock, jump_label_lock(), jump_label_unlock()) 350 351 351 352 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE 352 353 #define jump_label_enabled static_key_enabled

+268 -322

kernel/kprobes.c

··· 39 39 #include <linux/static_call.h> 40 40 #include <linux/perf_event.h> 41 41 #include <linux/execmem.h> 42 + #include <linux/cleanup.h> 42 43 43 44 #include <asm/sections.h> 44 45 #include <asm/cacheflush.h> ··· 141 140 kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c) 142 141 { 143 142 struct kprobe_insn_page *kip; 144 - kprobe_opcode_t *slot = NULL; 145 143 146 144 /* Since the slot array is not protected by rcu, we need a mutex */ 147 - mutex_lock(&c->mutex); 148 - retry: 149 - rcu_read_lock(); 150 - list_for_each_entry_rcu(kip, &c->pages, list) { 151 - if (kip->nused < slots_per_page(c)) { 152 - int i; 145 + guard(mutex)(&c->mutex); 146 + do { 147 + guard(rcu)(); 148 + list_for_each_entry_rcu(kip, &c->pages, list) { 149 + if (kip->nused < slots_per_page(c)) { 150 + int i; 153 151 154 - for (i = 0; i < slots_per_page(c); i++) { 155 - if (kip->slot_used[i] == SLOT_CLEAN) { 156 - kip->slot_used[i] = SLOT_USED; 157 - kip->nused++; 158 - slot = kip->insns + (i * c->insn_size); 159 - rcu_read_unlock(); 160 - goto out; 152 + for (i = 0; i < slots_per_page(c); i++) { 153 + if (kip->slot_used[i] == SLOT_CLEAN) { 154 + kip->slot_used[i] = SLOT_USED; 155 + kip->nused++; 156 + return kip->insns + (i * c->insn_size); 157 + } 161 158 } 159 + /* kip->nused is broken. Fix it. */ 160 + kip->nused = slots_per_page(c); 161 + WARN_ON(1); 162 162 } 163 - /* kip->nused is broken. Fix it. */ 164 - kip->nused = slots_per_page(c); 165 - WARN_ON(1); 166 163 } 167 - } 168 - rcu_read_unlock(); 169 - 170 164 /* If there are any garbage slots, collect it and try again. */ 171 - if (c->nr_garbage && collect_garbage_slots(c) == 0) 172 - goto retry; 165 + } while (c->nr_garbage && collect_garbage_slots(c) == 0); 173 166 174 167 /* All out of space. Need to allocate a new page. */ 175 168 kip = kmalloc(struct_size(kip, slot_used, slots_per_page(c)), GFP_KERNEL); 176 169 if (!kip) 177 - goto out; 170 + return NULL; 178 171 179 172 kip->insns = c->alloc(); 180 173 if (!kip->insns) { 181 174 kfree(kip); 182 - goto out; 175 + return NULL; 183 176 } 184 177 INIT_LIST_HEAD(&kip->list); 185 178 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c)); ··· 182 187 kip->ngarbage = 0; 183 188 kip->cache = c; 184 189 list_add_rcu(&kip->list, &c->pages); 185 - slot = kip->insns; 186 190 187 191 /* Record the perf ksymbol register event after adding the page */ 188 192 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns, 189 193 PAGE_SIZE, false, c->sym); 190 - out: 191 - mutex_unlock(&c->mutex); 192 - return slot; 194 + 195 + return kip->insns; 193 196 } 194 197 195 198 /* Return true if all garbages are collected, otherwise false. */ ··· 242 249 return 0; 243 250 } 244 251 245 - void __free_insn_slot(struct kprobe_insn_cache *c, 246 - kprobe_opcode_t *slot, int dirty) 252 + static long __find_insn_page(struct kprobe_insn_cache *c, 253 + kprobe_opcode_t *slot, struct kprobe_insn_page **pkip) 247 254 { 248 - struct kprobe_insn_page *kip; 255 + struct kprobe_insn_page *kip = NULL; 249 256 long idx; 250 257 251 - mutex_lock(&c->mutex); 252 - rcu_read_lock(); 258 + guard(rcu)(); 253 259 list_for_each_entry_rcu(kip, &c->pages, list) { 254 260 idx = ((long)slot - (long)kip->insns) / 255 261 (c->insn_size * sizeof(kprobe_opcode_t)); 256 - if (idx >= 0 && idx < slots_per_page(c)) 257 - goto out; 262 + if (idx >= 0 && idx < slots_per_page(c)) { 263 + *pkip = kip; 264 + return idx; 265 + } 258 266 } 259 267 /* Could not find this slot. */ 260 268 WARN_ON(1); 261 - kip = NULL; 262 - out: 263 - rcu_read_unlock(); 269 + *pkip = NULL; 270 + return -1; 271 + } 272 + 273 + void __free_insn_slot(struct kprobe_insn_cache *c, 274 + kprobe_opcode_t *slot, int dirty) 275 + { 276 + struct kprobe_insn_page *kip = NULL; 277 + long idx; 278 + 279 + guard(mutex)(&c->mutex); 280 + idx = __find_insn_page(c, slot, &kip); 264 281 /* Mark and sweep: this may sleep */ 265 282 if (kip) { 266 283 /* Check double free */ ··· 284 281 collect_one_slot(kip, idx); 285 282 } 286 283 } 287 - mutex_unlock(&c->mutex); 288 284 } 289 285 290 286 /* ··· 602 600 /* Kprobe jump optimizer */ 603 601 static void kprobe_optimizer(struct work_struct *work) 604 602 { 605 - mutex_lock(&kprobe_mutex); 606 - cpus_read_lock(); 607 - mutex_lock(&text_mutex); 603 + guard(mutex)(&kprobe_mutex); 608 604 609 - /* 610 - * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed) 611 - * kprobes before waiting for quiesence period. 612 - */ 613 - do_unoptimize_kprobes(); 605 + scoped_guard(cpus_read_lock) { 606 + guard(mutex)(&text_mutex); 614 607 615 - /* 616 - * Step 2: Wait for quiesence period to ensure all potentially 617 - * preempted tasks to have normally scheduled. Because optprobe 618 - * may modify multiple instructions, there is a chance that Nth 619 - * instruction is preempted. In that case, such tasks can return 620 - * to 2nd-Nth byte of jump instruction. This wait is for avoiding it. 621 - * Note that on non-preemptive kernel, this is transparently converted 622 - * to synchronoze_sched() to wait for all interrupts to have completed. 623 - */ 624 - synchronize_rcu_tasks(); 608 + /* 609 + * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed) 610 + * kprobes before waiting for quiesence period. 611 + */ 612 + do_unoptimize_kprobes(); 625 613 626 - /* Step 3: Optimize kprobes after quiesence period */ 627 - do_optimize_kprobes(); 614 + /* 615 + * Step 2: Wait for quiesence period to ensure all potentially 616 + * preempted tasks to have normally scheduled. Because optprobe 617 + * may modify multiple instructions, there is a chance that Nth 618 + * instruction is preempted. In that case, such tasks can return 619 + * to 2nd-Nth byte of jump instruction. This wait is for avoiding it. 620 + * Note that on non-preemptive kernel, this is transparently converted 621 + * to synchronoze_sched() to wait for all interrupts to have completed. 622 + */ 623 + synchronize_rcu_tasks(); 628 624 629 - /* Step 4: Free cleaned kprobes after quiesence period */ 630 - do_free_cleaned_kprobes(); 625 + /* Step 3: Optimize kprobes after quiesence period */ 626 + do_optimize_kprobes(); 631 627 632 - mutex_unlock(&text_mutex); 633 - cpus_read_unlock(); 628 + /* Step 4: Free cleaned kprobes after quiesence period */ 629 + do_free_cleaned_kprobes(); 630 + } 634 631 635 632 /* Step 5: Kick optimizer again if needed */ 636 633 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) 637 634 kick_kprobe_optimizer(); 638 - 639 - mutex_unlock(&kprobe_mutex); 640 635 } 641 636 642 - /* Wait for completing optimization and unoptimization */ 643 - void wait_for_kprobe_optimizer(void) 637 + static void wait_for_kprobe_optimizer_locked(void) 644 638 { 645 - mutex_lock(&kprobe_mutex); 639 + lockdep_assert_held(&kprobe_mutex); 646 640 647 641 while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) { 648 642 mutex_unlock(&kprobe_mutex); ··· 650 652 651 653 mutex_lock(&kprobe_mutex); 652 654 } 655 + } 653 656 654 - mutex_unlock(&kprobe_mutex); 657 + /* Wait for completing optimization and unoptimization */ 658 + void wait_for_kprobe_optimizer(void) 659 + { 660 + guard(mutex)(&kprobe_mutex); 661 + 662 + wait_for_kprobe_optimizer_locked(); 655 663 } 656 664 657 665 bool optprobe_queued_unopt(struct optimized_kprobe *op) ··· 856 852 return; 857 853 858 854 /* For preparing optimization, jump_label_text_reserved() is called. */ 859 - cpus_read_lock(); 860 - jump_label_lock(); 861 - mutex_lock(&text_mutex); 855 + guard(cpus_read_lock)(); 856 + guard(jump_label_lock)(); 857 + guard(mutex)(&text_mutex); 862 858 863 859 ap = alloc_aggr_kprobe(p); 864 860 if (!ap) 865 - goto out; 861 + return; 866 862 867 863 op = container_of(ap, struct optimized_kprobe, kp); 868 864 if (!arch_prepared_optinsn(&op->optinsn)) { 869 865 /* If failed to setup optimizing, fallback to kprobe. */ 870 866 arch_remove_optimized_kprobe(op); 871 867 kfree(op); 872 - goto out; 868 + return; 873 869 } 874 870 875 871 init_aggr_kprobe(ap, p); 876 872 optimize_kprobe(ap); /* This just kicks optimizer thread. */ 877 - 878 - out: 879 - mutex_unlock(&text_mutex); 880 - jump_label_unlock(); 881 - cpus_read_unlock(); 882 873 } 883 874 884 875 static void optimize_all_kprobes(void) ··· 882 883 struct kprobe *p; 883 884 unsigned int i; 884 885 885 - mutex_lock(&kprobe_mutex); 886 + guard(mutex)(&kprobe_mutex); 886 887 /* If optimization is already allowed, just return. */ 887 888 if (kprobes_allow_optimization) 888 - goto out; 889 + return; 889 890 890 891 cpus_read_lock(); 891 892 kprobes_allow_optimization = true; ··· 897 898 } 898 899 cpus_read_unlock(); 899 900 pr_info("kprobe jump-optimization is enabled. All kprobes are optimized if possible.\n"); 900 - out: 901 - mutex_unlock(&kprobe_mutex); 902 901 } 903 902 904 903 #ifdef CONFIG_SYSCTL ··· 906 909 struct kprobe *p; 907 910 unsigned int i; 908 911 909 - mutex_lock(&kprobe_mutex); 912 + guard(mutex)(&kprobe_mutex); 910 913 /* If optimization is already prohibited, just return. */ 911 - if (!kprobes_allow_optimization) { 912 - mutex_unlock(&kprobe_mutex); 914 + if (!kprobes_allow_optimization) 913 915 return; 914 - } 915 916 916 917 cpus_read_lock(); 917 918 kprobes_allow_optimization = false; ··· 921 926 } 922 927 } 923 928 cpus_read_unlock(); 924 - mutex_unlock(&kprobe_mutex); 925 - 926 929 /* Wait for unoptimizing completion. */ 927 - wait_for_kprobe_optimizer(); 930 + wait_for_kprobe_optimizer_locked(); 928 931 pr_info("kprobe jump-optimization is disabled. All kprobes are based on software breakpoint.\n"); 929 932 } 930 933 ··· 934 941 { 935 942 int ret; 936 943 937 - mutex_lock(&kprobe_sysctl_mutex); 944 + guard(mutex)(&kprobe_sysctl_mutex); 938 945 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0; 939 946 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 940 947 ··· 942 949 optimize_all_kprobes(); 943 950 else 944 951 unoptimize_all_kprobes(); 945 - mutex_unlock(&kprobe_sysctl_mutex); 946 952 947 953 return ret; 948 954 } ··· 1016 1024 #define __arm_kprobe(p) arch_arm_kprobe(p) 1017 1025 #define __disarm_kprobe(p, o) arch_disarm_kprobe(p) 1018 1026 #define kprobe_disarmed(p) kprobe_disabled(p) 1019 - #define wait_for_kprobe_optimizer() do {} while (0) 1027 + #define wait_for_kprobe_optimizer_locked() \ 1028 + lockdep_assert_held(&kprobe_mutex) 1020 1029 1021 1030 static int reuse_unused_kprobe(struct kprobe *ap) 1022 1031 { ··· 1071 1078 1072 1079 if (*cnt == 0) { 1073 1080 ret = register_ftrace_function(ops); 1074 - if (WARN(ret < 0, "Failed to register kprobe-ftrace (error %d)\n", ret)) 1075 - goto err_ftrace; 1081 + if (WARN(ret < 0, "Failed to register kprobe-ftrace (error %d)\n", ret)) { 1082 + /* 1083 + * At this point, sinec ops is not registered, we should be sefe from 1084 + * registering empty filter. 1085 + */ 1086 + ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0); 1087 + return ret; 1088 + } 1076 1089 } 1077 1090 1078 1091 (*cnt)++; 1079 - return ret; 1080 - 1081 - err_ftrace: 1082 - /* 1083 - * At this point, sinec ops is not registered, we should be sefe from 1084 - * registering empty filter. 1085 - */ 1086 - ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0); 1087 1092 return ret; 1088 1093 } 1089 1094 ··· 1154 1163 if (unlikely(kprobe_ftrace(kp))) 1155 1164 return arm_kprobe_ftrace(kp); 1156 1165 1157 - cpus_read_lock(); 1158 - mutex_lock(&text_mutex); 1166 + guard(cpus_read_lock)(); 1167 + guard(mutex)(&text_mutex); 1159 1168 __arm_kprobe(kp); 1160 - mutex_unlock(&text_mutex); 1161 - cpus_read_unlock(); 1162 - 1163 1169 return 0; 1164 1170 } 1165 1171 ··· 1165 1177 if (unlikely(kprobe_ftrace(kp))) 1166 1178 return disarm_kprobe_ftrace(kp); 1167 1179 1168 - cpus_read_lock(); 1169 - mutex_lock(&text_mutex); 1180 + guard(cpus_read_lock)(); 1181 + guard(mutex)(&text_mutex); 1170 1182 __disarm_kprobe(kp, reopt); 1171 - mutex_unlock(&text_mutex); 1172 - cpus_read_unlock(); 1173 - 1174 1183 return 0; 1175 1184 } 1176 1185 ··· 1284 1299 int ret = 0; 1285 1300 struct kprobe *ap = orig_p; 1286 1301 1287 - cpus_read_lock(); 1302 + scoped_guard(cpus_read_lock) { 1303 + /* For preparing optimization, jump_label_text_reserved() is called */ 1304 + guard(jump_label_lock)(); 1305 + guard(mutex)(&text_mutex); 1288 1306 1289 - /* For preparing optimization, jump_label_text_reserved() is called */ 1290 - jump_label_lock(); 1291 - mutex_lock(&text_mutex); 1292 - 1293 - if (!kprobe_aggrprobe(orig_p)) { 1294 - /* If 'orig_p' is not an 'aggr_kprobe', create new one. */ 1295 - ap = alloc_aggr_kprobe(orig_p); 1296 - if (!ap) { 1297 - ret = -ENOMEM; 1298 - goto out; 1307 + if (!kprobe_aggrprobe(orig_p)) { 1308 + /* If 'orig_p' is not an 'aggr_kprobe', create new one. */ 1309 + ap = alloc_aggr_kprobe(orig_p); 1310 + if (!ap) 1311 + return -ENOMEM; 1312 + init_aggr_kprobe(ap, orig_p); 1313 + } else if (kprobe_unused(ap)) { 1314 + /* This probe is going to die. Rescue it */ 1315 + ret = reuse_unused_kprobe(ap); 1316 + if (ret) 1317 + return ret; 1299 1318 } 1300 - init_aggr_kprobe(ap, orig_p); 1301 - } else if (kprobe_unused(ap)) { 1302 - /* This probe is going to die. Rescue it */ 1303 - ret = reuse_unused_kprobe(ap); 1304 - if (ret) 1305 - goto out; 1306 - } 1307 1319 1308 - if (kprobe_gone(ap)) { 1309 - /* 1310 - * Attempting to insert new probe at the same location that 1311 - * had a probe in the module vaddr area which already 1312 - * freed. So, the instruction slot has already been 1313 - * released. We need a new slot for the new probe. 1314 - */ 1315 - ret = arch_prepare_kprobe(ap); 1316 - if (ret) 1320 + if (kprobe_gone(ap)) { 1317 1321 /* 1318 - * Even if fail to allocate new slot, don't need to 1319 - * free the 'ap'. It will be used next time, or 1320 - * freed by unregister_kprobe(). 1322 + * Attempting to insert new probe at the same location that 1323 + * had a probe in the module vaddr area which already 1324 + * freed. So, the instruction slot has already been 1325 + * released. We need a new slot for the new probe. 1321 1326 */ 1322 - goto out; 1327 + ret = arch_prepare_kprobe(ap); 1328 + if (ret) 1329 + /* 1330 + * Even if fail to allocate new slot, don't need to 1331 + * free the 'ap'. It will be used next time, or 1332 + * freed by unregister_kprobe(). 1333 + */ 1334 + return ret; 1323 1335 1324 - /* Prepare optimized instructions if possible. */ 1325 - prepare_optimized_kprobe(ap); 1336 + /* Prepare optimized instructions if possible. */ 1337 + prepare_optimized_kprobe(ap); 1326 1338 1327 - /* 1328 - * Clear gone flag to prevent allocating new slot again, and 1329 - * set disabled flag because it is not armed yet. 1330 - */ 1331 - ap->flags = (ap->flags & ~KPROBE_FLAG_GONE) 1332 - | KPROBE_FLAG_DISABLED; 1339 + /* 1340 + * Clear gone flag to prevent allocating new slot again, and 1341 + * set disabled flag because it is not armed yet. 1342 + */ 1343 + ap->flags = (ap->flags & ~KPROBE_FLAG_GONE) 1344 + | KPROBE_FLAG_DISABLED; 1345 + } 1346 + 1347 + /* Copy the insn slot of 'p' to 'ap'. */ 1348 + copy_kprobe(ap, p); 1349 + ret = add_new_kprobe(ap, p); 1333 1350 } 1334 - 1335 - /* Copy the insn slot of 'p' to 'ap'. */ 1336 - copy_kprobe(ap, p); 1337 - ret = add_new_kprobe(ap, p); 1338 - 1339 - out: 1340 - mutex_unlock(&text_mutex); 1341 - jump_label_unlock(); 1342 - cpus_read_unlock(); 1343 1351 1344 1352 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) { 1345 1353 ap->flags &= ~KPROBE_FLAG_DISABLED; ··· 1426 1448 unsigned long offset, bool *on_func_entry) 1427 1449 { 1428 1450 if ((symbol_name && addr) || (!symbol_name && !addr)) 1429 - goto invalid; 1451 + return ERR_PTR(-EINVAL); 1430 1452 1431 1453 if (symbol_name) { 1432 1454 /* ··· 1456 1478 * at the start of the function. 1457 1479 */ 1458 1480 addr = arch_adjust_kprobe_addr((unsigned long)addr, offset, on_func_entry); 1459 - if (addr) 1460 - return addr; 1481 + if (!addr) 1482 + return ERR_PTR(-EINVAL); 1461 1483 1462 - invalid: 1463 - return ERR_PTR(-EINVAL); 1484 + return addr; 1464 1485 } 1465 1486 1466 1487 static kprobe_opcode_t *kprobe_addr(struct kprobe *p) 1467 1488 { 1468 1489 bool on_func_entry; 1490 + 1469 1491 return _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry); 1470 1492 } 1471 1493 ··· 1483 1505 if (unlikely(!ap)) 1484 1506 return NULL; 1485 1507 1486 - if (p != ap) { 1487 - list_for_each_entry(list_p, &ap->list, list) 1488 - if (list_p == p) 1489 - /* kprobe p is a valid probe */ 1490 - goto valid; 1491 - return NULL; 1492 - } 1493 - valid: 1494 - return ap; 1508 + if (p == ap) 1509 + return ap; 1510 + 1511 + list_for_each_entry(list_p, &ap->list, list) 1512 + if (list_p == p) 1513 + /* kprobe p is a valid probe */ 1514 + return ap; 1515 + 1516 + return NULL; 1495 1517 } 1496 1518 1497 1519 /* ··· 1500 1522 */ 1501 1523 static inline int warn_kprobe_rereg(struct kprobe *p) 1502 1524 { 1503 - int ret = 0; 1525 + guard(mutex)(&kprobe_mutex); 1504 1526 1505 - mutex_lock(&kprobe_mutex); 1506 1527 if (WARN_ON_ONCE(__get_valid_kprobe(p))) 1507 - ret = -EINVAL; 1508 - mutex_unlock(&kprobe_mutex); 1528 + return -EINVAL; 1509 1529 1510 - return ret; 1530 + return 0; 1511 1531 } 1512 1532 1513 1533 static int check_ftrace_location(struct kprobe *p) ··· 1541 1565 ret = check_ftrace_location(p); 1542 1566 if (ret) 1543 1567 return ret; 1544 - jump_label_lock(); 1545 - preempt_disable(); 1568 + 1569 + guard(jump_label_lock)(); 1546 1570 1547 1571 /* Ensure the address is in a text area, and find a module if exists. */ 1548 1572 *probed_mod = NULL; 1549 1573 if (!core_kernel_text((unsigned long) p->addr)) { 1574 + guard(preempt)(); 1550 1575 *probed_mod = __module_text_address((unsigned long) p->addr); 1551 - if (!(*probed_mod)) { 1552 - ret = -EINVAL; 1553 - goto out; 1554 - } 1576 + if (!(*probed_mod)) 1577 + return -EINVAL; 1578 + 1579 + /* 1580 + * We must hold a refcount of the probed module while updating 1581 + * its code to prohibit unexpected unloading. 1582 + */ 1583 + if (unlikely(!try_module_get(*probed_mod))) 1584 + return -ENOENT; 1555 1585 } 1556 1586 /* Ensure it is not in reserved area. */ 1557 1587 if (in_gate_area_no_mm((unsigned long) p->addr) || ··· 1566 1584 static_call_text_reserved(p->addr, p->addr) || 1567 1585 find_bug((unsigned long)p->addr) || 1568 1586 is_cfi_preamble_symbol((unsigned long)p->addr)) { 1569 - ret = -EINVAL; 1570 - goto out; 1587 + module_put(*probed_mod); 1588 + return -EINVAL; 1571 1589 } 1572 1590 1573 1591 /* Get module refcount and reject __init functions for loaded modules. */ 1574 1592 if (IS_ENABLED(CONFIG_MODULES) && *probed_mod) { 1575 - /* 1576 - * We must hold a refcount of the probed module while updating 1577 - * its code to prohibit unexpected unloading. 1578 - */ 1579 - if (unlikely(!try_module_get(*probed_mod))) { 1580 - ret = -ENOENT; 1581 - goto out; 1582 - } 1583 - 1584 1593 /* 1585 1594 * If the module freed '.init.text', we couldn't insert 1586 1595 * kprobes in there. ··· 1579 1606 if (within_module_init((unsigned long)p->addr, *probed_mod) && 1580 1607 !module_is_coming(*probed_mod)) { 1581 1608 module_put(*probed_mod); 1582 - *probed_mod = NULL; 1583 - ret = -ENOENT; 1609 + return -ENOENT; 1584 1610 } 1585 1611 } 1586 1612 1587 - out: 1588 - preempt_enable(); 1589 - jump_label_unlock(); 1613 + return 0; 1614 + } 1590 1615 1591 - return ret; 1616 + static int __register_kprobe(struct kprobe *p) 1617 + { 1618 + int ret; 1619 + struct kprobe *old_p; 1620 + 1621 + guard(mutex)(&kprobe_mutex); 1622 + 1623 + old_p = get_kprobe(p->addr); 1624 + if (old_p) 1625 + /* Since this may unoptimize 'old_p', locking 'text_mutex'. */ 1626 + return register_aggr_kprobe(old_p, p); 1627 + 1628 + scoped_guard(cpus_read_lock) { 1629 + /* Prevent text modification */ 1630 + guard(mutex)(&text_mutex); 1631 + ret = prepare_kprobe(p); 1632 + if (ret) 1633 + return ret; 1634 + } 1635 + 1636 + INIT_HLIST_NODE(&p->hlist); 1637 + hlist_add_head_rcu(&p->hlist, 1638 + &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 1639 + 1640 + if (!kprobes_all_disarmed && !kprobe_disabled(p)) { 1641 + ret = arm_kprobe(p); 1642 + if (ret) { 1643 + hlist_del_rcu(&p->hlist); 1644 + synchronize_rcu(); 1645 + } 1646 + } 1647 + 1648 + /* Try to optimize kprobe */ 1649 + try_to_optimize_kprobe(p); 1650 + return 0; 1592 1651 } 1593 1652 1594 1653 int register_kprobe(struct kprobe *p) 1595 1654 { 1596 1655 int ret; 1597 - struct kprobe *old_p; 1598 1656 struct module *probed_mod; 1599 1657 kprobe_opcode_t *addr; 1600 1658 bool on_func_entry; 1601 1659 1602 - /* Adjust probe address from symbol */ 1660 + /* Canonicalize probe address from symbol */ 1603 1661 addr = _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry); 1604 1662 if (IS_ERR(addr)) 1605 1663 return PTR_ERR(addr); ··· 1642 1638 1643 1639 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */ 1644 1640 p->flags &= KPROBE_FLAG_DISABLED; 1641 + if (on_func_entry) 1642 + p->flags |= KPROBE_FLAG_ON_FUNC_ENTRY; 1645 1643 p->nmissed = 0; 1646 1644 INIT_LIST_HEAD(&p->list); 1647 1645 ··· 1651 1645 if (ret) 1652 1646 return ret; 1653 1647 1654 - mutex_lock(&kprobe_mutex); 1655 - 1656 - if (on_func_entry) 1657 - p->flags |= KPROBE_FLAG_ON_FUNC_ENTRY; 1658 - 1659 - old_p = get_kprobe(p->addr); 1660 - if (old_p) { 1661 - /* Since this may unoptimize 'old_p', locking 'text_mutex'. */ 1662 - ret = register_aggr_kprobe(old_p, p); 1663 - goto out; 1664 - } 1665 - 1666 - cpus_read_lock(); 1667 - /* Prevent text modification */ 1668 - mutex_lock(&text_mutex); 1669 - ret = prepare_kprobe(p); 1670 - mutex_unlock(&text_mutex); 1671 - cpus_read_unlock(); 1672 - if (ret) 1673 - goto out; 1674 - 1675 - INIT_HLIST_NODE(&p->hlist); 1676 - hlist_add_head_rcu(&p->hlist, 1677 - &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 1678 - 1679 - if (!kprobes_all_disarmed && !kprobe_disabled(p)) { 1680 - ret = arm_kprobe(p); 1681 - if (ret) { 1682 - hlist_del_rcu(&p->hlist); 1683 - synchronize_rcu(); 1684 - goto out; 1685 - } 1686 - } 1687 - 1688 - /* Try to optimize kprobe */ 1689 - try_to_optimize_kprobe(p); 1690 - out: 1691 - mutex_unlock(&kprobe_mutex); 1648 + ret = __register_kprobe(p); 1692 1649 1693 1650 if (probed_mod) 1694 1651 module_put(probed_mod); ··· 1730 1761 if (IS_ERR(ap)) 1731 1762 return PTR_ERR(ap); 1732 1763 1733 - if (ap == p) 1734 - /* 1735 - * This probe is an independent(and non-optimized) kprobe 1736 - * (not an aggrprobe). Remove from the hash list. 1737 - */ 1738 - goto disarmed; 1764 + WARN_ON(ap != p && !kprobe_aggrprobe(ap)); 1739 1765 1740 - /* Following process expects this probe is an aggrprobe */ 1741 - WARN_ON(!kprobe_aggrprobe(ap)); 1742 - 1743 - if (list_is_singular(&ap->list) && kprobe_disarmed(ap)) 1766 + /* 1767 + * If the probe is an independent(and non-optimized) kprobe 1768 + * (not an aggrprobe), the last kprobe on the aggrprobe, or 1769 + * kprobe is already disarmed, just remove from the hash list. 1770 + */ 1771 + if (ap == p || 1772 + (list_is_singular(&ap->list) && kprobe_disarmed(ap))) { 1744 1773 /* 1745 1774 * !disarmed could be happen if the probe is under delayed 1746 1775 * unoptimizing. 1747 1776 */ 1748 - goto disarmed; 1749 - else { 1750 - /* If disabling probe has special handlers, update aggrprobe */ 1751 - if (p->post_handler && !kprobe_gone(p)) { 1752 - list_for_each_entry(list_p, &ap->list, list) { 1753 - if ((list_p != p) && (list_p->post_handler)) 1754 - goto noclean; 1755 - } 1777 + hlist_del_rcu(&ap->hlist); 1778 + return 0; 1779 + } 1780 + 1781 + /* If disabling probe has special handlers, update aggrprobe */ 1782 + if (p->post_handler && !kprobe_gone(p)) { 1783 + list_for_each_entry(list_p, &ap->list, list) { 1784 + if ((list_p != p) && (list_p->post_handler)) 1785 + break; 1786 + } 1787 + /* No other probe has post_handler */ 1788 + if (list_entry_is_head(list_p, &ap->list, list)) { 1756 1789 /* 1757 1790 * For the kprobe-on-ftrace case, we keep the 1758 1791 * post_handler setting to identify this aggrprobe ··· 1763 1792 if (!kprobe_ftrace(ap)) 1764 1793 ap->post_handler = NULL; 1765 1794 } 1766 - noclean: 1767 - /* 1768 - * Remove from the aggrprobe: this path will do nothing in 1769 - * __unregister_kprobe_bottom(). 1770 - */ 1771 - list_del_rcu(&p->list); 1772 - if (!kprobe_disabled(ap) && !kprobes_all_disarmed) 1773 - /* 1774 - * Try to optimize this probe again, because post 1775 - * handler may have been changed. 1776 - */ 1777 - optimize_kprobe(ap); 1778 1795 } 1796 + 1797 + /* 1798 + * Remove from the aggrprobe: this path will do nothing in 1799 + * __unregister_kprobe_bottom(). 1800 + */ 1801 + list_del_rcu(&p->list); 1802 + if (!kprobe_disabled(ap) && !kprobes_all_disarmed) 1803 + /* 1804 + * Try to optimize this probe again, because post 1805 + * handler may have been changed. 1806 + */ 1807 + optimize_kprobe(ap); 1779 1808 return 0; 1780 1809 1781 - disarmed: 1782 - hlist_del_rcu(&ap->hlist); 1783 - return 0; 1784 1810 } 1785 1811 1786 1812 static void __unregister_kprobe_bottom(struct kprobe *p) ··· 1826 1858 1827 1859 if (num <= 0) 1828 1860 return; 1829 - mutex_lock(&kprobe_mutex); 1830 - for (i = 0; i < num; i++) 1831 - if (__unregister_kprobe_top(kps[i]) < 0) 1832 - kps[i]->addr = NULL; 1833 - mutex_unlock(&kprobe_mutex); 1834 - 1861 + scoped_guard(mutex, &kprobe_mutex) { 1862 + for (i = 0; i < num; i++) 1863 + if (__unregister_kprobe_top(kps[i]) < 0) 1864 + kps[i]->addr = NULL; 1865 + } 1835 1866 synchronize_rcu(); 1836 1867 for (i = 0; i < num; i++) 1837 1868 if (kps[i]->addr) ··· 2269 2302 2270 2303 if (num <= 0) 2271 2304 return; 2272 - mutex_lock(&kprobe_mutex); 2273 2305 for (i = 0; i < num; i++) { 2306 + guard(mutex)(&kprobe_mutex); 2307 + 2274 2308 if (__unregister_kprobe_top(&rps[i]->kp) < 0) 2275 2309 rps[i]->kp.addr = NULL; 2276 2310 #ifdef CONFIG_KRETPROBE_ON_RETHOOK ··· 2280 2312 rcu_assign_pointer(rps[i]->rph->rp, NULL); 2281 2313 #endif 2282 2314 } 2283 - mutex_unlock(&kprobe_mutex); 2284 2315 2285 2316 synchronize_rcu(); 2286 2317 for (i = 0; i < num; i++) { ··· 2360 2393 /* Disable one kprobe */ 2361 2394 int disable_kprobe(struct kprobe *kp) 2362 2395 { 2363 - int ret = 0; 2364 2396 struct kprobe *p; 2365 2397 2366 - mutex_lock(&kprobe_mutex); 2398 + guard(mutex)(&kprobe_mutex); 2367 2399 2368 2400 /* Disable this kprobe */ 2369 2401 p = __disable_kprobe(kp); 2370 - if (IS_ERR(p)) 2371 - ret = PTR_ERR(p); 2372 2402 2373 - mutex_unlock(&kprobe_mutex); 2374 - return ret; 2403 + return IS_ERR(p) ? PTR_ERR(p) : 0; 2375 2404 } 2376 2405 EXPORT_SYMBOL_GPL(disable_kprobe); 2377 2406 ··· 2377 2414 int ret = 0; 2378 2415 struct kprobe *p; 2379 2416 2380 - mutex_lock(&kprobe_mutex); 2417 + guard(mutex)(&kprobe_mutex); 2381 2418 2382 2419 /* Check whether specified probe is valid. */ 2383 2420 p = __get_valid_kprobe(kp); 2384 - if (unlikely(p == NULL)) { 2385 - ret = -EINVAL; 2386 - goto out; 2387 - } 2421 + if (unlikely(p == NULL)) 2422 + return -EINVAL; 2388 2423 2389 - if (kprobe_gone(kp)) { 2424 + if (kprobe_gone(kp)) 2390 2425 /* This kprobe has gone, we couldn't enable it. */ 2391 - ret = -EINVAL; 2392 - goto out; 2393 - } 2426 + return -EINVAL; 2394 2427 2395 2428 if (p != kp) 2396 2429 kp->flags &= ~KPROBE_FLAG_DISABLED; ··· 2400 2441 kp->flags |= KPROBE_FLAG_DISABLED; 2401 2442 } 2402 2443 } 2403 - out: 2404 - mutex_unlock(&kprobe_mutex); 2405 2444 return ret; 2406 2445 } 2407 2446 EXPORT_SYMBOL_GPL(enable_kprobe); ··· 2587 2630 unsigned int i; 2588 2631 int checkcore = (val == MODULE_STATE_GOING); 2589 2632 2590 - if (val == MODULE_STATE_COMING) { 2591 - mutex_lock(&kprobe_mutex); 2633 + guard(mutex)(&kprobe_mutex); 2634 + 2635 + if (val == MODULE_STATE_COMING) 2592 2636 add_module_kprobe_blacklist(mod); 2593 - mutex_unlock(&kprobe_mutex); 2594 - } 2637 + 2595 2638 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE) 2596 2639 return NOTIFY_DONE; 2597 2640 ··· 2601 2644 * notified, only '.init.text' section would be freed. We need to 2602 2645 * disable kprobes which have been inserted in the sections. 2603 2646 */ 2604 - mutex_lock(&kprobe_mutex); 2605 2647 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2606 2648 head = &kprobe_table[i]; 2607 2649 hlist_for_each_entry(p, head, hlist) ··· 2623 2667 } 2624 2668 if (val == MODULE_STATE_GOING) 2625 2669 remove_module_kprobe_blacklist(mod); 2626 - mutex_unlock(&kprobe_mutex); 2627 2670 return NOTIFY_DONE; 2628 2671 } 2629 2672 ··· 2650 2695 struct kprobe *p; 2651 2696 int i; 2652 2697 2653 - mutex_lock(&kprobe_mutex); 2698 + guard(mutex)(&kprobe_mutex); 2654 2699 2655 2700 /* Kill all kprobes on initmem because the target code has been freed. */ 2656 2701 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { ··· 2660 2705 kill_kprobe(p); 2661 2706 } 2662 2707 } 2663 - 2664 - mutex_unlock(&kprobe_mutex); 2665 2708 } 2666 2709 2667 2710 static int __init init_kprobes(void) ··· 2855 2902 unsigned int i, total = 0, errors = 0; 2856 2903 int err, ret = 0; 2857 2904 2858 - mutex_lock(&kprobe_mutex); 2905 + guard(mutex)(&kprobe_mutex); 2859 2906 2860 2907 /* If kprobes are armed, just return */ 2861 2908 if (!kprobes_all_disarmed) 2862 - goto already_enabled; 2909 + return 0; 2863 2910 2864 2911 /* 2865 2912 * optimize_kprobe() called by arm_kprobe() checks ··· 2889 2936 else 2890 2937 pr_info("Kprobes globally enabled\n"); 2891 2938 2892 - already_enabled: 2893 - mutex_unlock(&kprobe_mutex); 2894 2939 return ret; 2895 2940 } 2896 2941 ··· 2899 2948 unsigned int i, total = 0, errors = 0; 2900 2949 int err, ret = 0; 2901 2950 2902 - mutex_lock(&kprobe_mutex); 2951 + guard(mutex)(&kprobe_mutex); 2903 2952 2904 2953 /* If kprobes are already disarmed, just return */ 2905 - if (kprobes_all_disarmed) { 2906 - mutex_unlock(&kprobe_mutex); 2954 + if (kprobes_all_disarmed) 2907 2955 return 0; 2908 - } 2909 2956 2910 2957 kprobes_all_disarmed = true; 2911 2958 ··· 2928 2979 else 2929 2980 pr_info("Kprobes globally disabled\n"); 2930 2981 2931 - mutex_unlock(&kprobe_mutex); 2932 - 2933 2982 /* Wait for disarming all kprobes by optimizer */ 2934 - wait_for_kprobe_optimizer(); 2935 - 2983 + wait_for_kprobe_optimizer_locked(); 2936 2984 return ret; 2937 2985 } 2938 2986

+17 -19

kernel/trace/trace_eprobe.c

··· 917 917 goto error; 918 918 } 919 919 920 - mutex_lock(&event_mutex); 921 - event_call = find_and_get_event(sys_name, sys_event); 922 - ep = alloc_event_probe(group, event, event_call, argc - 2); 923 - mutex_unlock(&event_mutex); 920 + scoped_guard(mutex, &event_mutex) { 921 + event_call = find_and_get_event(sys_name, sys_event); 922 + ep = alloc_event_probe(group, event, event_call, argc - 2); 923 + } 924 924 925 925 if (IS_ERR(ep)) { 926 926 ret = PTR_ERR(ep); ··· 952 952 if (ret < 0) 953 953 goto error; 954 954 init_trace_eprobe_call(ep); 955 - mutex_lock(&event_mutex); 956 - ret = trace_probe_register_event_call(&ep->tp); 957 - if (ret) { 958 - if (ret == -EEXIST) { 959 - trace_probe_log_set_index(0); 960 - trace_probe_log_err(0, EVENT_EXIST); 955 + scoped_guard(mutex, &event_mutex) { 956 + ret = trace_probe_register_event_call(&ep->tp); 957 + if (ret) { 958 + if (ret == -EEXIST) { 959 + trace_probe_log_set_index(0); 960 + trace_probe_log_err(0, EVENT_EXIST); 961 + } 962 + goto error; 961 963 } 962 - mutex_unlock(&event_mutex); 963 - goto error; 964 + ret = dyn_event_add(&ep->devent, &ep->tp.event->call); 965 + if (ret < 0) { 966 + trace_probe_unregister_event_call(&ep->tp); 967 + goto error; 968 + } 964 969 } 965 - ret = dyn_event_add(&ep->devent, &ep->tp.event->call); 966 - if (ret < 0) { 967 - trace_probe_unregister_event_call(&ep->tp); 968 - mutex_unlock(&event_mutex); 969 - goto error; 970 - } 971 - mutex_unlock(&event_mutex); 972 970 return ret; 973 971 parse_error: 974 972 ret = -EINVAL;

+73 -82

kernel/trace/trace_kprobe.c

··· 8 8 #define pr_fmt(fmt) "trace_kprobe: " fmt 9 9 10 10 #include <linux/bpf-cgroup.h> 11 + #include <linux/cleanup.h> 11 12 #include <linux/security.h> 12 13 #include <linux/module.h> 13 14 #include <linux/uaccess.h> ··· 258 257 } 259 258 } 260 259 260 + DEFINE_FREE(free_trace_kprobe, struct trace_kprobe *, 261 + if (!IS_ERR_OR_NULL(_T)) free_trace_kprobe(_T)) 262 + 261 263 /* 262 264 * Allocate new trace_probe and initialize it (including kprobes). 263 265 */ ··· 272 268 int maxactive, 273 269 int nargs, bool is_return) 274 270 { 275 - struct trace_kprobe *tk; 271 + struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 276 272 int ret = -ENOMEM; 277 273 278 274 tk = kzalloc(struct_size(tk, tp.args, nargs), GFP_KERNEL); ··· 281 277 282 278 tk->nhit = alloc_percpu(unsigned long); 283 279 if (!tk->nhit) 284 - goto error; 280 + return ERR_PTR(ret); 285 281 286 282 if (symbol) { 287 283 tk->symbol = kstrdup(symbol, GFP_KERNEL); 288 284 if (!tk->symbol) 289 - goto error; 285 + return ERR_PTR(ret); 290 286 tk->rp.kp.symbol_name = tk->symbol; 291 287 tk->rp.kp.offset = offs; 292 288 } else ··· 303 299 304 300 ret = trace_probe_init(&tk->tp, event, group, false, nargs); 305 301 if (ret < 0) 306 - goto error; 302 + return ERR_PTR(ret); 307 303 308 304 dyn_event_init(&tk->devent, &trace_kprobe_ops); 309 - return tk; 310 - error: 311 - free_trace_kprobe(tk); 312 - return ERR_PTR(ret); 305 + return_ptr(tk); 313 306 } 314 307 315 308 static struct trace_kprobe *find_trace_kprobe(const char *event, ··· 635 634 struct trace_kprobe *old_tk; 636 635 int ret; 637 636 638 - mutex_lock(&event_mutex); 637 + guard(mutex)(&event_mutex); 639 638 640 639 old_tk = find_trace_kprobe(trace_probe_name(&tk->tp), 641 640 trace_probe_group_name(&tk->tp)); ··· 643 642 if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) { 644 643 trace_probe_log_set_index(0); 645 644 trace_probe_log_err(0, DIFF_PROBE_TYPE); 646 - ret = -EEXIST; 647 - } else { 648 - ret = append_trace_kprobe(tk, old_tk); 645 + return -EEXIST; 649 646 } 650 - goto end; 647 + return append_trace_kprobe(tk, old_tk); 651 648 } 652 649 653 650 /* Register new event */ ··· 656 657 trace_probe_log_err(0, EVENT_EXIST); 657 658 } else 658 659 pr_warn("Failed to register probe event(%d)\n", ret); 659 - goto end; 660 + return ret; 660 661 } 661 662 662 663 /* Register k*probe */ ··· 671 672 else 672 673 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); 673 674 674 - end: 675 - mutex_unlock(&event_mutex); 676 675 return ret; 677 676 } 678 677 ··· 703 706 return NOTIFY_DONE; 704 707 705 708 /* Update probes on coming module */ 706 - mutex_lock(&event_mutex); 709 + guard(mutex)(&event_mutex); 707 710 for_each_trace_kprobe(tk, pos) { 708 711 if (trace_kprobe_within_module(tk, mod)) { 709 712 /* Don't need to check busy - this should have gone. */ ··· 715 718 module_name(mod), ret); 716 719 } 717 720 } 718 - mutex_unlock(&event_mutex); 719 721 720 722 return NOTIFY_DONE; 721 723 } ··· 836 840 static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, 837 841 struct pt_regs *regs); 838 842 839 - static int __trace_kprobe_create(int argc, const char *argv[]) 843 + static int trace_kprobe_create_internal(int argc, const char *argv[], 844 + struct traceprobe_parse_context *ctx) 840 845 { 841 846 /* 842 847 * Argument syntax: ··· 863 866 * Type of args: 864 867 * FETCHARG:TYPE : use TYPE instead of unsigned long. 865 868 */ 866 - struct trace_kprobe *tk = NULL; 869 + struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 867 870 int i, len, new_argc = 0, ret = 0; 868 871 bool is_return = false; 869 - char *symbol = NULL, *tmp = NULL; 870 - const char **new_argv = NULL; 872 + char *symbol __free(kfree) = NULL; 873 + char *tmp = NULL; 874 + const char **new_argv __free(kfree) = NULL; 871 875 const char *event = NULL, *group = KPROBE_EVENT_SYSTEM; 872 876 enum probe_print_type ptype; 873 877 int maxactive = 0; ··· 877 879 char buf[MAX_EVENT_NAME_LEN]; 878 880 char gbuf[MAX_EVENT_NAME_LEN]; 879 881 char abuf[MAX_BTF_ARGS_LEN]; 880 - char *dbuf = NULL; 881 - struct traceprobe_parse_context ctx = { .flags = TPARG_FL_KERNEL }; 882 + char *dbuf __free(kfree) = NULL; 882 883 883 884 switch (argv[0][0]) { 884 885 case 'r': ··· 891 894 if (argc < 2) 892 895 return -ECANCELED; 893 896 894 - trace_probe_log_init("trace_kprobe", argc, argv); 895 - 896 897 event = strchr(&argv[0][1], ':'); 897 898 if (event) 898 899 event++; ··· 898 903 if (isdigit(argv[0][1])) { 899 904 if (!is_return) { 900 905 trace_probe_log_err(1, BAD_MAXACT_TYPE); 901 - goto parse_error; 906 + return -EINVAL; 902 907 } 903 908 if (event) 904 909 len = event - &argv[0][1] - 1; ··· 906 911 len = strlen(&argv[0][1]); 907 912 if (len > MAX_EVENT_NAME_LEN - 1) { 908 913 trace_probe_log_err(1, BAD_MAXACT); 909 - goto parse_error; 914 + return -EINVAL; 910 915 } 911 916 memcpy(buf, &argv[0][1], len); 912 917 buf[len] = '\0'; 913 918 ret = kstrtouint(buf, 0, &maxactive); 914 919 if (ret || !maxactive) { 915 920 trace_probe_log_err(1, BAD_MAXACT); 916 - goto parse_error; 921 + return -EINVAL; 917 922 } 918 923 /* kretprobes instances are iterated over via a list. The 919 924 * maximum should stay reasonable. 920 925 */ 921 926 if (maxactive > KRETPROBE_MAXACTIVE_MAX) { 922 927 trace_probe_log_err(1, MAXACT_TOO_BIG); 923 - goto parse_error; 928 + return -EINVAL; 924 929 } 925 930 } 926 931 ··· 929 934 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { 930 935 trace_probe_log_set_index(1); 931 936 /* Check whether uprobe event specified */ 932 - if (strchr(argv[1], '/') && strchr(argv[1], ':')) { 933 - ret = -ECANCELED; 934 - goto error; 935 - } 937 + if (strchr(argv[1], '/') && strchr(argv[1], ':')) 938 + return -ECANCELED; 939 + 936 940 /* a symbol specified */ 937 941 symbol = kstrdup(argv[1], GFP_KERNEL); 938 - if (!symbol) { 939 - ret = -ENOMEM; 940 - goto error; 941 - } 942 + if (!symbol) 943 + return -ENOMEM; 942 944 943 945 tmp = strchr(symbol, '%'); 944 946 if (tmp) { ··· 944 952 is_return = true; 945 953 } else { 946 954 trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX); 947 - goto parse_error; 955 + return -EINVAL; 948 956 } 949 957 } 950 958 ··· 952 960 ret = traceprobe_split_symbol_offset(symbol, &offset); 953 961 if (ret || offset < 0 || offset > UINT_MAX) { 954 962 trace_probe_log_err(0, BAD_PROBE_ADDR); 955 - goto parse_error; 963 + return -EINVAL; 956 964 } 957 965 ret = validate_probe_symbol(symbol); 958 966 if (ret) { ··· 960 968 trace_probe_log_err(0, NON_UNIQ_SYMBOL); 961 969 else 962 970 trace_probe_log_err(0, BAD_PROBE_ADDR); 963 - goto parse_error; 971 + return -EINVAL; 964 972 } 965 973 if (is_return) 966 - ctx.flags |= TPARG_FL_RETURN; 974 + ctx->flags |= TPARG_FL_RETURN; 967 975 ret = kprobe_on_func_entry(NULL, symbol, offset); 968 976 if (ret == 0 && !is_return) 969 - ctx.flags |= TPARG_FL_FENTRY; 977 + ctx->flags |= TPARG_FL_FENTRY; 970 978 /* Defer the ENOENT case until register kprobe */ 971 979 if (ret == -EINVAL && is_return) { 972 980 trace_probe_log_err(0, BAD_RETPROBE); 973 - goto parse_error; 981 + return -EINVAL; 974 982 } 975 983 } 976 984 ··· 979 987 ret = traceprobe_parse_event_name(&event, &group, gbuf, 980 988 event - argv[0]); 981 989 if (ret) 982 - goto parse_error; 990 + return ret; 983 991 } 984 992 985 993 if (!event) { ··· 995 1003 } 996 1004 997 1005 argc -= 2; argv += 2; 998 - ctx.funcname = symbol; 1006 + ctx->funcname = symbol; 999 1007 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc, 1000 - abuf, MAX_BTF_ARGS_LEN, &ctx); 1008 + abuf, MAX_BTF_ARGS_LEN, ctx); 1001 1009 if (IS_ERR(new_argv)) { 1002 1010 ret = PTR_ERR(new_argv); 1003 1011 new_argv = NULL; 1004 - goto out; 1012 + return ret; 1005 1013 } 1006 1014 if (new_argv) { 1007 1015 argc = new_argc; 1008 1016 argv = new_argv; 1009 1017 } 1010 - if (argc > MAX_TRACE_ARGS) { 1011 - ret = -E2BIG; 1012 - goto out; 1013 - } 1018 + if (argc > MAX_TRACE_ARGS) 1019 + return -E2BIG; 1014 1020 1015 1021 ret = traceprobe_expand_dentry_args(argc, argv, &dbuf); 1016 1022 if (ret) 1017 - goto out; 1023 + return ret; 1018 1024 1019 1025 /* setup a probe */ 1020 1026 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, ··· 1021 1031 ret = PTR_ERR(tk); 1022 1032 /* This must return -ENOMEM, else there is a bug */ 1023 1033 WARN_ON_ONCE(ret != -ENOMEM); 1024 - goto out; /* We know tk is not allocated */ 1034 + return ret; /* We know tk is not allocated */ 1025 1035 } 1026 1036 1027 1037 /* parse arguments */ 1028 1038 for (i = 0; i < argc; i++) { 1029 1039 trace_probe_log_set_index(i + 2); 1030 - ctx.offset = 0; 1031 - ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], &ctx); 1040 + ctx->offset = 0; 1041 + ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], ctx); 1032 1042 if (ret) 1033 - goto error; /* This can be -ENOMEM */ 1043 + return ret; /* This can be -ENOMEM */ 1034 1044 } 1035 1045 /* entry handler for kretprobe */ 1036 1046 if (is_return && tk->tp.entry_arg) { ··· 1041 1051 ptype = is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1042 1052 ret = traceprobe_set_print_fmt(&tk->tp, ptype); 1043 1053 if (ret < 0) 1044 - goto error; 1054 + return ret; 1045 1055 1046 1056 ret = register_trace_kprobe(tk); 1047 1057 if (ret) { ··· 1052 1062 trace_probe_log_err(0, BAD_PROBE_ADDR); 1053 1063 else if (ret != -ENOMEM && ret != -EEXIST) 1054 1064 trace_probe_log_err(0, FAIL_REG_PROBE); 1055 - goto error; 1065 + return ret; 1056 1066 } 1067 + /* 1068 + * Here, 'tk' has been registered to the list successfully, 1069 + * so we don't need to free it. 1070 + */ 1071 + tk = NULL; 1057 1072 1058 - out: 1073 + return 0; 1074 + } 1075 + 1076 + static int trace_kprobe_create_cb(int argc, const char *argv[]) 1077 + { 1078 + struct traceprobe_parse_context ctx = { .flags = TPARG_FL_KERNEL }; 1079 + int ret; 1080 + 1081 + trace_probe_log_init("trace_kprobe", argc, argv); 1082 + 1083 + ret = trace_kprobe_create_internal(argc, argv, &ctx); 1084 + 1059 1085 traceprobe_finish_parse(&ctx); 1060 1086 trace_probe_log_clear(); 1061 - kfree(new_argv); 1062 - kfree(symbol); 1063 - kfree(dbuf); 1064 1087 return ret; 1065 - 1066 - parse_error: 1067 - ret = -EINVAL; 1068 - error: 1069 - free_trace_kprobe(tk); 1070 - goto out; 1071 1088 } 1072 1089 1073 1090 static int trace_kprobe_create(const char *raw_command) 1074 1091 { 1075 - return trace_probe_create(raw_command, __trace_kprobe_create); 1092 + return trace_probe_create(raw_command, trace_kprobe_create_cb); 1076 1093 } 1077 1094 1078 1095 static int create_or_delete_trace_kprobe(const char *raw_command) ··· 1895 1898 bool is_return) 1896 1899 { 1897 1900 enum probe_print_type ptype; 1898 - struct trace_kprobe *tk; 1901 + struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 1899 1902 int ret; 1900 1903 char *event; 1901 1904 ··· 1926 1929 1927 1930 ptype = trace_kprobe_is_return(tk) ? 1928 1931 PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1929 - if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0) { 1930 - ret = -ENOMEM; 1931 - goto error; 1932 - } 1932 + if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0) 1933 + return ERR_PTR(-ENOMEM); 1933 1934 1934 1935 ret = __register_trace_kprobe(tk); 1935 1936 if (ret < 0) 1936 - goto error; 1937 + return ERR_PTR(ret); 1937 1938 1938 - return trace_probe_event_call(&tk->tp); 1939 - error: 1940 - free_trace_kprobe(tk); 1941 - return ERR_PTR(ret); 1939 + return trace_probe_event_call(&(no_free_ptr(tk)->tp)); 1942 1940 } 1943 1941 1944 1942 void destroy_local_trace_kprobe(struct trace_event_call *event_call) ··· 1962 1970 struct trace_kprobe *tk; 1963 1971 struct dyn_event *pos; 1964 1972 1965 - mutex_lock(&event_mutex); 1973 + guard(mutex)(&event_mutex); 1966 1974 for_each_trace_kprobe(tk, pos) { 1967 1975 list_for_each_entry(file, &tr->events, list) 1968 1976 if (file->event_call == trace_probe_event_call(&tk->tp)) 1969 1977 trace_event_enable_disable(file, 1, 0); 1970 1978 } 1971 - mutex_unlock(&event_mutex); 1972 1979 } 1973 1980 1974 1981 static __init void setup_boot_kprobe_events(void)

+17 -34

kernel/trace/trace_probe.c

··· 1409 1409 struct traceprobe_parse_context *ctx) 1410 1410 { 1411 1411 struct fetch_insn *code, *tmp = NULL; 1412 - char *type, *arg; 1412 + char *type, *arg __free(kfree) = NULL; 1413 1413 int ret, len; 1414 1414 1415 1415 len = strlen(argv); ··· 1426 1426 return -ENOMEM; 1427 1427 1428 1428 parg->comm = kstrdup(arg, GFP_KERNEL); 1429 - if (!parg->comm) { 1430 - ret = -ENOMEM; 1431 - goto out; 1432 - } 1429 + if (!parg->comm) 1430 + return -ENOMEM; 1433 1431 1434 1432 type = parse_probe_arg_type(arg, parg, ctx); 1435 - if (IS_ERR(type)) { 1436 - ret = PTR_ERR(type); 1437 - goto out; 1438 - } 1433 + if (IS_ERR(type)) 1434 + return PTR_ERR(type); 1439 1435 1440 1436 code = tmp = kcalloc(FETCH_INSN_MAX, sizeof(*code), GFP_KERNEL); 1441 - if (!code) { 1442 - ret = -ENOMEM; 1443 - goto out; 1444 - } 1437 + if (!code) 1438 + return -ENOMEM; 1445 1439 code[FETCH_INSN_MAX - 1].op = FETCH_OP_END; 1446 1440 1447 1441 ctx->last_type = NULL; ··· 1491 1497 kfree(code->data); 1492 1498 } 1493 1499 kfree(tmp); 1494 - out: 1495 - kfree(arg); 1496 1500 1497 1501 return ret; 1498 1502 } ··· 1660 1668 { 1661 1669 const struct btf_param *params = NULL; 1662 1670 int i, j, n, used, ret, args_idx = -1; 1663 - const char **new_argv = NULL; 1671 + const char **new_argv __free(kfree) = NULL; 1664 1672 1665 1673 ret = argv_has_var_arg(argc, argv, &args_idx, ctx); 1666 1674 if (ret < 0) ··· 1699 1707 ret = sprint_nth_btf_arg(n, "", buf + used, 1700 1708 bufsize - used, ctx); 1701 1709 if (ret < 0) 1702 - goto error; 1710 + return ERR_PTR(ret); 1703 1711 1704 1712 new_argv[j++] = buf + used; 1705 1713 used += ret + 1; ··· 1713 1721 n = simple_strtoul(argv[i] + 4, &type, 10); 1714 1722 if (type && !(*type == ':' || *type == '\0')) { 1715 1723 trace_probe_log_err(0, BAD_VAR); 1716 - ret = -ENOENT; 1717 - goto error; 1724 + return ERR_PTR(-ENOENT); 1718 1725 } 1719 1726 /* Note: $argN starts from $arg1 */ 1720 1727 ret = sprint_nth_btf_arg(n - 1, type, buf + used, 1721 1728 bufsize - used, ctx); 1722 1729 if (ret < 0) 1723 - goto error; 1730 + return ERR_PTR(ret); 1724 1731 new_argv[j++] = buf + used; 1725 1732 used += ret + 1; 1726 1733 } else 1727 1734 new_argv[j++] = argv[i]; 1728 1735 } 1729 1736 1730 - return new_argv; 1731 - 1732 - error: 1733 - kfree(new_argv); 1734 - return ERR_PTR(ret); 1737 + return_ptr(new_argv); 1735 1738 } 1736 1739 1737 1740 /* @buf: *buf must be equal to NULL. Caller must to free *buf */ ··· 1734 1747 { 1735 1748 int i, used, ret; 1736 1749 const int bufsize = MAX_DENTRY_ARGS_LEN; 1737 - char *tmpbuf = NULL; 1750 + char *tmpbuf __free(kfree) = NULL; 1738 1751 1739 1752 if (*buf) 1740 1753 return -EINVAL; 1741 1754 1742 1755 used = 0; 1743 1756 for (i = 0; i < argc; i++) { 1744 - char *tmp; 1757 + char *tmp __free(kfree) = NULL; 1745 1758 char *equal; 1746 1759 size_t arg_len; 1747 1760 ··· 1756 1769 1757 1770 tmp = kstrdup(argv[i], GFP_KERNEL); 1758 1771 if (!tmp) 1759 - goto nomem; 1772 + return -ENOMEM; 1760 1773 1761 1774 equal = strchr(tmp, '='); 1762 1775 if (equal) ··· 1777 1790 offsetof(struct file, f_path.dentry), 1778 1791 equal ? equal + 1 : tmp); 1779 1792 1780 - kfree(tmp); 1781 1793 if (ret >= bufsize - used) 1782 - goto nomem; 1794 + return -ENOMEM; 1783 1795 argv[i] = tmpbuf + used; 1784 1796 used += ret + 1; 1785 1797 } 1786 1798 1787 - *buf = tmpbuf; 1799 + *buf = no_free_ptr(tmpbuf); 1788 1800 return 0; 1789 - nomem: 1790 - kfree(tmpbuf); 1791 - return -ENOMEM; 1792 1801 } 1793 1802 1794 1803 void traceprobe_finish_parse(struct traceprobe_parse_context *ctx)

+5 -10

kernel/trace/trace_uprobe.c

··· 498 498 struct trace_uprobe *old_tu; 499 499 int ret; 500 500 501 - mutex_lock(&event_mutex); 501 + guard(mutex)(&event_mutex); 502 502 503 503 ret = validate_ref_ctr_offset(tu); 504 504 if (ret) 505 - goto end; 505 + return ret; 506 506 507 507 /* register as an event */ 508 508 old_tu = find_probe_event(trace_probe_name(&tu->tp), ··· 511 511 if (is_ret_probe(tu) != is_ret_probe(old_tu)) { 512 512 trace_probe_log_set_index(0); 513 513 trace_probe_log_err(0, DIFF_PROBE_TYPE); 514 - ret = -EEXIST; 515 - } else { 516 - ret = append_trace_uprobe(tu, old_tu); 514 + return -EEXIST; 517 515 } 518 - goto end; 516 + return append_trace_uprobe(tu, old_tu); 519 517 } 520 518 521 519 ret = register_uprobe_event(tu); ··· 523 525 trace_probe_log_err(0, EVENT_EXIST); 524 526 } else 525 527 pr_warn("Failed to register probe event(%d)\n", ret); 526 - goto end; 528 + return ret; 527 529 } 528 530 529 531 dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp)); 530 - 531 - end: 532 - mutex_unlock(&event_mutex); 533 532 534 533 return ret; 535 534 }

Configure Feed

Configure Feed