tracing: Move saved_cmdline code into trace_sched_switch.c

+3 -512

kernel/trace/trace.c

··· 39 39 #include <linux/ctype.h> 40 40 #include <linux/init.h> 41 41 #include <linux/panic_notifier.h> 42 - #include <linux/kmemleak.h> 43 42 #include <linux/poll.h> 44 43 #include <linux/nmi.h> 45 44 #include <linux/fs.h> ··· 104 105 * tracing is active, only save the comm when a trace event 105 106 * occurred. 106 107 */ 107 - static DEFINE_PER_CPU(bool, trace_taskinfo_save); 108 + DEFINE_PER_CPU(bool, trace_taskinfo_save); 108 109 109 110 /* 110 111 * Kill all tracing for good (never come back). ··· 2319 2320 mutex_unlock(&trace_types_lock); 2320 2321 } 2321 2322 2322 - /* 2323 - * The tgid_map array maps from pid to tgid; i.e. the value stored at index i 2324 - * is the tgid last observed corresponding to pid=i. 2325 - */ 2326 - static int *tgid_map; 2327 - 2328 - /* The maximum valid index into tgid_map. */ 2329 - static size_t tgid_map_max; 2330 - 2331 - #define SAVED_CMDLINES_DEFAULT 128 2332 - #define NO_CMDLINE_MAP UINT_MAX 2333 - /* 2334 - * Preemption must be disabled before acquiring trace_cmdline_lock. 2335 - * The various trace_arrays' max_lock must be acquired in a context 2336 - * where interrupt is disabled. 2337 - */ 2338 - static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; 2339 - struct saved_cmdlines_buffer { 2340 - unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 2341 - unsigned *map_cmdline_to_pid; 2342 - unsigned cmdline_num; 2343 - int cmdline_idx; 2344 - char saved_cmdlines[]; 2345 - }; 2346 - static struct saved_cmdlines_buffer *savedcmd; 2347 - 2348 - /* Holds the size of a cmdline and pid element */ 2349 - #define SAVED_CMDLINE_MAP_ELEMENT_SIZE(s) \ 2350 - (TASK_COMM_LEN + sizeof((s)->map_cmdline_to_pid[0])) 2351 - 2352 - static inline char *get_saved_cmdlines(int idx) 2353 - { 2354 - return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN]; 2355 - } 2356 - 2357 - static inline void set_cmdline(int idx, const char *cmdline) 2358 - { 2359 - strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN); 2360 - } 2361 - 2362 - static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s) 2363 - { 2364 - int order = get_order(sizeof(*s) + s->cmdline_num * TASK_COMM_LEN); 2365 - 2366 - kmemleak_free(s); 2367 - free_pages((unsigned long)s, order); 2368 - } 2369 - 2370 - static struct saved_cmdlines_buffer *allocate_cmdlines_buffer(unsigned int val) 2371 - { 2372 - struct saved_cmdlines_buffer *s; 2373 - struct page *page; 2374 - int orig_size, size; 2375 - int order; 2376 - 2377 - /* Figure out how much is needed to hold the given number of cmdlines */ 2378 - orig_size = sizeof(*s) + val * SAVED_CMDLINE_MAP_ELEMENT_SIZE(s); 2379 - order = get_order(orig_size); 2380 - size = 1 << (order + PAGE_SHIFT); 2381 - page = alloc_pages(GFP_KERNEL, order); 2382 - if (!page) 2383 - return NULL; 2384 - 2385 - s = page_address(page); 2386 - kmemleak_alloc(s, size, 1, GFP_KERNEL); 2387 - memset(s, 0, sizeof(*s)); 2388 - 2389 - /* Round up to actual allocation */ 2390 - val = (size - sizeof(*s)) / SAVED_CMDLINE_MAP_ELEMENT_SIZE(s); 2391 - s->cmdline_num = val; 2392 - 2393 - /* Place map_cmdline_to_pid array right after saved_cmdlines */ 2394 - s->map_cmdline_to_pid = (unsigned *)&s->saved_cmdlines[val * TASK_COMM_LEN]; 2395 - 2396 - s->cmdline_idx = 0; 2397 - memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP, 2398 - sizeof(s->map_pid_to_cmdline)); 2399 - memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP, 2400 - val * sizeof(*s->map_cmdline_to_pid)); 2401 - 2402 - return s; 2403 - } 2404 - 2405 - static int trace_create_savedcmd(void) 2406 - { 2407 - savedcmd = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT); 2408 - 2409 - return savedcmd ? 0 : -ENOMEM; 2410 - } 2411 - 2412 2323 int is_tracing_stopped(void) 2413 2324 { 2414 2325 return global_trace.stop_count; ··· 2409 2500 void tracing_stop(void) 2410 2501 { 2411 2502 return tracing_stop_tr(&global_trace); 2412 - } 2413 - 2414 - static int trace_save_cmdline(struct task_struct *tsk) 2415 - { 2416 - unsigned tpid, idx; 2417 - 2418 - /* treat recording of idle task as a success */ 2419 - if (!tsk->pid) 2420 - return 1; 2421 - 2422 - tpid = tsk->pid & (PID_MAX_DEFAULT - 1); 2423 - 2424 - /* 2425 - * It's not the end of the world if we don't get 2426 - * the lock, but we also don't want to spin 2427 - * nor do we want to disable interrupts, 2428 - * so if we miss here, then better luck next time. 2429 - * 2430 - * This is called within the scheduler and wake up, so interrupts 2431 - * had better been disabled and run queue lock been held. 2432 - */ 2433 - lockdep_assert_preemption_disabled(); 2434 - if (!arch_spin_trylock(&trace_cmdline_lock)) 2435 - return 0; 2436 - 2437 - idx = savedcmd->map_pid_to_cmdline[tpid]; 2438 - if (idx == NO_CMDLINE_MAP) { 2439 - idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num; 2440 - 2441 - savedcmd->map_pid_to_cmdline[tpid] = idx; 2442 - savedcmd->cmdline_idx = idx; 2443 - } 2444 - 2445 - savedcmd->map_cmdline_to_pid[idx] = tsk->pid; 2446 - set_cmdline(idx, tsk->comm); 2447 - 2448 - arch_spin_unlock(&trace_cmdline_lock); 2449 - 2450 - return 1; 2451 - } 2452 - 2453 - static void __trace_find_cmdline(int pid, char comm[]) 2454 - { 2455 - unsigned map; 2456 - int tpid; 2457 - 2458 - if (!pid) { 2459 - strcpy(comm, "<idle>"); 2460 - return; 2461 - } 2462 - 2463 - if (WARN_ON_ONCE(pid < 0)) { 2464 - strcpy(comm, "<XXX>"); 2465 - return; 2466 - } 2467 - 2468 - tpid = pid & (PID_MAX_DEFAULT - 1); 2469 - map = savedcmd->map_pid_to_cmdline[tpid]; 2470 - if (map != NO_CMDLINE_MAP) { 2471 - tpid = savedcmd->map_cmdline_to_pid[map]; 2472 - if (tpid == pid) { 2473 - strscpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN); 2474 - return; 2475 - } 2476 - } 2477 - strcpy(comm, "<...>"); 2478 - } 2479 - 2480 - void trace_find_cmdline(int pid, char comm[]) 2481 - { 2482 - preempt_disable(); 2483 - arch_spin_lock(&trace_cmdline_lock); 2484 - 2485 - __trace_find_cmdline(pid, comm); 2486 - 2487 - arch_spin_unlock(&trace_cmdline_lock); 2488 - preempt_enable(); 2489 - } 2490 - 2491 - static int *trace_find_tgid_ptr(int pid) 2492 - { 2493 - /* 2494 - * Pairs with the smp_store_release in set_tracer_flag() to ensure that 2495 - * if we observe a non-NULL tgid_map then we also observe the correct 2496 - * tgid_map_max. 2497 - */ 2498 - int *map = smp_load_acquire(&tgid_map); 2499 - 2500 - if (unlikely(!map || pid > tgid_map_max)) 2501 - return NULL; 2502 - 2503 - return &map[pid]; 2504 - } 2505 - 2506 - int trace_find_tgid(int pid) 2507 - { 2508 - int *ptr = trace_find_tgid_ptr(pid); 2509 - 2510 - return ptr ? *ptr : 0; 2511 - } 2512 - 2513 - static int trace_save_tgid(struct task_struct *tsk) 2514 - { 2515 - int *ptr; 2516 - 2517 - /* treat recording of idle task as a success */ 2518 - if (!tsk->pid) 2519 - return 1; 2520 - 2521 - ptr = trace_find_tgid_ptr(tsk->pid); 2522 - if (!ptr) 2523 - return 0; 2524 - 2525 - *ptr = tsk->tgid; 2526 - return 1; 2527 - } 2528 - 2529 - static bool tracing_record_taskinfo_skip(int flags) 2530 - { 2531 - if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID)))) 2532 - return true; 2533 - if (!__this_cpu_read(trace_taskinfo_save)) 2534 - return true; 2535 - return false; 2536 - } 2537 - 2538 - /** 2539 - * tracing_record_taskinfo - record the task info of a task 2540 - * 2541 - * @task: task to record 2542 - * @flags: TRACE_RECORD_CMDLINE for recording comm 2543 - * TRACE_RECORD_TGID for recording tgid 2544 - */ 2545 - void tracing_record_taskinfo(struct task_struct *task, int flags) 2546 - { 2547 - bool done; 2548 - 2549 - if (tracing_record_taskinfo_skip(flags)) 2550 - return; 2551 - 2552 - /* 2553 - * Record as much task information as possible. If some fail, continue 2554 - * to try to record the others. 2555 - */ 2556 - done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task); 2557 - done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task); 2558 - 2559 - /* If recording any information failed, retry again soon. */ 2560 - if (!done) 2561 - return; 2562 - 2563 - __this_cpu_write(trace_taskinfo_save, false); 2564 - } 2565 - 2566 - /** 2567 - * tracing_record_taskinfo_sched_switch - record task info for sched_switch 2568 - * 2569 - * @prev: previous task during sched_switch 2570 - * @next: next task during sched_switch 2571 - * @flags: TRACE_RECORD_CMDLINE for recording comm 2572 - * TRACE_RECORD_TGID for recording tgid 2573 - */ 2574 - void tracing_record_taskinfo_sched_switch(struct task_struct *prev, 2575 - struct task_struct *next, int flags) 2576 - { 2577 - bool done; 2578 - 2579 - if (tracing_record_taskinfo_skip(flags)) 2580 - return; 2581 - 2582 - /* 2583 - * Record as much task information as possible. If some fail, continue 2584 - * to try to record the others. 2585 - */ 2586 - done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev); 2587 - done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next); 2588 - done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev); 2589 - done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next); 2590 - 2591 - /* If recording any information failed, retry again soon. */ 2592 - if (!done) 2593 - return; 2594 - 2595 - __this_cpu_write(trace_taskinfo_save, false); 2596 - } 2597 - 2598 - /* Helpers to record a specific task information */ 2599 - void tracing_record_cmdline(struct task_struct *task) 2600 - { 2601 - tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE); 2602 - } 2603 - 2604 - void tracing_record_tgid(struct task_struct *task) 2605 - { 2606 - tracing_record_taskinfo(task, TRACE_RECORD_TGID); 2607 2503 } 2608 2504 2609 2505 /* ··· 5167 5453 return 0; 5168 5454 } 5169 5455 5170 - static int trace_alloc_tgid_map(void) 5171 - { 5172 - int *map; 5173 - 5174 - if (tgid_map) 5175 - return 0; 5176 - 5177 - tgid_map_max = pid_max; 5178 - map = kvcalloc(tgid_map_max + 1, sizeof(*tgid_map), 5179 - GFP_KERNEL); 5180 - if (!map) 5181 - return -ENOMEM; 5182 - 5183 - /* 5184 - * Pairs with smp_load_acquire() in 5185 - * trace_find_tgid_ptr() to ensure that if it observes 5186 - * the tgid_map we just allocated then it also observes 5187 - * the corresponding tgid_map_max value. 5188 - */ 5189 - smp_store_release(&tgid_map, map); 5190 - return 0; 5191 - } 5192 - 5193 5456 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled) 5194 5457 { 5195 5458 if ((mask == TRACE_ITER_RECORD_TGID) || ··· 5191 5500 trace_event_enable_cmd_record(enabled); 5192 5501 5193 5502 if (mask == TRACE_ITER_RECORD_TGID) { 5503 + 5194 5504 if (trace_alloc_tgid_map() < 0) { 5195 5505 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID; 5196 5506 return -ENOMEM; ··· 5634 5942 .open = tracing_open_generic, 5635 5943 .read = tracing_readme_read, 5636 5944 .llseek = generic_file_llseek, 5637 - }; 5638 - 5639 - static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos) 5640 - { 5641 - int pid = ++(*pos); 5642 - 5643 - return trace_find_tgid_ptr(pid); 5644 - } 5645 - 5646 - static void *saved_tgids_start(struct seq_file *m, loff_t *pos) 5647 - { 5648 - int pid = *pos; 5649 - 5650 - return trace_find_tgid_ptr(pid); 5651 - } 5652 - 5653 - static void saved_tgids_stop(struct seq_file *m, void *v) 5654 - { 5655 - } 5656 - 5657 - static int saved_tgids_show(struct seq_file *m, void *v) 5658 - { 5659 - int *entry = (int *)v; 5660 - int pid = entry - tgid_map; 5661 - int tgid = *entry; 5662 - 5663 - if (tgid == 0) 5664 - return SEQ_SKIP; 5665 - 5666 - seq_printf(m, "%d %d\n", pid, tgid); 5667 - return 0; 5668 - } 5669 - 5670 - static const struct seq_operations tracing_saved_tgids_seq_ops = { 5671 - .start = saved_tgids_start, 5672 - .stop = saved_tgids_stop, 5673 - .next = saved_tgids_next, 5674 - .show = saved_tgids_show, 5675 - }; 5676 - 5677 - static int tracing_saved_tgids_open(struct inode *inode, struct file *filp) 5678 - { 5679 - int ret; 5680 - 5681 - ret = tracing_check_open_get_tr(NULL); 5682 - if (ret) 5683 - return ret; 5684 - 5685 - return seq_open(filp, &tracing_saved_tgids_seq_ops); 5686 - } 5687 - 5688 - 5689 - static const struct file_operations tracing_saved_tgids_fops = { 5690 - .open = tracing_saved_tgids_open, 5691 - .read = seq_read, 5692 - .llseek = seq_lseek, 5693 - .release = seq_release, 5694 - }; 5695 - 5696 - static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos) 5697 - { 5698 - unsigned int *ptr = v; 5699 - 5700 - if (*pos || m->count) 5701 - ptr++; 5702 - 5703 - (*pos)++; 5704 - 5705 - for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num]; 5706 - ptr++) { 5707 - if (*ptr == -1 || *ptr == NO_CMDLINE_MAP) 5708 - continue; 5709 - 5710 - return ptr; 5711 - } 5712 - 5713 - return NULL; 5714 - } 5715 - 5716 - static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos) 5717 - { 5718 - void *v; 5719 - loff_t l = 0; 5720 - 5721 - preempt_disable(); 5722 - arch_spin_lock(&trace_cmdline_lock); 5723 - 5724 - v = &savedcmd->map_cmdline_to_pid[0]; 5725 - while (l <= *pos) { 5726 - v = saved_cmdlines_next(m, v, &l); 5727 - if (!v) 5728 - return NULL; 5729 - } 5730 - 5731 - return v; 5732 - } 5733 - 5734 - static void saved_cmdlines_stop(struct seq_file *m, void *v) 5735 - { 5736 - arch_spin_unlock(&trace_cmdline_lock); 5737 - preempt_enable(); 5738 - } 5739 - 5740 - static int saved_cmdlines_show(struct seq_file *m, void *v) 5741 - { 5742 - char buf[TASK_COMM_LEN]; 5743 - unsigned int *pid = v; 5744 - 5745 - __trace_find_cmdline(*pid, buf); 5746 - seq_printf(m, "%d %s\n", *pid, buf); 5747 - return 0; 5748 - } 5749 - 5750 - static const struct seq_operations tracing_saved_cmdlines_seq_ops = { 5751 - .start = saved_cmdlines_start, 5752 - .next = saved_cmdlines_next, 5753 - .stop = saved_cmdlines_stop, 5754 - .show = saved_cmdlines_show, 5755 - }; 5756 - 5757 - static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp) 5758 - { 5759 - int ret; 5760 - 5761 - ret = tracing_check_open_get_tr(NULL); 5762 - if (ret) 5763 - return ret; 5764 - 5765 - return seq_open(filp, &tracing_saved_cmdlines_seq_ops); 5766 - } 5767 - 5768 - static const struct file_operations tracing_saved_cmdlines_fops = { 5769 - .open = tracing_saved_cmdlines_open, 5770 - .read = seq_read, 5771 - .llseek = seq_lseek, 5772 - .release = seq_release, 5773 - }; 5774 - 5775 - static ssize_t 5776 - tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf, 5777 - size_t cnt, loff_t *ppos) 5778 - { 5779 - char buf[64]; 5780 - int r; 5781 - 5782 - preempt_disable(); 5783 - arch_spin_lock(&trace_cmdline_lock); 5784 - r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num); 5785 - arch_spin_unlock(&trace_cmdline_lock); 5786 - preempt_enable(); 5787 - 5788 - return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5789 - } 5790 - 5791 - static int tracing_resize_saved_cmdlines(unsigned int val) 5792 - { 5793 - struct saved_cmdlines_buffer *s, *savedcmd_temp; 5794 - 5795 - s = allocate_cmdlines_buffer(val); 5796 - if (!s) 5797 - return -ENOMEM; 5798 - 5799 - preempt_disable(); 5800 - arch_spin_lock(&trace_cmdline_lock); 5801 - savedcmd_temp = savedcmd; 5802 - savedcmd = s; 5803 - arch_spin_unlock(&trace_cmdline_lock); 5804 - preempt_enable(); 5805 - free_saved_cmdlines_buffer(savedcmd_temp); 5806 - 5807 - return 0; 5808 - } 5809 - 5810 - static ssize_t 5811 - tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf, 5812 - size_t cnt, loff_t *ppos) 5813 - { 5814 - unsigned long val; 5815 - int ret; 5816 - 5817 - ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 5818 - if (ret) 5819 - return ret; 5820 - 5821 - /* must have at least 1 entry or less than PID_MAX_DEFAULT */ 5822 - if (!val || val > PID_MAX_DEFAULT) 5823 - return -EINVAL; 5824 - 5825 - ret = tracing_resize_saved_cmdlines((unsigned int)val); 5826 - if (ret < 0) 5827 - return ret; 5828 - 5829 - *ppos += cnt; 5830 - 5831 - return cnt; 5832 - } 5833 - 5834 - static const struct file_operations tracing_saved_cmdlines_size_fops = { 5835 - .open = tracing_open_generic, 5836 - .read = tracing_saved_cmdlines_size_read, 5837 - .write = tracing_saved_cmdlines_size_write, 5838 5945 }; 5839 5946 5840 5947 #ifdef CONFIG_TRACE_EVAL_MAP_FILE ··· 10211 10720 out_free_pipe_cpumask: 10212 10721 free_cpumask_var(global_trace.pipe_cpumask); 10213 10722 out_free_savedcmd: 10214 - free_saved_cmdlines_buffer(savedcmd); 10723 + trace_free_saved_cmdlines_buffer(); 10215 10724 out_free_temp_buffer: 10216 10725 ring_buffer_free(temp_buffer); 10217 10726 out_rm_hp_state:

+10

kernel/trace/trace.h

··· 1375 1375 trace_buffer_unlock_commit_regs(tr, buffer, event, trace_ctx, NULL); 1376 1376 } 1377 1377 1378 + DECLARE_PER_CPU(bool, trace_taskinfo_save); 1379 + int trace_save_cmdline(struct task_struct *tsk); 1380 + int trace_create_savedcmd(void); 1381 + int trace_alloc_tgid_map(void); 1382 + void trace_free_saved_cmdlines_buffer(void); 1383 + 1384 + extern const struct file_operations tracing_saved_cmdlines_fops; 1385 + extern const struct file_operations tracing_saved_tgids_fops; 1386 + extern const struct file_operations tracing_saved_cmdlines_size_fops; 1387 + 1378 1388 DECLARE_PER_CPU(struct ring_buffer_event *, trace_buffered_event); 1379 1389 DECLARE_PER_CPU(int, trace_buffered_event_cnt); 1380 1390 void trace_buffered_event_disable(void);

+515

kernel/trace/trace_sched_switch.c

··· 8 8 #include <linux/module.h> 9 9 #include <linux/kallsyms.h> 10 10 #include <linux/uaccess.h> 11 + #include <linux/kmemleak.h> 11 12 #include <linux/ftrace.h> 12 13 #include <trace/events/sched.h> 13 14 ··· 149 148 { 150 149 tracing_stop_sched_switch(RECORD_TGID); 151 150 } 151 + 152 + /* 153 + * The tgid_map array maps from pid to tgid; i.e. the value stored at index i 154 + * is the tgid last observed corresponding to pid=i. 155 + */ 156 + static int *tgid_map; 157 + 158 + /* The maximum valid index into tgid_map. */ 159 + static size_t tgid_map_max; 160 + 161 + #define SAVED_CMDLINES_DEFAULT 128 162 + #define NO_CMDLINE_MAP UINT_MAX 163 + /* 164 + * Preemption must be disabled before acquiring trace_cmdline_lock. 165 + * The various trace_arrays' max_lock must be acquired in a context 166 + * where interrupt is disabled. 167 + */ 168 + static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; 169 + struct saved_cmdlines_buffer { 170 + unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 171 + unsigned *map_cmdline_to_pid; 172 + unsigned cmdline_num; 173 + int cmdline_idx; 174 + char saved_cmdlines[]; 175 + }; 176 + static struct saved_cmdlines_buffer *savedcmd; 177 + 178 + /* Holds the size of a cmdline and pid element */ 179 + #define SAVED_CMDLINE_MAP_ELEMENT_SIZE(s) \ 180 + (TASK_COMM_LEN + sizeof((s)->map_cmdline_to_pid[0])) 181 + 182 + static inline char *get_saved_cmdlines(int idx) 183 + { 184 + return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN]; 185 + } 186 + 187 + static inline void set_cmdline(int idx, const char *cmdline) 188 + { 189 + strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN); 190 + } 191 + 192 + static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s) 193 + { 194 + int order = get_order(sizeof(*s) + s->cmdline_num * TASK_COMM_LEN); 195 + 196 + kmemleak_free(s); 197 + free_pages((unsigned long)s, order); 198 + } 199 + 200 + static struct saved_cmdlines_buffer *allocate_cmdlines_buffer(unsigned int val) 201 + { 202 + struct saved_cmdlines_buffer *s; 203 + struct page *page; 204 + int orig_size, size; 205 + int order; 206 + 207 + /* Figure out how much is needed to hold the given number of cmdlines */ 208 + orig_size = sizeof(*s) + val * SAVED_CMDLINE_MAP_ELEMENT_SIZE(s); 209 + order = get_order(orig_size); 210 + size = 1 << (order + PAGE_SHIFT); 211 + page = alloc_pages(GFP_KERNEL, order); 212 + if (!page) 213 + return NULL; 214 + 215 + s = page_address(page); 216 + kmemleak_alloc(s, size, 1, GFP_KERNEL); 217 + memset(s, 0, sizeof(*s)); 218 + 219 + /* Round up to actual allocation */ 220 + val = (size - sizeof(*s)) / SAVED_CMDLINE_MAP_ELEMENT_SIZE(s); 221 + s->cmdline_num = val; 222 + 223 + /* Place map_cmdline_to_pid array right after saved_cmdlines */ 224 + s->map_cmdline_to_pid = (unsigned *)&s->saved_cmdlines[val * TASK_COMM_LEN]; 225 + 226 + s->cmdline_idx = 0; 227 + memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP, 228 + sizeof(s->map_pid_to_cmdline)); 229 + memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP, 230 + val * sizeof(*s->map_cmdline_to_pid)); 231 + 232 + return s; 233 + } 234 + 235 + int trace_create_savedcmd(void) 236 + { 237 + savedcmd = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT); 238 + 239 + return savedcmd ? 0 : -ENOMEM; 240 + } 241 + 242 + int trace_save_cmdline(struct task_struct *tsk) 243 + { 244 + unsigned tpid, idx; 245 + 246 + /* treat recording of idle task as a success */ 247 + if (!tsk->pid) 248 + return 1; 249 + 250 + tpid = tsk->pid & (PID_MAX_DEFAULT - 1); 251 + 252 + /* 253 + * It's not the end of the world if we don't get 254 + * the lock, but we also don't want to spin 255 + * nor do we want to disable interrupts, 256 + * so if we miss here, then better luck next time. 257 + * 258 + * This is called within the scheduler and wake up, so interrupts 259 + * had better been disabled and run queue lock been held. 260 + */ 261 + lockdep_assert_preemption_disabled(); 262 + if (!arch_spin_trylock(&trace_cmdline_lock)) 263 + return 0; 264 + 265 + idx = savedcmd->map_pid_to_cmdline[tpid]; 266 + if (idx == NO_CMDLINE_MAP) { 267 + idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num; 268 + 269 + savedcmd->map_pid_to_cmdline[tpid] = idx; 270 + savedcmd->cmdline_idx = idx; 271 + } 272 + 273 + savedcmd->map_cmdline_to_pid[idx] = tsk->pid; 274 + set_cmdline(idx, tsk->comm); 275 + 276 + arch_spin_unlock(&trace_cmdline_lock); 277 + 278 + return 1; 279 + } 280 + 281 + static void __trace_find_cmdline(int pid, char comm[]) 282 + { 283 + unsigned map; 284 + int tpid; 285 + 286 + if (!pid) { 287 + strcpy(comm, "<idle>"); 288 + return; 289 + } 290 + 291 + if (WARN_ON_ONCE(pid < 0)) { 292 + strcpy(comm, "<XXX>"); 293 + return; 294 + } 295 + 296 + tpid = pid & (PID_MAX_DEFAULT - 1); 297 + map = savedcmd->map_pid_to_cmdline[tpid]; 298 + if (map != NO_CMDLINE_MAP) { 299 + tpid = savedcmd->map_cmdline_to_pid[map]; 300 + if (tpid == pid) { 301 + strscpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN); 302 + return; 303 + } 304 + } 305 + strcpy(comm, "<...>"); 306 + } 307 + 308 + void trace_find_cmdline(int pid, char comm[]) 309 + { 310 + preempt_disable(); 311 + arch_spin_lock(&trace_cmdline_lock); 312 + 313 + __trace_find_cmdline(pid, comm); 314 + 315 + arch_spin_unlock(&trace_cmdline_lock); 316 + preempt_enable(); 317 + } 318 + 319 + static int *trace_find_tgid_ptr(int pid) 320 + { 321 + /* 322 + * Pairs with the smp_store_release in set_tracer_flag() to ensure that 323 + * if we observe a non-NULL tgid_map then we also observe the correct 324 + * tgid_map_max. 325 + */ 326 + int *map = smp_load_acquire(&tgid_map); 327 + 328 + if (unlikely(!map || pid > tgid_map_max)) 329 + return NULL; 330 + 331 + return &map[pid]; 332 + } 333 + 334 + int trace_find_tgid(int pid) 335 + { 336 + int *ptr = trace_find_tgid_ptr(pid); 337 + 338 + return ptr ? *ptr : 0; 339 + } 340 + 341 + static int trace_save_tgid(struct task_struct *tsk) 342 + { 343 + int *ptr; 344 + 345 + /* treat recording of idle task as a success */ 346 + if (!tsk->pid) 347 + return 1; 348 + 349 + ptr = trace_find_tgid_ptr(tsk->pid); 350 + if (!ptr) 351 + return 0; 352 + 353 + *ptr = tsk->tgid; 354 + return 1; 355 + } 356 + 357 + static bool tracing_record_taskinfo_skip(int flags) 358 + { 359 + if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID)))) 360 + return true; 361 + if (!__this_cpu_read(trace_taskinfo_save)) 362 + return true; 363 + return false; 364 + } 365 + 366 + /** 367 + * tracing_record_taskinfo - record the task info of a task 368 + * 369 + * @task: task to record 370 + * @flags: TRACE_RECORD_CMDLINE for recording comm 371 + * TRACE_RECORD_TGID for recording tgid 372 + */ 373 + void tracing_record_taskinfo(struct task_struct *task, int flags) 374 + { 375 + bool done; 376 + 377 + if (tracing_record_taskinfo_skip(flags)) 378 + return; 379 + 380 + /* 381 + * Record as much task information as possible. If some fail, continue 382 + * to try to record the others. 383 + */ 384 + done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task); 385 + done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task); 386 + 387 + /* If recording any information failed, retry again soon. */ 388 + if (!done) 389 + return; 390 + 391 + __this_cpu_write(trace_taskinfo_save, false); 392 + } 393 + 394 + /** 395 + * tracing_record_taskinfo_sched_switch - record task info for sched_switch 396 + * 397 + * @prev: previous task during sched_switch 398 + * @next: next task during sched_switch 399 + * @flags: TRACE_RECORD_CMDLINE for recording comm 400 + * TRACE_RECORD_TGID for recording tgid 401 + */ 402 + void tracing_record_taskinfo_sched_switch(struct task_struct *prev, 403 + struct task_struct *next, int flags) 404 + { 405 + bool done; 406 + 407 + if (tracing_record_taskinfo_skip(flags)) 408 + return; 409 + 410 + /* 411 + * Record as much task information as possible. If some fail, continue 412 + * to try to record the others. 413 + */ 414 + done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev); 415 + done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next); 416 + done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev); 417 + done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next); 418 + 419 + /* If recording any information failed, retry again soon. */ 420 + if (!done) 421 + return; 422 + 423 + __this_cpu_write(trace_taskinfo_save, false); 424 + } 425 + 426 + /* Helpers to record a specific task information */ 427 + void tracing_record_cmdline(struct task_struct *task) 428 + { 429 + tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE); 430 + } 431 + 432 + void tracing_record_tgid(struct task_struct *task) 433 + { 434 + tracing_record_taskinfo(task, TRACE_RECORD_TGID); 435 + } 436 + 437 + int trace_alloc_tgid_map(void) 438 + { 439 + int *map; 440 + 441 + if (tgid_map) 442 + return 0; 443 + 444 + tgid_map_max = pid_max; 445 + map = kvcalloc(tgid_map_max + 1, sizeof(*tgid_map), 446 + GFP_KERNEL); 447 + if (!map) 448 + return -ENOMEM; 449 + 450 + /* 451 + * Pairs with smp_load_acquire() in 452 + * trace_find_tgid_ptr() to ensure that if it observes 453 + * the tgid_map we just allocated then it also observes 454 + * the corresponding tgid_map_max value. 455 + */ 456 + smp_store_release(&tgid_map, map); 457 + return 0; 458 + } 459 + 460 + static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos) 461 + { 462 + int pid = ++(*pos); 463 + 464 + return trace_find_tgid_ptr(pid); 465 + } 466 + 467 + static void *saved_tgids_start(struct seq_file *m, loff_t *pos) 468 + { 469 + int pid = *pos; 470 + 471 + return trace_find_tgid_ptr(pid); 472 + } 473 + 474 + static void saved_tgids_stop(struct seq_file *m, void *v) 475 + { 476 + } 477 + 478 + static int saved_tgids_show(struct seq_file *m, void *v) 479 + { 480 + int *entry = (int *)v; 481 + int pid = entry - tgid_map; 482 + int tgid = *entry; 483 + 484 + if (tgid == 0) 485 + return SEQ_SKIP; 486 + 487 + seq_printf(m, "%d %d\n", pid, tgid); 488 + return 0; 489 + } 490 + 491 + static const struct seq_operations tracing_saved_tgids_seq_ops = { 492 + .start = saved_tgids_start, 493 + .stop = saved_tgids_stop, 494 + .next = saved_tgids_next, 495 + .show = saved_tgids_show, 496 + }; 497 + 498 + static int tracing_saved_tgids_open(struct inode *inode, struct file *filp) 499 + { 500 + int ret; 501 + 502 + ret = tracing_check_open_get_tr(NULL); 503 + if (ret) 504 + return ret; 505 + 506 + return seq_open(filp, &tracing_saved_tgids_seq_ops); 507 + } 508 + 509 + 510 + const struct file_operations tracing_saved_tgids_fops = { 511 + .open = tracing_saved_tgids_open, 512 + .read = seq_read, 513 + .llseek = seq_lseek, 514 + .release = seq_release, 515 + }; 516 + 517 + static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos) 518 + { 519 + unsigned int *ptr = v; 520 + 521 + if (*pos || m->count) 522 + ptr++; 523 + 524 + (*pos)++; 525 + 526 + for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num]; 527 + ptr++) { 528 + if (*ptr == -1 || *ptr == NO_CMDLINE_MAP) 529 + continue; 530 + 531 + return ptr; 532 + } 533 + 534 + return NULL; 535 + } 536 + 537 + static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos) 538 + { 539 + void *v; 540 + loff_t l = 0; 541 + 542 + preempt_disable(); 543 + arch_spin_lock(&trace_cmdline_lock); 544 + 545 + v = &savedcmd->map_cmdline_to_pid[0]; 546 + while (l <= *pos) { 547 + v = saved_cmdlines_next(m, v, &l); 548 + if (!v) 549 + return NULL; 550 + } 551 + 552 + return v; 553 + } 554 + 555 + static void saved_cmdlines_stop(struct seq_file *m, void *v) 556 + { 557 + arch_spin_unlock(&trace_cmdline_lock); 558 + preempt_enable(); 559 + } 560 + 561 + static int saved_cmdlines_show(struct seq_file *m, void *v) 562 + { 563 + char buf[TASK_COMM_LEN]; 564 + unsigned int *pid = v; 565 + 566 + __trace_find_cmdline(*pid, buf); 567 + seq_printf(m, "%d %s\n", *pid, buf); 568 + return 0; 569 + } 570 + 571 + static const struct seq_operations tracing_saved_cmdlines_seq_ops = { 572 + .start = saved_cmdlines_start, 573 + .next = saved_cmdlines_next, 574 + .stop = saved_cmdlines_stop, 575 + .show = saved_cmdlines_show, 576 + }; 577 + 578 + static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp) 579 + { 580 + int ret; 581 + 582 + ret = tracing_check_open_get_tr(NULL); 583 + if (ret) 584 + return ret; 585 + 586 + return seq_open(filp, &tracing_saved_cmdlines_seq_ops); 587 + } 588 + 589 + const struct file_operations tracing_saved_cmdlines_fops = { 590 + .open = tracing_saved_cmdlines_open, 591 + .read = seq_read, 592 + .llseek = seq_lseek, 593 + .release = seq_release, 594 + }; 595 + 596 + static ssize_t 597 + tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf, 598 + size_t cnt, loff_t *ppos) 599 + { 600 + char buf[64]; 601 + int r; 602 + 603 + preempt_disable(); 604 + arch_spin_lock(&trace_cmdline_lock); 605 + r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num); 606 + arch_spin_unlock(&trace_cmdline_lock); 607 + preempt_enable(); 608 + 609 + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 610 + } 611 + 612 + void trace_free_saved_cmdlines_buffer(void) 613 + { 614 + free_saved_cmdlines_buffer(savedcmd); 615 + } 616 + 617 + static int tracing_resize_saved_cmdlines(unsigned int val) 618 + { 619 + struct saved_cmdlines_buffer *s, *savedcmd_temp; 620 + 621 + s = allocate_cmdlines_buffer(val); 622 + if (!s) 623 + return -ENOMEM; 624 + 625 + preempt_disable(); 626 + arch_spin_lock(&trace_cmdline_lock); 627 + savedcmd_temp = savedcmd; 628 + savedcmd = s; 629 + arch_spin_unlock(&trace_cmdline_lock); 630 + preempt_enable(); 631 + free_saved_cmdlines_buffer(savedcmd_temp); 632 + 633 + return 0; 634 + } 635 + 636 + static ssize_t 637 + tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf, 638 + size_t cnt, loff_t *ppos) 639 + { 640 + unsigned long val; 641 + int ret; 642 + 643 + ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 644 + if (ret) 645 + return ret; 646 + 647 + /* must have at least 1 entry or less than PID_MAX_DEFAULT */ 648 + if (!val || val > PID_MAX_DEFAULT) 649 + return -EINVAL; 650 + 651 + ret = tracing_resize_saved_cmdlines((unsigned int)val); 652 + if (ret < 0) 653 + return ret; 654 + 655 + *ppos += cnt; 656 + 657 + return cnt; 658 + } 659 + 660 + const struct file_operations tracing_saved_cmdlines_size_fops = { 661 + .open = tracing_open_generic, 662 + .read = tracing_saved_cmdlines_size_read, 663 + .write = tracing_saved_cmdlines_size_write, 664 + };

Configure Feed

Configure Feed