Merge tag 'trace-v6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

+21

Documentation/trace/ftrace.rst

··· 218 218 219 219 This displays the total combined size of all the trace buffers. 220 220 221 + buffer_subbuf_size_kb: 222 + 223 + This sets or displays the sub buffer size. The ring buffer is broken up 224 + into several same size "sub buffers". An event can not be bigger than 225 + the size of the sub buffer. Normally, the sub buffer is the size of the 226 + architecture's page (4K on x86). The sub buffer also contains meta data 227 + at the start which also limits the size of an event. That means when 228 + the sub buffer is a page size, no event can be larger than the page 229 + size minus the sub buffer meta data. 230 + 231 + Note, the buffer_subbuf_size_kb is a way for the user to specify the 232 + minimum size of the subbuffer. The kernel may make it bigger due to the 233 + implementation details, or simply fail the operation if the kernel can 234 + not handle the request. 235 + 236 + Changing the sub buffer size allows for events to be larger than the 237 + page size. 238 + 239 + Note: When changing the sub-buffer size, tracing is stopped and any 240 + data in the ring buffer and the snapshot buffer will be discarded. 241 + 221 242 free_buffer: 222 243 223 244 If a process is performing tracing, and the ring buffer should be

+1 -1

drivers/scsi/qla2xxx/qla_os.c

··· 2889 2889 static void 2890 2890 qla_trace_init(void) 2891 2891 { 2892 - qla_trc_array = trace_array_get_by_name("qla2xxx"); 2892 + qla_trc_array = trace_array_get_by_name("qla2xxx", NULL); 2893 2893 if (!qla_trc_array) { 2894 2894 ql_log(ql_log_fatal, NULL, 0x0001, 2895 2895 "Unable to create qla2xxx trace instance, instance logging will be disabled.\n");

+14 -4

include/linux/ring_buffer.h

··· 141 141 bool ring_buffer_iter_dropped(struct ring_buffer_iter *iter); 142 142 143 143 unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu); 144 + unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer); 144 145 145 146 void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu); 146 147 void ring_buffer_reset_online_cpus(struct trace_buffer *buffer); ··· 192 191 size_t ring_buffer_nr_pages(struct trace_buffer *buffer, int cpu); 193 192 size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu); 194 193 195 - void *ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu); 196 - void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data); 197 - int ring_buffer_read_page(struct trace_buffer *buffer, void **data_page, 194 + struct buffer_data_read_page; 195 + struct buffer_data_read_page * 196 + ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu); 197 + void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, 198 + struct buffer_data_read_page *page); 199 + int ring_buffer_read_page(struct trace_buffer *buffer, 200 + struct buffer_data_read_page *data_page, 198 201 size_t len, int cpu, int full); 202 + void *ring_buffer_read_page_data(struct buffer_data_read_page *page); 199 203 200 204 struct trace_seq; 201 205 202 206 int ring_buffer_print_entry_header(struct trace_seq *s); 203 - int ring_buffer_print_page_header(struct trace_seq *s); 207 + int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s); 208 + 209 + int ring_buffer_subbuf_order_get(struct trace_buffer *buffer); 210 + int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order); 211 + int ring_buffer_subbuf_size_get(struct trace_buffer *buffer); 204 212 205 213 enum ring_buffer_flags { 206 214 RB_FL_OVERWRITE = 1 << 0,

+2 -2

include/linux/trace.h

··· 51 51 const char *fmt, ...); 52 52 int trace_array_init_printk(struct trace_array *tr); 53 53 void trace_array_put(struct trace_array *tr); 54 - struct trace_array *trace_array_get_by_name(const char *name); 54 + struct trace_array *trace_array_get_by_name(const char *name, const char *systems); 55 55 int trace_array_destroy(struct trace_array *tr); 56 56 57 57 /* For osnoise tracer */ ··· 84 84 static inline void trace_array_put(struct trace_array *tr) 85 85 { 86 86 } 87 - static inline struct trace_array *trace_array_get_by_name(const char *name) 87 + static inline struct trace_array *trace_array_get_by_name(const char *name, const char *systems) 88 88 { 89 89 return NULL; 90 90 }

+6 -3

include/linux/trace_seq.h

··· 8 8 9 9 /* 10 10 * Trace sequences are used to allow a function to call several other functions 11 - * to create a string of data to use (up to a max of PAGE_SIZE). 11 + * to create a string of data to use. 12 12 */ 13 13 14 + #define TRACE_SEQ_BUFFER_SIZE (PAGE_SIZE * 2 - \ 15 + (sizeof(struct seq_buf) + sizeof(size_t) + sizeof(int))) 16 + 14 17 struct trace_seq { 15 - char buffer[PAGE_SIZE]; 18 + char buffer[TRACE_SEQ_BUFFER_SIZE]; 16 19 struct seq_buf seq; 17 20 size_t readpos; 18 21 int full; ··· 24 21 static inline void 25 22 trace_seq_init(struct trace_seq *s) 26 23 { 27 - seq_buf_init(&s->seq, s->buffer, PAGE_SIZE); 24 + seq_buf_init(&s->seq, s->buffer, TRACE_SEQ_BUFFER_SIZE); 28 25 s->full = 0; 29 26 s->readpos = 0; 30 27 }

+463 -280

kernel/trace/ring_buffer.c

··· 27 27 #include <linux/cpu.h> 28 28 #include <linux/oom.h> 29 29 30 + #include <asm/local64.h> 30 31 #include <asm/local.h> 31 32 32 33 /* ··· 318 317 unsigned char data[] RB_ALIGN_DATA; /* data of buffer page */ 319 318 }; 320 319 320 + struct buffer_data_read_page { 321 + unsigned order; /* order of the page */ 322 + struct buffer_data_page *data; /* actual data, stored in this page */ 323 + }; 324 + 321 325 /* 322 326 * Note, the buffer_page list must be first. The buffer pages 323 327 * are allocated in cache lines, which means that each buffer ··· 337 331 unsigned read; /* index for next read */ 338 332 local_t entries; /* entries on this page */ 339 333 unsigned long real_end; /* real end of data */ 334 + unsigned order; /* order of the page */ 340 335 struct buffer_data_page *page; /* Actual data page */ 341 336 }; 342 337 ··· 368 361 369 362 static void free_buffer_page(struct buffer_page *bpage) 370 363 { 371 - free_page((unsigned long)bpage->page); 364 + free_pages((unsigned long)bpage->page, bpage->order); 372 365 kfree(bpage); 373 366 } 374 367 ··· 378 371 static inline bool test_time_stamp(u64 delta) 379 372 { 380 373 return !!(delta & TS_DELTA_TEST); 381 - } 382 - 383 - #define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE) 384 - 385 - /* Max payload is BUF_PAGE_SIZE - header (8bytes) */ 386 - #define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2)) 387 - 388 - int ring_buffer_print_page_header(struct trace_seq *s) 389 - { 390 - struct buffer_data_page field; 391 - 392 - trace_seq_printf(s, "\tfield: u64 timestamp;\t" 393 - "offset:0;\tsize:%u;\tsigned:%u;\n", 394 - (unsigned int)sizeof(field.time_stamp), 395 - (unsigned int)is_signed_type(u64)); 396 - 397 - trace_seq_printf(s, "\tfield: local_t commit;\t" 398 - "offset:%u;\tsize:%u;\tsigned:%u;\n", 399 - (unsigned int)offsetof(typeof(field), commit), 400 - (unsigned int)sizeof(field.commit), 401 - (unsigned int)is_signed_type(long)); 402 - 403 - trace_seq_printf(s, "\tfield: int overwrite;\t" 404 - "offset:%u;\tsize:%u;\tsigned:%u;\n", 405 - (unsigned int)offsetof(typeof(field), commit), 406 - 1, 407 - (unsigned int)is_signed_type(long)); 408 - 409 - trace_seq_printf(s, "\tfield: char data;\t" 410 - "offset:%u;\tsize:%u;\tsigned:%u;\n", 411 - (unsigned int)offsetof(typeof(field), data), 412 - (unsigned int)BUF_PAGE_SIZE, 413 - (unsigned int)is_signed_type(char)); 414 - 415 - return !trace_seq_has_overflowed(s); 416 374 } 417 375 418 376 struct rb_irq_work { ··· 435 463 RB_CTX_MAX 436 464 }; 437 465 438 - #if BITS_PER_LONG == 32 439 - #define RB_TIME_32 440 - #endif 441 - 442 - /* To test on 64 bit machines */ 443 - //#define RB_TIME_32 444 - 445 - #ifdef RB_TIME_32 446 - 447 - struct rb_time_struct { 448 - local_t cnt; 449 - local_t top; 450 - local_t bottom; 451 - local_t msb; 452 - }; 453 - #else 454 - #include <asm/local64.h> 455 466 struct rb_time_struct { 456 467 local64_t time; 457 468 }; 458 - #endif 459 469 typedef struct rb_time_struct rb_time_t; 460 470 461 471 #define MAX_NEST 5 ··· 511 557 512 558 struct rb_irq_work irq_work; 513 559 bool time_stamp_abs; 560 + 561 + unsigned int subbuf_size; 562 + unsigned int subbuf_order; 563 + unsigned int max_data_size; 514 564 }; 515 565 516 566 struct ring_buffer_iter { ··· 528 570 u64 read_stamp; 529 571 u64 page_stamp; 530 572 struct ring_buffer_event *event; 573 + size_t event_size; 531 574 int missed_events; 532 575 }; 533 576 534 - #ifdef RB_TIME_32 535 - 536 - /* 537 - * On 32 bit machines, local64_t is very expensive. As the ring 538 - * buffer doesn't need all the features of a true 64 bit atomic, 539 - * on 32 bit, it uses these functions (64 still uses local64_t). 540 - * 541 - * For the ring buffer, 64 bit required operations for the time is 542 - * the following: 543 - * 544 - * - Reads may fail if it interrupted a modification of the time stamp. 545 - * It will succeed if it did not interrupt another write even if 546 - * the read itself is interrupted by a write. 547 - * It returns whether it was successful or not. 548 - * 549 - * - Writes always succeed and will overwrite other writes and writes 550 - * that were done by events interrupting the current write. 551 - * 552 - * - A write followed by a read of the same time stamp will always succeed, 553 - * but may not contain the same value. 554 - * 555 - * - A cmpxchg will fail if it interrupted another write or cmpxchg. 556 - * Other than that, it acts like a normal cmpxchg. 557 - * 558 - * The 60 bit time stamp is broken up by 30 bits in a top and bottom half 559 - * (bottom being the least significant 30 bits of the 60 bit time stamp). 560 - * 561 - * The two most significant bits of each half holds a 2 bit counter (0-3). 562 - * Each update will increment this counter by one. 563 - * When reading the top and bottom, if the two counter bits match then the 564 - * top and bottom together make a valid 60 bit number. 565 - */ 566 - #define RB_TIME_SHIFT 30 567 - #define RB_TIME_VAL_MASK ((1 << RB_TIME_SHIFT) - 1) 568 - #define RB_TIME_MSB_SHIFT 60 569 - 570 - static inline int rb_time_cnt(unsigned long val) 577 + int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s) 571 578 { 572 - return (val >> RB_TIME_SHIFT) & 3; 579 + struct buffer_data_page field; 580 + 581 + trace_seq_printf(s, "\tfield: u64 timestamp;\t" 582 + "offset:0;\tsize:%u;\tsigned:%u;\n", 583 + (unsigned int)sizeof(field.time_stamp), 584 + (unsigned int)is_signed_type(u64)); 585 + 586 + trace_seq_printf(s, "\tfield: local_t commit;\t" 587 + "offset:%u;\tsize:%u;\tsigned:%u;\n", 588 + (unsigned int)offsetof(typeof(field), commit), 589 + (unsigned int)sizeof(field.commit), 590 + (unsigned int)is_signed_type(long)); 591 + 592 + trace_seq_printf(s, "\tfield: int overwrite;\t" 593 + "offset:%u;\tsize:%u;\tsigned:%u;\n", 594 + (unsigned int)offsetof(typeof(field), commit), 595 + 1, 596 + (unsigned int)is_signed_type(long)); 597 + 598 + trace_seq_printf(s, "\tfield: char data;\t" 599 + "offset:%u;\tsize:%u;\tsigned:%u;\n", 600 + (unsigned int)offsetof(typeof(field), data), 601 + (unsigned int)buffer->subbuf_size, 602 + (unsigned int)is_signed_type(char)); 603 + 604 + return !trace_seq_has_overflowed(s); 573 605 } 574 606 575 - static inline u64 rb_time_val(unsigned long top, unsigned long bottom) 576 - { 577 - u64 val; 578 - 579 - val = top & RB_TIME_VAL_MASK; 580 - val <<= RB_TIME_SHIFT; 581 - val |= bottom & RB_TIME_VAL_MASK; 582 - 583 - return val; 584 - } 585 - 586 - static inline bool __rb_time_read(rb_time_t *t, u64 *ret, unsigned long *cnt) 587 - { 588 - unsigned long top, bottom, msb; 589 - unsigned long c; 590 - 591 - /* 592 - * If the read is interrupted by a write, then the cnt will 593 - * be different. Loop until both top and bottom have been read 594 - * without interruption. 595 - */ 596 - do { 597 - c = local_read(&t->cnt); 598 - top = local_read(&t->top); 599 - bottom = local_read(&t->bottom); 600 - msb = local_read(&t->msb); 601 - } while (c != local_read(&t->cnt)); 602 - 603 - *cnt = rb_time_cnt(top); 604 - 605 - /* If top, msb or bottom counts don't match, this interrupted a write */ 606 - if (*cnt != rb_time_cnt(msb) || *cnt != rb_time_cnt(bottom)) 607 - return false; 608 - 609 - /* The shift to msb will lose its cnt bits */ 610 - *ret = rb_time_val(top, bottom) | ((u64)msb << RB_TIME_MSB_SHIFT); 611 - return true; 612 - } 613 - 614 - static bool rb_time_read(rb_time_t *t, u64 *ret) 615 - { 616 - unsigned long cnt; 617 - 618 - return __rb_time_read(t, ret, &cnt); 619 - } 620 - 621 - static inline unsigned long rb_time_val_cnt(unsigned long val, unsigned long cnt) 622 - { 623 - return (val & RB_TIME_VAL_MASK) | ((cnt & 3) << RB_TIME_SHIFT); 624 - } 625 - 626 - static inline void rb_time_split(u64 val, unsigned long *top, unsigned long *bottom, 627 - unsigned long *msb) 628 - { 629 - *top = (unsigned long)((val >> RB_TIME_SHIFT) & RB_TIME_VAL_MASK); 630 - *bottom = (unsigned long)(val & RB_TIME_VAL_MASK); 631 - *msb = (unsigned long)(val >> RB_TIME_MSB_SHIFT); 632 - } 633 - 634 - static inline void rb_time_val_set(local_t *t, unsigned long val, unsigned long cnt) 635 - { 636 - val = rb_time_val_cnt(val, cnt); 637 - local_set(t, val); 638 - } 639 - 640 - static void rb_time_set(rb_time_t *t, u64 val) 641 - { 642 - unsigned long cnt, top, bottom, msb; 643 - 644 - rb_time_split(val, &top, &bottom, &msb); 645 - 646 - /* Writes always succeed with a valid number even if it gets interrupted. */ 647 - do { 648 - cnt = local_inc_return(&t->cnt); 649 - rb_time_val_set(&t->top, top, cnt); 650 - rb_time_val_set(&t->bottom, bottom, cnt); 651 - rb_time_val_set(&t->msb, val >> RB_TIME_MSB_SHIFT, cnt); 652 - } while (cnt != local_read(&t->cnt)); 653 - } 654 - 655 - static inline bool 656 - rb_time_read_cmpxchg(local_t *l, unsigned long expect, unsigned long set) 657 - { 658 - return local_try_cmpxchg(l, &expect, set); 659 - } 660 - 661 - #else /* 64 bits */ 662 - 663 - /* local64_t always succeeds */ 664 - 665 - static inline bool rb_time_read(rb_time_t *t, u64 *ret) 607 + static inline void rb_time_read(rb_time_t *t, u64 *ret) 666 608 { 667 609 *ret = local64_read(&t->time); 668 - return true; 669 610 } 670 611 static void rb_time_set(rb_time_t *t, u64 val) 671 612 { 672 613 local64_set(&t->time, val); 673 614 } 674 - #endif 675 615 676 616 /* 677 617 * Enable this to make sure that the event passed to ··· 676 820 WARN_ONCE(1, "nest (%d) greater than max", nest); 677 821 678 822 fail: 679 - /* Can only fail on 32 bit */ 680 - if (!rb_time_read(&cpu_buffer->write_stamp, &ts)) 681 - /* Screw it, just read the current time */ 682 - ts = rb_time_stamp(cpu_buffer->buffer); 823 + rb_time_read(&cpu_buffer->write_stamp, &ts); 683 824 684 825 return ts; 685 826 } ··· 1472 1619 1473 1620 list_add(&bpage->list, pages); 1474 1621 1475 - page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), mflags, 0); 1622 + page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), mflags, 1623 + cpu_buffer->buffer->subbuf_order); 1476 1624 if (!page) 1477 1625 goto free_pages; 1478 1626 bpage->page = page_address(page); 1627 + bpage->order = cpu_buffer->buffer->subbuf_order; 1479 1628 rb_init_page(bpage->page); 1480 1629 1481 1630 if (user_thread && fatal_signal_pending(current)) ··· 1556 1701 rb_check_bpage(cpu_buffer, bpage); 1557 1702 1558 1703 cpu_buffer->reader_page = bpage; 1559 - page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, 0); 1704 + 1705 + page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, cpu_buffer->buffer->subbuf_order); 1560 1706 if (!page) 1561 1707 goto fail_free_reader; 1562 1708 bpage->page = page_address(page); ··· 1640 1784 if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) 1641 1785 goto fail_free_buffer; 1642 1786 1643 - nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); 1787 + /* Default buffer page size - one system page */ 1788 + buffer->subbuf_order = 0; 1789 + buffer->subbuf_size = PAGE_SIZE - BUF_PAGE_HDR_SIZE; 1790 + 1791 + /* Max payload is buffer page size - header (8bytes) */ 1792 + buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2); 1793 + 1794 + nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); 1644 1795 buffer->flags = flags; 1645 1796 buffer->clock = trace_clock_local; 1646 1797 buffer->reader_lock_key = key; ··· 1966 2103 * @size: the new size. 1967 2104 * @cpu_id: the cpu buffer to resize 1968 2105 * 1969 - * Minimum size is 2 * BUF_PAGE_SIZE. 2106 + * Minimum size is 2 * buffer->subbuf_size. 1970 2107 * 1971 2108 * Returns 0 on success and < 0 on failure. 1972 2109 */ ··· 1988 2125 !cpumask_test_cpu(cpu_id, buffer->cpumask)) 1989 2126 return 0; 1990 2127 1991 - nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); 2128 + nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); 1992 2129 1993 2130 /* we need a minimum of two pages */ 1994 2131 if (nr_pages < 2) ··· 2235 2372 */ 2236 2373 barrier(); 2237 2374 2238 - if ((iter->head + length) > commit || length > BUF_PAGE_SIZE) 2375 + if ((iter->head + length) > commit || length > iter->event_size) 2239 2376 /* Writer corrupted the read? */ 2240 2377 goto reset; 2241 2378 ··· 2275 2412 } 2276 2413 2277 2414 static __always_inline unsigned 2278 - rb_event_index(struct ring_buffer_event *event) 2415 + rb_event_index(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event *event) 2279 2416 { 2280 2417 unsigned long addr = (unsigned long)event; 2281 2418 2282 - return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE; 2419 + addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1; 2420 + 2421 + return addr - BUF_PAGE_HDR_SIZE; 2283 2422 } 2284 2423 2285 2424 static void rb_inc_iter(struct ring_buffer_iter *iter) ··· 2470 2605 rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, 2471 2606 unsigned long tail, struct rb_event_info *info) 2472 2607 { 2608 + unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); 2473 2609 struct buffer_page *tail_page = info->tail_page; 2474 2610 struct ring_buffer_event *event; 2475 2611 unsigned long length = info->length; ··· 2479 2613 * Only the event that crossed the page boundary 2480 2614 * must fill the old tail_page with padding. 2481 2615 */ 2482 - if (tail >= BUF_PAGE_SIZE) { 2616 + if (tail >= bsize) { 2483 2617 /* 2484 2618 * If the page was filled, then we still need 2485 2619 * to update the real_end. Reset it to zero 2486 2620 * and the reader will ignore it. 2487 2621 */ 2488 - if (tail == BUF_PAGE_SIZE) 2622 + if (tail == bsize) 2489 2623 tail_page->real_end = 0; 2490 2624 2491 2625 local_sub(length, &tail_page->write); ··· 2513 2647 * If we are less than the minimum size, we don't need to 2514 2648 * worry about it. 2515 2649 */ 2516 - if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) { 2650 + if (tail > (bsize - RB_EVNT_MIN_SIZE)) { 2517 2651 /* No room for any events */ 2518 2652 2519 2653 /* Mark the rest of the page with padding */ ··· 2528 2662 } 2529 2663 2530 2664 /* Put in a discarded event */ 2531 - event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE; 2665 + event->array[0] = (bsize - tail) - RB_EVNT_HDR_SIZE; 2532 2666 event->type_len = RINGBUF_TYPE_PADDING; 2533 2667 /* time delta must be non zero */ 2534 2668 event->time_delta = 1; 2535 2669 2536 2670 /* account for padding bytes */ 2537 - local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes); 2671 + local_add(bsize - tail, &cpu_buffer->entries_bytes); 2538 2672 2539 2673 /* Make sure the padding is visible before the tail_page->write update */ 2540 2674 smp_wmb(); 2541 2675 2542 2676 /* Set write to end of buffer */ 2543 - length = (tail + length) - BUF_PAGE_SIZE; 2677 + length = (tail + length) - bsize; 2544 2678 local_sub(length, &tail_page->write); 2545 2679 } 2546 2680 ··· 2654 2788 2655 2789 /* Slow path */ 2656 2790 static struct ring_buffer_event * 2657 - rb_add_time_stamp(struct ring_buffer_event *event, u64 delta, bool abs) 2791 + rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer, 2792 + struct ring_buffer_event *event, u64 delta, bool abs) 2658 2793 { 2659 2794 if (abs) 2660 2795 event->type_len = RINGBUF_TYPE_TIME_STAMP; ··· 2663 2796 event->type_len = RINGBUF_TYPE_TIME_EXTEND; 2664 2797 2665 2798 /* Not the first event on the page, or not delta? */ 2666 - if (abs || rb_event_index(event)) { 2799 + if (abs || rb_event_index(cpu_buffer, event)) { 2667 2800 event->time_delta = delta & TS_MASK; 2668 2801 event->array[0] = delta >> TS_SHIFT; 2669 2802 } else { ··· 2693 2826 (unsigned long long)info->ts, 2694 2827 (unsigned long long)info->before, 2695 2828 (unsigned long long)info->after, 2696 - (unsigned long long)(rb_time_read(&cpu_buffer->write_stamp, &write_stamp) ? write_stamp : 0), 2829 + (unsigned long long)({rb_time_read(&cpu_buffer->write_stamp, &write_stamp); write_stamp;}), 2697 2830 sched_clock_stable() ? "" : 2698 2831 "If you just came from a suspend/resume,\n" 2699 2832 "please switch to the trace global clock:\n" ··· 2737 2870 if (!abs) 2738 2871 info->delta = 0; 2739 2872 } 2740 - *event = rb_add_time_stamp(*event, info->delta, abs); 2873 + *event = rb_add_time_stamp(cpu_buffer, *event, info->delta, abs); 2741 2874 *length -= RB_LEN_TIME_EXTEND; 2742 2875 *delta = 0; 2743 2876 } ··· 2821 2954 struct buffer_page *bpage; 2822 2955 unsigned long addr; 2823 2956 2824 - new_index = rb_event_index(event); 2957 + new_index = rb_event_index(cpu_buffer, event); 2825 2958 old_index = new_index + rb_event_ts_length(event); 2826 2959 addr = (unsigned long)event; 2827 - addr &= PAGE_MASK; 2960 + addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); 2828 2961 2829 2962 bpage = READ_ONCE(cpu_buffer->tail_page); 2830 2963 ··· 3211 3344 #define CHECK_FULL_PAGE 1L 3212 3345 3213 3346 #ifdef CONFIG_RING_BUFFER_VALIDATE_TIME_DELTAS 3347 + 3348 + static const char *show_irq_str(int bits) 3349 + { 3350 + const char *type[] = { 3351 + ".", // 0 3352 + "s", // 1 3353 + "h", // 2 3354 + "Hs", // 3 3355 + "n", // 4 3356 + "Ns", // 5 3357 + "Nh", // 6 3358 + "NHs", // 7 3359 + }; 3360 + 3361 + return type[bits]; 3362 + } 3363 + 3364 + /* Assume this is an trace event */ 3365 + static const char *show_flags(struct ring_buffer_event *event) 3366 + { 3367 + struct trace_entry *entry; 3368 + int bits = 0; 3369 + 3370 + if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry)) 3371 + return "X"; 3372 + 3373 + entry = ring_buffer_event_data(event); 3374 + 3375 + if (entry->flags & TRACE_FLAG_SOFTIRQ) 3376 + bits |= 1; 3377 + 3378 + if (entry->flags & TRACE_FLAG_HARDIRQ) 3379 + bits |= 2; 3380 + 3381 + if (entry->flags & TRACE_FLAG_NMI) 3382 + bits |= 4; 3383 + 3384 + return show_irq_str(bits); 3385 + } 3386 + 3387 + static const char *show_irq(struct ring_buffer_event *event) 3388 + { 3389 + struct trace_entry *entry; 3390 + 3391 + if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry)) 3392 + return ""; 3393 + 3394 + entry = ring_buffer_event_data(event); 3395 + if (entry->flags & TRACE_FLAG_IRQS_OFF) 3396 + return "d"; 3397 + return ""; 3398 + } 3399 + 3400 + static const char *show_interrupt_level(void) 3401 + { 3402 + unsigned long pc = preempt_count(); 3403 + unsigned char level = 0; 3404 + 3405 + if (pc & SOFTIRQ_OFFSET) 3406 + level |= 1; 3407 + 3408 + if (pc & HARDIRQ_MASK) 3409 + level |= 2; 3410 + 3411 + if (pc & NMI_MASK) 3412 + level |= 4; 3413 + 3414 + return show_irq_str(level); 3415 + } 3416 + 3214 3417 static void dump_buffer_page(struct buffer_data_page *bpage, 3215 3418 struct rb_event_info *info, 3216 3419 unsigned long tail) ··· 3301 3364 case RINGBUF_TYPE_TIME_EXTEND: 3302 3365 delta = rb_event_time_stamp(event); 3303 3366 ts += delta; 3304 - pr_warn(" [%lld] delta:%lld TIME EXTEND\n", ts, delta); 3367 + pr_warn(" 0x%x: [%lld] delta:%lld TIME EXTEND\n", 3368 + e, ts, delta); 3305 3369 break; 3306 3370 3307 3371 case RINGBUF_TYPE_TIME_STAMP: 3308 3372 delta = rb_event_time_stamp(event); 3309 3373 ts = rb_fix_abs_ts(delta, ts); 3310 - pr_warn(" [%lld] absolute:%lld TIME STAMP\n", ts, delta); 3374 + pr_warn(" 0x%x: [%lld] absolute:%lld TIME STAMP\n", 3375 + e, ts, delta); 3311 3376 break; 3312 3377 3313 3378 case RINGBUF_TYPE_PADDING: 3314 3379 ts += event->time_delta; 3315 - pr_warn(" [%lld] delta:%d PADDING\n", ts, event->time_delta); 3380 + pr_warn(" 0x%x: [%lld] delta:%d PADDING\n", 3381 + e, ts, event->time_delta); 3316 3382 break; 3317 3383 3318 3384 case RINGBUF_TYPE_DATA: 3319 3385 ts += event->time_delta; 3320 - pr_warn(" [%lld] delta:%d\n", ts, event->time_delta); 3386 + pr_warn(" 0x%x: [%lld] delta:%d %s%s\n", 3387 + e, ts, event->time_delta, 3388 + show_flags(event), show_irq(event)); 3321 3389 break; 3322 3390 3323 3391 default: 3324 3392 break; 3325 3393 } 3326 3394 } 3395 + pr_warn("expected end:0x%lx last event actually ended at:0x%x\n", tail, e); 3327 3396 } 3328 3397 3329 3398 static DEFINE_PER_CPU(atomic_t, checking); 3330 3399 static atomic_t ts_dump; 3400 + 3401 + #define buffer_warn_return(fmt, ...) \ 3402 + do { \ 3403 + /* If another report is happening, ignore this one */ \ 3404 + if (atomic_inc_return(&ts_dump) != 1) { \ 3405 + atomic_dec(&ts_dump); \ 3406 + goto out; \ 3407 + } \ 3408 + atomic_inc(&cpu_buffer->record_disabled); \ 3409 + pr_warn(fmt, ##__VA_ARGS__); \ 3410 + dump_buffer_page(bpage, info, tail); \ 3411 + atomic_dec(&ts_dump); \ 3412 + /* There's some cases in boot up that this can happen */ \ 3413 + if (WARN_ON_ONCE(system_state != SYSTEM_BOOTING)) \ 3414 + /* Do not re-enable checking */ \ 3415 + return; \ 3416 + } while (0) 3331 3417 3332 3418 /* 3333 3419 * Check if the current event time stamp matches the deltas on ··· 3405 3445 3406 3446 case RINGBUF_TYPE_TIME_STAMP: 3407 3447 delta = rb_event_time_stamp(event); 3408 - ts = rb_fix_abs_ts(delta, ts); 3448 + delta = rb_fix_abs_ts(delta, ts); 3449 + if (delta < ts) { 3450 + buffer_warn_return("[CPU: %d]ABSOLUTE TIME WENT BACKWARDS: last ts: %lld absolute ts: %lld\n", 3451 + cpu_buffer->cpu, ts, delta); 3452 + } 3453 + ts = delta; 3409 3454 break; 3410 3455 3411 3456 case RINGBUF_TYPE_PADDING: ··· 3427 3462 } 3428 3463 if ((full && ts > info->ts) || 3429 3464 (!full && ts + info->delta != info->ts)) { 3430 - /* If another report is happening, ignore this one */ 3431 - if (atomic_inc_return(&ts_dump) != 1) { 3432 - atomic_dec(&ts_dump); 3433 - goto out; 3434 - } 3435 - atomic_inc(&cpu_buffer->record_disabled); 3436 - /* There's some cases in boot up that this can happen */ 3437 - WARN_ON_ONCE(system_state != SYSTEM_BOOTING); 3438 - pr_warn("[CPU: %d]TIME DOES NOT MATCH expected:%lld actual:%lld delta:%lld before:%lld after:%lld%s\n", 3439 - cpu_buffer->cpu, 3440 - ts + info->delta, info->ts, info->delta, 3441 - info->before, info->after, 3442 - full ? " (full)" : ""); 3443 - dump_buffer_page(bpage, info, tail); 3444 - atomic_dec(&ts_dump); 3445 - /* Do not re-enable checking */ 3446 - return; 3465 + buffer_warn_return("[CPU: %d]TIME DOES NOT MATCH expected:%lld actual:%lld delta:%lld before:%lld after:%lld%s context:%s\n", 3466 + cpu_buffer->cpu, 3467 + ts + info->delta, info->ts, info->delta, 3468 + info->before, info->after, 3469 + full ? " (full)" : "", show_interrupt_level()); 3447 3470 } 3448 3471 out: 3449 3472 atomic_dec(this_cpu_ptr(&checking)); ··· 3451 3498 struct ring_buffer_event *event; 3452 3499 struct buffer_page *tail_page; 3453 3500 unsigned long tail, write, w; 3454 - bool a_ok; 3455 - bool b_ok; 3456 3501 3457 3502 /* Don't let the compiler play games with cpu_buffer->tail_page */ 3458 3503 tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page); 3459 3504 3460 3505 /*A*/ w = local_read(&tail_page->write) & RB_WRITE_MASK; 3461 3506 barrier(); 3462 - b_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before); 3463 - a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after); 3507 + rb_time_read(&cpu_buffer->before_stamp, &info->before); 3508 + rb_time_read(&cpu_buffer->write_stamp, &info->after); 3464 3509 barrier(); 3465 3510 info->ts = rb_time_stamp(cpu_buffer->buffer); 3466 3511 ··· 3473 3522 if (!w) { 3474 3523 /* Use the sub-buffer timestamp */ 3475 3524 info->delta = 0; 3476 - } else if (unlikely(!a_ok || !b_ok || info->before != info->after)) { 3525 + } else if (unlikely(info->before != info->after)) { 3477 3526 info->add_timestamp |= RB_ADD_STAMP_FORCE | RB_ADD_STAMP_EXTEND; 3478 3527 info->length += RB_LEN_TIME_EXTEND; 3479 3528 } else { ··· 3495 3544 tail = write - info->length; 3496 3545 3497 3546 /* See if we shot pass the end of this buffer page */ 3498 - if (unlikely(write > BUF_PAGE_SIZE)) { 3547 + if (unlikely(write > cpu_buffer->buffer->subbuf_size)) { 3499 3548 check_buffer(cpu_buffer, info, CHECK_FULL_PAGE); 3500 3549 return rb_move_tail(cpu_buffer, tail, info); 3501 3550 } ··· 3522 3571 /* SLOW PATH - Interrupted between A and C */ 3523 3572 3524 3573 /* Save the old before_stamp */ 3525 - a_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before); 3526 - RB_WARN_ON(cpu_buffer, !a_ok); 3574 + rb_time_read(&cpu_buffer->before_stamp, &info->before); 3527 3575 3528 3576 /* 3529 3577 * Read a new timestamp and update the before_stamp to make ··· 3534 3584 rb_time_set(&cpu_buffer->before_stamp, ts); 3535 3585 3536 3586 barrier(); 3537 - /*E*/ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after); 3538 - /* Was interrupted before here, write_stamp must be valid */ 3539 - RB_WARN_ON(cpu_buffer, !a_ok); 3587 + /*E*/ rb_time_read(&cpu_buffer->write_stamp, &info->after); 3540 3588 barrier(); 3541 3589 /*F*/ if (write == (local_read(&tail_page->write) & RB_WRITE_MASK) && 3542 3590 info->after == info->before && info->after < ts) { ··· 3626 3678 if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) { 3627 3679 add_ts_default = RB_ADD_STAMP_ABSOLUTE; 3628 3680 info.length += RB_LEN_TIME_EXTEND; 3629 - if (info.length > BUF_MAX_DATA_SIZE) 3681 + if (info.length > cpu_buffer->buffer->max_data_size) 3630 3682 goto out_fail; 3631 3683 } else { 3632 3684 add_ts_default = RB_ADD_STAMP_NONE; ··· 3701 3753 if (unlikely(atomic_read(&cpu_buffer->record_disabled))) 3702 3754 goto out; 3703 3755 3704 - if (unlikely(length > BUF_MAX_DATA_SIZE)) 3756 + if (unlikely(length > buffer->max_data_size)) 3705 3757 goto out; 3706 3758 3707 3759 if (unlikely(trace_recursive_lock(cpu_buffer))) ··· 3735 3787 struct buffer_page *bpage = cpu_buffer->commit_page; 3736 3788 struct buffer_page *start; 3737 3789 3738 - addr &= PAGE_MASK; 3790 + addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); 3739 3791 3740 3792 /* Do the likely case first */ 3741 3793 if (likely(bpage->page == (void *)addr)) { ··· 3851 3903 if (atomic_read(&cpu_buffer->record_disabled)) 3852 3904 goto out; 3853 3905 3854 - if (length > BUF_MAX_DATA_SIZE) 3906 + if (length > buffer->max_data_size) 3855 3907 goto out; 3856 3908 3857 3909 if (unlikely(trace_recursive_lock(cpu_buffer))) ··· 4431 4483 rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) 4432 4484 { 4433 4485 struct buffer_page *reader = NULL; 4486 + unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); 4434 4487 unsigned long overwrite; 4435 4488 unsigned long flags; 4436 4489 int nr_loops = 0; ··· 4567 4618 #define USECS_WAIT 1000000 4568 4619 for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) { 4569 4620 /* If the write is past the end of page, a writer is still updating it */ 4570 - if (likely(!reader || rb_page_write(reader) <= BUF_PAGE_SIZE)) 4621 + if (likely(!reader || rb_page_write(reader) <= bsize)) 4571 4622 break; 4572 4623 4573 4624 udelay(1); ··· 5011 5062 return NULL; 5012 5063 5013 5064 /* Holds the entire event: data and meta data */ 5014 - iter->event = kmalloc(BUF_PAGE_SIZE, flags); 5065 + iter->event_size = buffer->subbuf_size; 5066 + iter->event = kmalloc(iter->event_size, flags); 5015 5067 if (!iter->event) { 5016 5068 kfree(iter); 5017 5069 return NULL; ··· 5128 5178 */ 5129 5179 unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu) 5130 5180 { 5131 - /* 5132 - * Earlier, this method returned 5133 - * BUF_PAGE_SIZE * buffer->nr_pages 5134 - * Since the nr_pages field is now removed, we have converted this to 5135 - * return the per cpu buffer value. 5136 - */ 5137 5181 if (!cpumask_test_cpu(cpu, buffer->cpumask)) 5138 5182 return 0; 5139 5183 5140 - return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages; 5184 + return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages; 5141 5185 } 5142 5186 EXPORT_SYMBOL_GPL(ring_buffer_size); 5187 + 5188 + /** 5189 + * ring_buffer_max_event_size - return the max data size of an event 5190 + * @buffer: The ring buffer. 5191 + * 5192 + * Returns the maximum size an event can be. 5193 + */ 5194 + unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer) 5195 + { 5196 + /* If abs timestamp is requested, events have a timestamp too */ 5197 + if (ring_buffer_time_stamp_abs(buffer)) 5198 + return buffer->max_data_size - RB_LEN_TIME_EXTEND; 5199 + return buffer->max_data_size; 5200 + } 5201 + EXPORT_SYMBOL_GPL(ring_buffer_max_event_size); 5143 5202 5144 5203 static void rb_clear_buffer_page(struct buffer_page *page) 5145 5204 { ··· 5420 5461 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages) 5421 5462 goto out; 5422 5463 5464 + if (buffer_a->subbuf_order != buffer_b->subbuf_order) 5465 + goto out; 5466 + 5423 5467 ret = -EAGAIN; 5424 5468 5425 5469 if (atomic_read(&buffer_a->record_disabled)) ··· 5494 5532 * Returns: 5495 5533 * The page allocated, or ERR_PTR 5496 5534 */ 5497 - void *ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) 5535 + struct buffer_data_read_page * 5536 + ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) 5498 5537 { 5499 5538 struct ring_buffer_per_cpu *cpu_buffer; 5500 - struct buffer_data_page *bpage = NULL; 5539 + struct buffer_data_read_page *bpage = NULL; 5501 5540 unsigned long flags; 5502 5541 struct page *page; 5503 5542 5504 5543 if (!cpumask_test_cpu(cpu, buffer->cpumask)) 5505 5544 return ERR_PTR(-ENODEV); 5506 5545 5546 + bpage = kzalloc(sizeof(*bpage), GFP_KERNEL); 5547 + if (!bpage) 5548 + return ERR_PTR(-ENOMEM); 5549 + 5550 + bpage->order = buffer->subbuf_order; 5507 5551 cpu_buffer = buffer->buffers[cpu]; 5508 5552 local_irq_save(flags); 5509 5553 arch_spin_lock(&cpu_buffer->lock); 5510 5554 5511 5555 if (cpu_buffer->free_page) { 5512 - bpage = cpu_buffer->free_page; 5556 + bpage->data = cpu_buffer->free_page; 5513 5557 cpu_buffer->free_page = NULL; 5514 5558 } 5515 5559 5516 5560 arch_spin_unlock(&cpu_buffer->lock); 5517 5561 local_irq_restore(flags); 5518 5562 5519 - if (bpage) 5563 + if (bpage->data) 5520 5564 goto out; 5521 5565 5522 - page = alloc_pages_node(cpu_to_node(cpu), 5523 - GFP_KERNEL | __GFP_NORETRY, 0); 5524 - if (!page) 5566 + page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL | __GFP_NORETRY, 5567 + cpu_buffer->buffer->subbuf_order); 5568 + if (!page) { 5569 + kfree(bpage); 5525 5570 return ERR_PTR(-ENOMEM); 5571 + } 5526 5572 5527 - bpage = page_address(page); 5573 + bpage->data = page_address(page); 5528 5574 5529 5575 out: 5530 - rb_init_page(bpage); 5576 + rb_init_page(bpage->data); 5531 5577 5532 5578 return bpage; 5533 5579 } ··· 5545 5575 * ring_buffer_free_read_page - free an allocated read page 5546 5576 * @buffer: the buffer the page was allocate for 5547 5577 * @cpu: the cpu buffer the page came from 5548 - * @data: the page to free 5578 + * @data_page: the page to free 5549 5579 * 5550 5580 * Free a page allocated from ring_buffer_alloc_read_page. 5551 5581 */ 5552 - void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data) 5582 + void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, 5583 + struct buffer_data_read_page *data_page) 5553 5584 { 5554 5585 struct ring_buffer_per_cpu *cpu_buffer; 5555 - struct buffer_data_page *bpage = data; 5586 + struct buffer_data_page *bpage = data_page->data; 5556 5587 struct page *page = virt_to_page(bpage); 5557 5588 unsigned long flags; 5558 5589 ··· 5562 5591 5563 5592 cpu_buffer = buffer->buffers[cpu]; 5564 5593 5565 - /* If the page is still in use someplace else, we can't reuse it */ 5566 - if (page_ref_count(page) > 1) 5594 + /* 5595 + * If the page is still in use someplace else, or order of the page 5596 + * is different from the subbuffer order of the buffer - 5597 + * we can't reuse it 5598 + */ 5599 + if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order) 5567 5600 goto out; 5568 5601 5569 5602 local_irq_save(flags); ··· 5582 5607 local_irq_restore(flags); 5583 5608 5584 5609 out: 5585 - free_page((unsigned long)bpage); 5610 + free_pages((unsigned long)bpage, data_page->order); 5611 + kfree(data_page); 5586 5612 } 5587 5613 EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); 5588 5614 ··· 5604 5628 * rpage = ring_buffer_alloc_read_page(buffer, cpu); 5605 5629 * if (IS_ERR(rpage)) 5606 5630 * return PTR_ERR(rpage); 5607 - * ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0); 5631 + * ret = ring_buffer_read_page(buffer, rpage, len, cpu, 0); 5608 5632 * if (ret >= 0) 5609 - * process_page(rpage, ret); 5633 + * process_page(ring_buffer_read_page_data(rpage), ret); 5634 + * ring_buffer_free_read_page(buffer, cpu, rpage); 5610 5635 * 5611 5636 * When @full is set, the function will not return true unless 5612 5637 * the writer is off the reader page. ··· 5622 5645 * <0 if no data has been transferred. 5623 5646 */ 5624 5647 int ring_buffer_read_page(struct trace_buffer *buffer, 5625 - void **data_page, size_t len, int cpu, int full) 5648 + struct buffer_data_read_page *data_page, 5649 + size_t len, int cpu, int full) 5626 5650 { 5627 5651 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; 5628 5652 struct ring_buffer_event *event; ··· 5648 5670 5649 5671 len -= BUF_PAGE_HDR_SIZE; 5650 5672 5651 - if (!data_page) 5673 + if (!data_page || !data_page->data) 5674 + goto out; 5675 + if (data_page->order != buffer->subbuf_order) 5652 5676 goto out; 5653 5677 5654 - bpage = *data_page; 5678 + bpage = data_page->data; 5655 5679 if (!bpage) 5656 5680 goto out; 5657 5681 ··· 5747 5767 /* swap the pages */ 5748 5768 rb_init_page(bpage); 5749 5769 bpage = reader->page; 5750 - reader->page = *data_page; 5770 + reader->page = data_page->data; 5751 5771 local_set(&reader->write, 0); 5752 5772 local_set(&reader->entries, 0); 5753 5773 reader->read = 0; 5754 - *data_page = bpage; 5774 + data_page->data = bpage; 5755 5775 5756 5776 /* 5757 5777 * Use the real_end for the data size, ··· 5773 5793 /* If there is room at the end of the page to save the 5774 5794 * missed events, then record it there. 5775 5795 */ 5776 - if (BUF_PAGE_SIZE - commit >= sizeof(missed_events)) { 5796 + if (buffer->subbuf_size - commit >= sizeof(missed_events)) { 5777 5797 memcpy(&bpage->data[commit], &missed_events, 5778 5798 sizeof(missed_events)); 5779 5799 local_add(RB_MISSED_STORED, &bpage->commit); ··· 5785 5805 /* 5786 5806 * This page may be off to user land. Zero it out here. 5787 5807 */ 5788 - if (commit < BUF_PAGE_SIZE) 5789 - memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit); 5808 + if (commit < buffer->subbuf_size) 5809 + memset(&bpage->data[commit], 0, buffer->subbuf_size - commit); 5790 5810 5791 5811 out_unlock: 5792 5812 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); ··· 5795 5815 return ret; 5796 5816 } 5797 5817 EXPORT_SYMBOL_GPL(ring_buffer_read_page); 5818 + 5819 + /** 5820 + * ring_buffer_read_page_data - get pointer to the data in the page. 5821 + * @page: the page to get the data from 5822 + * 5823 + * Returns pointer to the actual data in this page. 5824 + */ 5825 + void *ring_buffer_read_page_data(struct buffer_data_read_page *page) 5826 + { 5827 + return page->data; 5828 + } 5829 + EXPORT_SYMBOL_GPL(ring_buffer_read_page_data); 5830 + 5831 + /** 5832 + * ring_buffer_subbuf_size_get - get size of the sub buffer. 5833 + * @buffer: the buffer to get the sub buffer size from 5834 + * 5835 + * Returns size of the sub buffer, in bytes. 5836 + */ 5837 + int ring_buffer_subbuf_size_get(struct trace_buffer *buffer) 5838 + { 5839 + return buffer->subbuf_size + BUF_PAGE_HDR_SIZE; 5840 + } 5841 + EXPORT_SYMBOL_GPL(ring_buffer_subbuf_size_get); 5842 + 5843 + /** 5844 + * ring_buffer_subbuf_order_get - get order of system sub pages in one buffer page. 5845 + * @buffer: The ring_buffer to get the system sub page order from 5846 + * 5847 + * By default, one ring buffer sub page equals to one system page. This parameter 5848 + * is configurable, per ring buffer. The size of the ring buffer sub page can be 5849 + * extended, but must be an order of system page size. 5850 + * 5851 + * Returns the order of buffer sub page size, in system pages: 5852 + * 0 means the sub buffer size is 1 system page and so forth. 5853 + * In case of an error < 0 is returned. 5854 + */ 5855 + int ring_buffer_subbuf_order_get(struct trace_buffer *buffer) 5856 + { 5857 + if (!buffer) 5858 + return -EINVAL; 5859 + 5860 + return buffer->subbuf_order; 5861 + } 5862 + EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_get); 5863 + 5864 + /** 5865 + * ring_buffer_subbuf_order_set - set the size of ring buffer sub page. 5866 + * @buffer: The ring_buffer to set the new page size. 5867 + * @order: Order of the system pages in one sub buffer page 5868 + * 5869 + * By default, one ring buffer pages equals to one system page. This API can be 5870 + * used to set new size of the ring buffer page. The size must be order of 5871 + * system page size, that's why the input parameter @order is the order of 5872 + * system pages that are allocated for one ring buffer page: 5873 + * 0 - 1 system page 5874 + * 1 - 2 system pages 5875 + * 3 - 4 system pages 5876 + * ... 5877 + * 5878 + * Returns 0 on success or < 0 in case of an error. 5879 + */ 5880 + int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) 5881 + { 5882 + struct ring_buffer_per_cpu *cpu_buffer; 5883 + struct buffer_page *bpage, *tmp; 5884 + int old_order, old_size; 5885 + int nr_pages; 5886 + int psize; 5887 + int err; 5888 + int cpu; 5889 + 5890 + if (!buffer || order < 0) 5891 + return -EINVAL; 5892 + 5893 + if (buffer->subbuf_order == order) 5894 + return 0; 5895 + 5896 + psize = (1 << order) * PAGE_SIZE; 5897 + if (psize <= BUF_PAGE_HDR_SIZE) 5898 + return -EINVAL; 5899 + 5900 + old_order = buffer->subbuf_order; 5901 + old_size = buffer->subbuf_size; 5902 + 5903 + /* prevent another thread from changing buffer sizes */ 5904 + mutex_lock(&buffer->mutex); 5905 + atomic_inc(&buffer->record_disabled); 5906 + 5907 + /* Make sure all commits have finished */ 5908 + synchronize_rcu(); 5909 + 5910 + buffer->subbuf_order = order; 5911 + buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE; 5912 + 5913 + /* Make sure all new buffers are allocated, before deleting the old ones */ 5914 + for_each_buffer_cpu(buffer, cpu) { 5915 + 5916 + if (!cpumask_test_cpu(cpu, buffer->cpumask)) 5917 + continue; 5918 + 5919 + cpu_buffer = buffer->buffers[cpu]; 5920 + 5921 + /* Update the number of pages to match the new size */ 5922 + nr_pages = old_size * buffer->buffers[cpu]->nr_pages; 5923 + nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size); 5924 + 5925 + /* we need a minimum of two pages */ 5926 + if (nr_pages < 2) 5927 + nr_pages = 2; 5928 + 5929 + cpu_buffer->nr_pages_to_update = nr_pages; 5930 + 5931 + /* Include the reader page */ 5932 + nr_pages++; 5933 + 5934 + /* Allocate the new size buffer */ 5935 + INIT_LIST_HEAD(&cpu_buffer->new_pages); 5936 + if (__rb_allocate_pages(cpu_buffer, nr_pages, 5937 + &cpu_buffer->new_pages)) { 5938 + /* not enough memory for new pages */ 5939 + err = -ENOMEM; 5940 + goto error; 5941 + } 5942 + } 5943 + 5944 + for_each_buffer_cpu(buffer, cpu) { 5945 + 5946 + if (!cpumask_test_cpu(cpu, buffer->cpumask)) 5947 + continue; 5948 + 5949 + cpu_buffer = buffer->buffers[cpu]; 5950 + 5951 + /* Clear the head bit to make the link list normal to read */ 5952 + rb_head_page_deactivate(cpu_buffer); 5953 + 5954 + /* Now walk the list and free all the old sub buffers */ 5955 + list_for_each_entry_safe(bpage, tmp, cpu_buffer->pages, list) { 5956 + list_del_init(&bpage->list); 5957 + free_buffer_page(bpage); 5958 + } 5959 + /* The above loop stopped an the last page needing to be freed */ 5960 + bpage = list_entry(cpu_buffer->pages, struct buffer_page, list); 5961 + free_buffer_page(bpage); 5962 + 5963 + /* Free the current reader page */ 5964 + free_buffer_page(cpu_buffer->reader_page); 5965 + 5966 + /* One page was allocated for the reader page */ 5967 + cpu_buffer->reader_page = list_entry(cpu_buffer->new_pages.next, 5968 + struct buffer_page, list); 5969 + list_del_init(&cpu_buffer->reader_page->list); 5970 + 5971 + /* The cpu_buffer pages are a link list with no head */ 5972 + cpu_buffer->pages = cpu_buffer->new_pages.next; 5973 + cpu_buffer->new_pages.next->prev = cpu_buffer->new_pages.prev; 5974 + cpu_buffer->new_pages.prev->next = cpu_buffer->new_pages.next; 5975 + 5976 + /* Clear the new_pages list */ 5977 + INIT_LIST_HEAD(&cpu_buffer->new_pages); 5978 + 5979 + cpu_buffer->head_page 5980 + = list_entry(cpu_buffer->pages, struct buffer_page, list); 5981 + cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page; 5982 + 5983 + cpu_buffer->nr_pages = cpu_buffer->nr_pages_to_update; 5984 + cpu_buffer->nr_pages_to_update = 0; 5985 + 5986 + free_pages((unsigned long)cpu_buffer->free_page, old_order); 5987 + cpu_buffer->free_page = NULL; 5988 + 5989 + rb_head_page_activate(cpu_buffer); 5990 + 5991 + rb_check_pages(cpu_buffer); 5992 + } 5993 + 5994 + atomic_dec(&buffer->record_disabled); 5995 + mutex_unlock(&buffer->mutex); 5996 + 5997 + return 0; 5998 + 5999 + error: 6000 + buffer->subbuf_order = old_order; 6001 + buffer->subbuf_size = old_size; 6002 + 6003 + atomic_dec(&buffer->record_disabled); 6004 + mutex_unlock(&buffer->mutex); 6005 + 6006 + for_each_buffer_cpu(buffer, cpu) { 6007 + cpu_buffer = buffer->buffers[cpu]; 6008 + 6009 + if (!cpu_buffer->nr_pages_to_update) 6010 + continue; 6011 + 6012 + list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, list) { 6013 + list_del_init(&bpage->list); 6014 + free_buffer_page(bpage); 6015 + } 6016 + } 6017 + 6018 + return err; 6019 + } 6020 + EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_set); 5798 6021 5799 6022 /* 5800 6023 * We only allocate new buffers, never free them if the CPU goes down.

+6 -4

kernel/trace/ring_buffer_benchmark.c

··· 104 104 105 105 static enum event_status read_page(int cpu) 106 106 { 107 + struct buffer_data_read_page *bpage; 107 108 struct ring_buffer_event *event; 108 109 struct rb_page *rpage; 109 110 unsigned long commit; 110 - void *bpage; 111 + int page_size; 111 112 int *entry; 112 113 int ret; 113 114 int inc; ··· 118 117 if (IS_ERR(bpage)) 119 118 return EVENT_DROPPED; 120 119 121 - ret = ring_buffer_read_page(buffer, &bpage, PAGE_SIZE, cpu, 1); 120 + page_size = ring_buffer_subbuf_size_get(buffer); 121 + ret = ring_buffer_read_page(buffer, bpage, page_size, cpu, 1); 122 122 if (ret >= 0) { 123 - rpage = bpage; 123 + rpage = ring_buffer_read_page_data(bpage); 124 124 /* The commit may have missed event flags set, clear them */ 125 125 commit = local_read(&rpage->commit) & 0xfffff; 126 126 for (i = 0; i < commit && !test_error ; i += inc) { 127 127 128 - if (i >= (PAGE_SIZE - offsetof(struct rb_page, data))) { 128 + if (i >= (page_size - offsetof(struct rb_page, data))) { 129 129 TEST_ERROR(); 130 130 break; 131 131 }

+196 -38

kernel/trace/trace.c

··· 1263 1263 1264 1264 int tracing_alloc_snapshot_instance(struct trace_array *tr) 1265 1265 { 1266 + int order; 1266 1267 int ret; 1267 1268 1268 1269 if (!tr->allocated_snapshot) { 1270 + 1271 + /* Make the snapshot buffer have the same order as main buffer */ 1272 + order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); 1273 + ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order); 1274 + if (ret < 0) 1275 + return ret; 1269 1276 1270 1277 /* allocate spare buffer */ 1271 1278 ret = resize_buffer_duplicate_size(&tr->max_buffer, ··· 1293 1286 * The max_tr ring buffer has some state (e.g. ring->clock) and 1294 1287 * we want preserve it. 1295 1288 */ 1289 + ring_buffer_subbuf_order_set(tr->max_buffer.buffer, 0); 1296 1290 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); 1297 1291 set_buffer_entries(&tr->max_buffer, 1); 1298 1292 tracing_reset_online_cpus(&tr->max_buffer); ··· 3775 3767 3776 3768 /* OK if part of the temp seq buffer */ 3777 3769 if ((addr >= (unsigned long)iter->tmp_seq.buffer) && 3778 - (addr < (unsigned long)iter->tmp_seq.buffer + PAGE_SIZE)) 3770 + (addr < (unsigned long)iter->tmp_seq.buffer + TRACE_SEQ_BUFFER_SIZE)) 3779 3771 return true; 3780 3772 3781 3773 /* Core rodata can not be freed */ ··· 5040 5032 return 0; 5041 5033 } 5042 5034 5043 - static int tracing_release_generic_tr(struct inode *inode, struct file *file) 5035 + int tracing_release_generic_tr(struct inode *inode, struct file *file) 5044 5036 { 5045 5037 struct trace_array *tr = inode->i_private; 5046 5038 ··· 6954 6946 goto out; 6955 6947 } 6956 6948 6957 - if (cnt >= PAGE_SIZE) 6958 - cnt = PAGE_SIZE - 1; 6949 + if (cnt >= TRACE_SEQ_BUFFER_SIZE) 6950 + cnt = TRACE_SEQ_BUFFER_SIZE - 1; 6959 6951 6960 6952 /* reset all but tr, trace, and overruns */ 6961 6953 trace_iterator_reset(iter); ··· 7300 7292 enum event_trigger_type tt = ETT_NONE; 7301 7293 struct trace_buffer *buffer; 7302 7294 struct print_entry *entry; 7295 + int meta_size; 7303 7296 ssize_t written; 7304 - int size; 7297 + size_t size; 7305 7298 int len; 7306 7299 7307 7300 /* Used in tracing_mark_raw_write() as well */ ··· 7315 7306 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) 7316 7307 return -EINVAL; 7317 7308 7318 - if (cnt > TRACE_BUF_SIZE) 7319 - cnt = TRACE_BUF_SIZE; 7309 + if ((ssize_t)cnt < 0) 7310 + return -EINVAL; 7320 7311 7321 - BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); 7322 - 7323 - size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */ 7312 + meta_size = sizeof(*entry) + 2; /* add '\0' and possible '\n' */ 7313 + again: 7314 + size = cnt + meta_size; 7324 7315 7325 7316 /* If less than "<faulted>", then make sure we can still add that */ 7326 7317 if (cnt < FAULTED_SIZE) 7327 7318 size += FAULTED_SIZE - cnt; 7328 7319 7320 + if (size > TRACE_SEQ_BUFFER_SIZE) { 7321 + cnt -= size - TRACE_SEQ_BUFFER_SIZE; 7322 + goto again; 7323 + } 7324 + 7329 7325 buffer = tr->array_buffer.buffer; 7330 7326 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 7331 7327 tracing_gen_ctx()); 7332 - if (unlikely(!event)) 7328 + if (unlikely(!event)) { 7329 + /* 7330 + * If the size was greater than what was allowed, then 7331 + * make it smaller and try again. 7332 + */ 7333 + if (size > ring_buffer_max_event_size(buffer)) { 7334 + /* cnt < FAULTED size should never be bigger than max */ 7335 + if (WARN_ON_ONCE(cnt < FAULTED_SIZE)) 7336 + return -EBADF; 7337 + cnt = ring_buffer_max_event_size(buffer) - meta_size; 7338 + /* The above should only happen once */ 7339 + if (WARN_ON_ONCE(cnt + meta_size == size)) 7340 + return -EBADF; 7341 + goto again; 7342 + } 7343 + 7333 7344 /* Ring buffer disabled, return as if not open for write */ 7334 7345 return -EBADF; 7346 + } 7335 7347 7336 7348 entry = ring_buffer_event_data(event); 7337 7349 entry->ip = _THIS_IP_; ··· 7387 7357 return written; 7388 7358 } 7389 7359 7390 - /* Limit it for now to 3K (including tag) */ 7391 - #define RAW_DATA_MAX_SIZE (1024*3) 7392 - 7393 7360 static ssize_t 7394 7361 tracing_mark_raw_write(struct file *filp, const char __user *ubuf, 7395 7362 size_t cnt, loff_t *fpos) ··· 7408 7381 return -EINVAL; 7409 7382 7410 7383 /* The marker must at least have a tag id */ 7411 - if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE) 7384 + if (cnt < sizeof(unsigned int)) 7412 7385 return -EINVAL; 7413 - 7414 - if (cnt > TRACE_BUF_SIZE) 7415 - cnt = TRACE_BUF_SIZE; 7416 - 7417 - BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); 7418 7386 7419 7387 size = sizeof(*entry) + cnt; 7420 7388 if (cnt < FAULT_SIZE_ID) 7421 7389 size += FAULT_SIZE_ID - cnt; 7422 7390 7423 7391 buffer = tr->array_buffer.buffer; 7392 + 7393 + if (size > ring_buffer_max_event_size(buffer)) 7394 + return -EINVAL; 7395 + 7424 7396 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size, 7425 7397 tracing_gen_ctx()); 7426 7398 if (!event) ··· 7604 7578 struct trace_iterator iter; 7605 7579 void *spare; 7606 7580 unsigned int spare_cpu; 7581 + unsigned int spare_size; 7607 7582 unsigned int read; 7608 7583 }; 7609 7584 ··· 8309 8282 { 8310 8283 struct ftrace_buffer_info *info = filp->private_data; 8311 8284 struct trace_iterator *iter = &info->iter; 8285 + void *trace_data; 8286 + int page_size; 8312 8287 ssize_t ret = 0; 8313 8288 ssize_t size; 8314 8289 ··· 8322 8293 return -EBUSY; 8323 8294 #endif 8324 8295 8296 + page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); 8297 + 8298 + /* Make sure the spare matches the current sub buffer size */ 8299 + if (info->spare) { 8300 + if (page_size != info->spare_size) { 8301 + ring_buffer_free_read_page(iter->array_buffer->buffer, 8302 + info->spare_cpu, info->spare); 8303 + info->spare = NULL; 8304 + } 8305 + } 8306 + 8325 8307 if (!info->spare) { 8326 8308 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer, 8327 8309 iter->cpu_file); ··· 8341 8301 info->spare = NULL; 8342 8302 } else { 8343 8303 info->spare_cpu = iter->cpu_file; 8304 + info->spare_size = page_size; 8344 8305 } 8345 8306 } 8346 8307 if (!info->spare) 8347 8308 return ret; 8348 8309 8349 8310 /* Do we have previous read data to read? */ 8350 - if (info->read < PAGE_SIZE) 8311 + if (info->read < page_size) 8351 8312 goto read; 8352 8313 8353 8314 again: 8354 8315 trace_access_lock(iter->cpu_file); 8355 8316 ret = ring_buffer_read_page(iter->array_buffer->buffer, 8356 - &info->spare, 8317 + info->spare, 8357 8318 count, 8358 8319 iter->cpu_file, 0); 8359 8320 trace_access_unlock(iter->cpu_file); ··· 8375 8334 8376 8335 info->read = 0; 8377 8336 read: 8378 - size = PAGE_SIZE - info->read; 8337 + size = page_size - info->read; 8379 8338 if (size > count) 8380 8339 size = count; 8381 - 8382 - ret = copy_to_user(ubuf, info->spare + info->read, size); 8340 + trace_data = ring_buffer_read_page_data(info->spare); 8341 + ret = copy_to_user(ubuf, trace_data + info->read, size); 8383 8342 if (ret == size) 8384 8343 return -EFAULT; 8385 8344 ··· 8490 8449 .spd_release = buffer_spd_release, 8491 8450 }; 8492 8451 struct buffer_ref *ref; 8452 + int page_size; 8493 8453 int entries, i; 8494 8454 ssize_t ret = 0; 8495 8455 ··· 8499 8457 return -EBUSY; 8500 8458 #endif 8501 8459 8502 - if (*ppos & (PAGE_SIZE - 1)) 8460 + page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); 8461 + if (*ppos & (page_size - 1)) 8503 8462 return -EINVAL; 8504 8463 8505 - if (len & (PAGE_SIZE - 1)) { 8506 - if (len < PAGE_SIZE) 8464 + if (len & (page_size - 1)) { 8465 + if (len < page_size) 8507 8466 return -EINVAL; 8508 - len &= PAGE_MASK; 8467 + len &= (~(page_size - 1)); 8509 8468 } 8510 8469 8511 8470 if (splice_grow_spd(pipe, &spd)) ··· 8516 8473 trace_access_lock(iter->cpu_file); 8517 8474 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); 8518 8475 8519 - for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) { 8476 + for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= page_size) { 8520 8477 struct page *page; 8521 8478 int r; 8522 8479 ··· 8537 8494 } 8538 8495 ref->cpu = iter->cpu_file; 8539 8496 8540 - r = ring_buffer_read_page(ref->buffer, &ref->page, 8497 + r = ring_buffer_read_page(ref->buffer, ref->page, 8541 8498 len, iter->cpu_file, 1); 8542 8499 if (r < 0) { 8543 8500 ring_buffer_free_read_page(ref->buffer, ref->cpu, ··· 8546 8503 break; 8547 8504 } 8548 8505 8549 - page = virt_to_page(ref->page); 8506 + page = virt_to_page(ring_buffer_read_page_data(ref->page)); 8550 8507 8551 8508 spd.pages[i] = page; 8552 - spd.partial[i].len = PAGE_SIZE; 8509 + spd.partial[i].len = page_size; 8553 8510 spd.partial[i].offset = 0; 8554 8511 spd.partial[i].private = (unsigned long)ref; 8555 8512 spd.nr_pages++; 8556 - *ppos += PAGE_SIZE; 8513 + *ppos += page_size; 8557 8514 8558 8515 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); 8559 8516 } ··· 9397 9354 .llseek = default_llseek, 9398 9355 }; 9399 9356 9357 + static ssize_t 9358 + buffer_subbuf_size_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) 9359 + { 9360 + struct trace_array *tr = filp->private_data; 9361 + size_t size; 9362 + char buf[64]; 9363 + int order; 9364 + int r; 9365 + 9366 + order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); 9367 + size = (PAGE_SIZE << order) / 1024; 9368 + 9369 + r = sprintf(buf, "%zd\n", size); 9370 + 9371 + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 9372 + } 9373 + 9374 + static ssize_t 9375 + buffer_subbuf_size_write(struct file *filp, const char __user *ubuf, 9376 + size_t cnt, loff_t *ppos) 9377 + { 9378 + struct trace_array *tr = filp->private_data; 9379 + unsigned long val; 9380 + int old_order; 9381 + int order; 9382 + int pages; 9383 + int ret; 9384 + 9385 + ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 9386 + if (ret) 9387 + return ret; 9388 + 9389 + val *= 1024; /* value passed in is in KB */ 9390 + 9391 + pages = DIV_ROUND_UP(val, PAGE_SIZE); 9392 + order = fls(pages - 1); 9393 + 9394 + /* limit between 1 and 128 system pages */ 9395 + if (order < 0 || order > 7) 9396 + return -EINVAL; 9397 + 9398 + /* Do not allow tracing while changing the order of the ring buffer */ 9399 + tracing_stop_tr(tr); 9400 + 9401 + old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); 9402 + if (old_order == order) 9403 + goto out; 9404 + 9405 + ret = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, order); 9406 + if (ret) 9407 + goto out; 9408 + 9409 + #ifdef CONFIG_TRACER_MAX_TRACE 9410 + 9411 + if (!tr->allocated_snapshot) 9412 + goto out_max; 9413 + 9414 + ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order); 9415 + if (ret) { 9416 + /* Put back the old order */ 9417 + cnt = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, old_order); 9418 + if (WARN_ON_ONCE(cnt)) { 9419 + /* 9420 + * AARGH! We are left with different orders! 9421 + * The max buffer is our "snapshot" buffer. 9422 + * When a tracer needs a snapshot (one of the 9423 + * latency tracers), it swaps the max buffer 9424 + * with the saved snap shot. We succeeded to 9425 + * update the order of the main buffer, but failed to 9426 + * update the order of the max buffer. But when we tried 9427 + * to reset the main buffer to the original size, we 9428 + * failed there too. This is very unlikely to 9429 + * happen, but if it does, warn and kill all 9430 + * tracing. 9431 + */ 9432 + tracing_disabled = 1; 9433 + } 9434 + goto out; 9435 + } 9436 + out_max: 9437 + #endif 9438 + (*ppos)++; 9439 + out: 9440 + if (ret) 9441 + cnt = ret; 9442 + tracing_start_tr(tr); 9443 + return cnt; 9444 + } 9445 + 9446 + static const struct file_operations buffer_subbuf_size_fops = { 9447 + .open = tracing_open_generic_tr, 9448 + .read = buffer_subbuf_size_read, 9449 + .write = buffer_subbuf_size_write, 9450 + .release = tracing_release_generic_tr, 9451 + .llseek = default_llseek, 9452 + }; 9453 + 9400 9454 static struct dentry *trace_instance_dir; 9401 9455 9402 9456 static void ··· 9644 9504 return ret; 9645 9505 } 9646 9506 9647 - static struct trace_array *trace_array_create(const char *name) 9507 + static struct trace_array * 9508 + trace_array_create_systems(const char *name, const char *systems) 9648 9509 { 9649 9510 struct trace_array *tr; 9650 9511 int ret; ··· 9664 9523 9665 9524 if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL)) 9666 9525 goto out_free_tr; 9526 + 9527 + if (systems) { 9528 + tr->system_names = kstrdup_const(systems, GFP_KERNEL); 9529 + if (!tr->system_names) 9530 + goto out_free_tr; 9531 + } 9667 9532 9668 9533 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS; 9669 9534 ··· 9717 9570 free_trace_buffers(tr); 9718 9571 free_cpumask_var(tr->pipe_cpumask); 9719 9572 free_cpumask_var(tr->tracing_cpumask); 9573 + kfree_const(tr->system_names); 9720 9574 kfree(tr->name); 9721 9575 kfree(tr); 9722 9576 9723 9577 return ERR_PTR(ret); 9578 + } 9579 + 9580 + static struct trace_array *trace_array_create(const char *name) 9581 + { 9582 + return trace_array_create_systems(name, NULL); 9724 9583 } 9725 9584 9726 9585 static int instance_mkdir(const char *name) ··· 9754 9601 /** 9755 9602 * trace_array_get_by_name - Create/Lookup a trace array, given its name. 9756 9603 * @name: The name of the trace array to be looked up/created. 9604 + * @systems: A list of systems to create event directories for (NULL for all) 9757 9605 * 9758 9606 * Returns pointer to trace array with given name. 9759 9607 * NULL, if it cannot be created. ··· 9768 9614 * trace_array_put() is called, user space can not delete it. 9769 9615 * 9770 9616 */ 9771 - struct trace_array *trace_array_get_by_name(const char *name) 9617 + struct trace_array *trace_array_get_by_name(const char *name, const char *systems) 9772 9618 { 9773 9619 struct trace_array *tr; 9774 9620 ··· 9780 9626 goto out_unlock; 9781 9627 } 9782 9628 9783 - tr = trace_array_create(name); 9629 + tr = trace_array_create_systems(name, systems); 9784 9630 9785 9631 if (IS_ERR(tr)) 9786 9632 tr = NULL; ··· 9827 9673 9828 9674 free_cpumask_var(tr->pipe_cpumask); 9829 9675 free_cpumask_var(tr->tracing_cpumask); 9676 + kfree_const(tr->system_names); 9830 9677 kfree(tr->name); 9831 9678 kfree(tr); 9832 9679 ··· 9959 9804 9960 9805 trace_create_file("buffer_percent", TRACE_MODE_WRITE, d_tracer, 9961 9806 tr, &buffer_percent_fops); 9807 + 9808 + trace_create_file("buffer_subbuf_size_kb", TRACE_MODE_WRITE, d_tracer, 9809 + tr, &buffer_subbuf_size_fops); 9962 9810 9963 9811 create_trace_options_dir(tr); 9964 9812 ··· 10549 10391 if (IS_ENABLED(CONFIG_TRACER_MAX_TRACE)) 10550 10392 do_allocate_snapshot(tok); 10551 10393 10552 - tr = trace_array_get_by_name(tok); 10394 + tr = trace_array_get_by_name(tok, NULL); 10553 10395 if (!tr) { 10554 10396 pr_warn("Failed to create instance buffer %s\n", curr_str); 10555 10397 continue;

+2

kernel/trace/trace.h

··· 377 377 unsigned char trace_flags_index[TRACE_FLAGS_MAX_SIZE]; 378 378 unsigned int flags; 379 379 raw_spinlock_t start_lock; 380 + const char *system_names; 380 381 struct list_head err_log; 381 382 struct dentry *dir; 382 383 struct dentry *options; ··· 616 615 void tracing_reset_all_online_cpus_unlocked(void); 617 616 int tracing_open_generic(struct inode *inode, struct file *filp); 618 617 int tracing_open_generic_tr(struct inode *inode, struct file *filp); 618 + int tracing_release_generic_tr(struct inode *inode, struct file *file); 619 619 int tracing_open_file_tr(struct inode *inode, struct file *filp); 620 620 int tracing_release_file_tr(struct inode *inode, struct file *filp); 621 621 int tracing_single_release_file_tr(struct inode *inode, struct file *filp);

+1 -1

kernel/trace/trace_boot.c

··· 633 633 if (!p || *p == '\0') 634 634 continue; 635 635 636 - tr = trace_array_get_by_name(p); 636 + tr = trace_array_get_by_name(p, NULL); 637 637 if (!tr) { 638 638 pr_err("Failed to get trace instance %s\n", p); 639 639 continue;

+90 -17

kernel/trace/trace_events.c

··· 1893 1893 } 1894 1894 1895 1895 static ssize_t 1896 - show_header(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) 1896 + show_header_page_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) 1897 1897 { 1898 - int (*func)(struct trace_seq *s) = filp->private_data; 1898 + struct trace_array *tr = filp->private_data; 1899 1899 struct trace_seq *s; 1900 1900 int r; 1901 1901 ··· 1908 1908 1909 1909 trace_seq_init(s); 1910 1910 1911 - func(s); 1911 + ring_buffer_print_page_header(tr->array_buffer.buffer, s); 1912 + r = simple_read_from_buffer(ubuf, cnt, ppos, 1913 + s->buffer, trace_seq_used(s)); 1914 + 1915 + kfree(s); 1916 + 1917 + return r; 1918 + } 1919 + 1920 + static ssize_t 1921 + show_header_event_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) 1922 + { 1923 + struct trace_seq *s; 1924 + int r; 1925 + 1926 + if (*ppos) 1927 + return 0; 1928 + 1929 + s = kmalloc(sizeof(*s), GFP_KERNEL); 1930 + if (!s) 1931 + return -ENOMEM; 1932 + 1933 + trace_seq_init(s); 1934 + 1935 + ring_buffer_print_entry_header(s); 1912 1936 r = simple_read_from_buffer(ubuf, cnt, ppos, 1913 1937 s->buffer, trace_seq_used(s)); 1914 1938 ··· 2189 2165 .release = subsystem_release, 2190 2166 }; 2191 2167 2192 - static const struct file_operations ftrace_show_header_fops = { 2193 - .open = tracing_open_generic, 2194 - .read = show_header, 2168 + static const struct file_operations ftrace_show_header_page_fops = { 2169 + .open = tracing_open_generic_tr, 2170 + .read = show_header_page_file, 2195 2171 .llseek = default_llseek, 2172 + .release = tracing_release_generic_tr, 2173 + }; 2174 + 2175 + static const struct file_operations ftrace_show_header_event_fops = { 2176 + .open = tracing_open_generic_tr, 2177 + .read = show_header_event_file, 2178 + .llseek = default_llseek, 2179 + .release = tracing_release_generic_tr, 2196 2180 }; 2197 2181 2198 2182 static int ··· 2928 2896 up_write(&trace_event_sem); 2929 2897 } 2930 2898 2899 + static bool event_in_systems(struct trace_event_call *call, 2900 + const char *systems) 2901 + { 2902 + const char *system; 2903 + const char *p; 2904 + 2905 + if (!systems) 2906 + return true; 2907 + 2908 + system = call->class->system; 2909 + p = strstr(systems, system); 2910 + if (!p) 2911 + return false; 2912 + 2913 + if (p != systems && !isspace(*(p - 1)) && *(p - 1) != ',') 2914 + return false; 2915 + 2916 + p += strlen(system); 2917 + return !*p || isspace(*p) || *p == ','; 2918 + } 2919 + 2931 2920 static struct trace_event_file * 2932 2921 trace_create_new_event(struct trace_event_call *call, 2933 2922 struct trace_array *tr) ··· 2958 2905 struct trace_event_file *file; 2959 2906 unsigned int first; 2960 2907 2908 + if (!event_in_systems(call, tr->system_names)) 2909 + return NULL; 2910 + 2961 2911 file = kmem_cache_alloc(file_cachep, GFP_TRACE); 2962 2912 if (!file) 2963 - return NULL; 2913 + return ERR_PTR(-ENOMEM); 2964 2914 2965 2915 pid_list = rcu_dereference_protected(tr->filtered_pids, 2966 2916 lockdep_is_held(&event_mutex)); ··· 3028 2972 struct trace_event_file *file; 3029 2973 3030 2974 file = trace_create_new_event(call, tr); 2975 + /* 2976 + * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed 2977 + * allocation, or NULL if the event is not part of the tr->system_names. 2978 + * When the event is not part of the tr->system_names, return zero, not 2979 + * an error. 2980 + */ 3031 2981 if (!file) 3032 - return -ENOMEM; 2982 + return 0; 2983 + 2984 + if (IS_ERR(file)) 2985 + return PTR_ERR(file); 3033 2986 3034 2987 if (eventdir_initialized) 3035 2988 return event_create_dir(tr->event_dir, file); ··· 3077 3012 int ret; 3078 3013 3079 3014 file = trace_create_new_event(call, tr); 3015 + /* 3016 + * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed 3017 + * allocation, or NULL if the event is not part of the tr->system_names. 3018 + * When the event is not part of the tr->system_names, return zero, not 3019 + * an error. 3020 + */ 3080 3021 if (!file) 3081 - return -ENOMEM; 3022 + return 0; 3023 + 3024 + if (IS_ERR(file)) 3025 + return PTR_ERR(file); 3082 3026 3083 3027 ret = event_define_fields(call); 3084 3028 if (ret) ··· 3826 3752 return 1; 3827 3753 } 3828 3754 3829 - if (strcmp(name, "header_page") == 0) 3830 - *data = ring_buffer_print_page_header; 3755 + if (strcmp(name, "header_page") == 0) { 3756 + *mode = TRACE_MODE_READ; 3757 + *fops = &ftrace_show_header_page_fops; 3831 3758 3832 - else if (strcmp(name, "header_event") == 0) 3833 - *data = ring_buffer_print_entry_header; 3834 - 3835 - else 3759 + } else if (strcmp(name, "header_event") == 0) { 3760 + *mode = TRACE_MODE_READ; 3761 + *fops = &ftrace_show_header_event_fops; 3762 + } else 3836 3763 return 0; 3837 3764 3838 - *mode = TRACE_MODE_READ; 3839 - *fops = &ftrace_show_header_fops; 3840 3765 return 1; 3841 3766 } 3842 3767

+24 -25

kernel/trace/trace_events_hist.c

··· 4805 4805 int len; 4806 4806 4807 4807 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4808 + enum handler_id hid = 0; 4809 + char *action_str; 4810 + 4808 4811 str = hist_data->attrs->action_str[i]; 4809 4812 4810 - if ((len = str_has_prefix(str, "onmatch("))) { 4811 - char *action_str = str + len; 4813 + if ((len = str_has_prefix(str, "onmatch("))) 4814 + hid = HANDLER_ONMATCH; 4815 + else if ((len = str_has_prefix(str, "onmax("))) 4816 + hid = HANDLER_ONMAX; 4817 + else if ((len = str_has_prefix(str, "onchange("))) 4818 + hid = HANDLER_ONCHANGE; 4812 4819 4820 + action_str = str + len; 4821 + 4822 + switch (hid) { 4823 + case HANDLER_ONMATCH: 4813 4824 data = onmatch_parse(tr, action_str); 4814 - if (IS_ERR(data)) { 4815 - ret = PTR_ERR(data); 4816 - break; 4817 - } 4818 - } else if ((len = str_has_prefix(str, "onmax("))) { 4819 - char *action_str = str + len; 4825 + break; 4826 + case HANDLER_ONMAX: 4827 + case HANDLER_ONCHANGE: 4828 + data = track_data_parse(hist_data, action_str, hid); 4829 + break; 4830 + default: 4831 + data = ERR_PTR(-EINVAL); 4832 + break; 4833 + } 4820 4834 4821 - data = track_data_parse(hist_data, action_str, 4822 - HANDLER_ONMAX); 4823 - if (IS_ERR(data)) { 4824 - ret = PTR_ERR(data); 4825 - break; 4826 - } 4827 - } else if ((len = str_has_prefix(str, "onchange("))) { 4828 - char *action_str = str + len; 4829 - 4830 - data = track_data_parse(hist_data, action_str, 4831 - HANDLER_ONCHANGE); 4832 - if (IS_ERR(data)) { 4833 - ret = PTR_ERR(data); 4834 - break; 4835 - } 4836 - } else { 4837 - ret = -EINVAL; 4835 + if (IS_ERR(data)) { 4836 + ret = PTR_ERR(data); 4838 4837 break; 4839 4838 } 4840 4839

-3

kernel/trace/trace_seq.c

··· 13 13 * trace_seq_init() more than once to reset the trace_seq to start 14 14 * from scratch. 15 15 * 16 - * The buffer size is currently PAGE_SIZE, although it may become dynamic 17 - * in the future. 18 - * 19 16 * A write to the buffer will either succeed or fail. That is, unlike 20 17 * sprintf() there will not be a partial write (well it may write into 21 18 * the buffer but it wont update the pointers). This allows users to

+1 -1

samples/ftrace/sample-trace-array.c

··· 105 105 * NOTE: This function increments the reference counter 106 106 * associated with the trace array - "tr". 107 107 */ 108 - tr = trace_array_get_by_name("sample-instance"); 108 + tr = trace_array_get_by_name("sample-instance", "sched,timer,kprobes"); 109 109 110 110 if (!tr) 111 111 return -1;

+95

tools/testing/selftests/ftrace/test.d/00basic/ringbuffer_subbuf_size.tc

··· 1 + #!/bin/sh 2 + # SPDX-License-Identifier: GPL-2.0 3 + # description: Change the ringbuffer sub-buffer size 4 + # requires: buffer_subbuf_size_kb 5 + # flags: instance 6 + 7 + get_buffer_data_size() { 8 + sed -ne 's/^.*data.*size:$[0-9][0-9]*$.*/\1/p' events/header_page 9 + } 10 + 11 + get_buffer_data_offset() { 12 + sed -ne 's/^.*data.*offset:$[0-9][0-9]*$.*/\1/p' events/header_page 13 + } 14 + 15 + get_event_header_size() { 16 + type_len=`sed -ne 's/^.*type_len.*:[^0-9]*$[0-9][0-9]*$.*/\1/p' events/header_event` 17 + time_len=`sed -ne 's/^.*time_delta.*:[^0-9]*$[0-9][0-9]*$.*/\1/p' events/header_event` 18 + array_len=`sed -ne 's/^.*array.*:[^0-9]*$[0-9][0-9]*$.*/\1/p' events/header_event` 19 + total_bits=$((type_len+time_len+array_len)) 20 + total_bits=$((total_bits+7)) 21 + echo $((total_bits/8)) 22 + } 23 + 24 + get_print_event_buf_offset() { 25 + sed -ne 's/^.*buf.*offset:$[0-9][0-9]*$.*/\1/p' events/ftrace/print/format 26 + } 27 + 28 + event_header_size=`get_event_header_size` 29 + print_header_size=`get_print_event_buf_offset` 30 + 31 + data_offset=`get_buffer_data_offset` 32 + 33 + marker_meta=$((event_header_size+print_header_size)) 34 + 35 + make_str() { 36 + cnt=$1 37 + printf -- 'X%.0s' $(seq $cnt) 38 + } 39 + 40 + write_buffer() { 41 + size=$1 42 + 43 + str=`make_str $size` 44 + 45 + # clear the buffer 46 + echo > trace 47 + 48 + # write the string into the marker 49 + echo $str > trace_marker 50 + 51 + echo $str 52 + } 53 + 54 + test_buffer() { 55 + size_kb=$1 56 + page_size=$((size_kb*1024)) 57 + 58 + size=`get_buffer_data_size` 59 + 60 + # the size must be greater than or equal to page_size - data_offset 61 + page_size=$((page_size-data_offset)) 62 + if [ $size -lt $page_size ]; then 63 + exit fail 64 + fi 65 + 66 + # Now add a little more the meta data overhead will overflow 67 + 68 + str=`write_buffer $size` 69 + 70 + # Make sure the line was broken 71 + new_str=`awk ' /tracing_mark_write:/ { sub(/^.*tracing_mark_write: /,"");printf "%s", $0; exit}' trace` 72 + 73 + if [ "$new_str" = "$str" ]; then 74 + exit fail; 75 + fi 76 + 77 + # Make sure the entire line can be found 78 + new_str=`awk ' /tracing_mark_write:/ { sub(/^.*tracing_mark_write: /,"");printf "%s", $0; }' trace` 79 + 80 + if [ "$new_str" != "$str" ]; then 81 + exit fail; 82 + fi 83 + } 84 + 85 + ORIG=`cat buffer_subbuf_size_kb` 86 + 87 + # Could test bigger sizes than 32K, but then creating the string 88 + # to write into the ring buffer takes too long 89 + for a in 4 8 16 32 ; do 90 + echo $a > buffer_subbuf_size_kb 91 + test_buffer $a 92 + done 93 + 94 + echo $ORIG > buffer_subbuf_size_kb 95 +

+82

tools/testing/selftests/ftrace/test.d/00basic/trace_marker.tc

··· 1 + #!/bin/sh 2 + # SPDX-License-Identifier: GPL-2.0 3 + # description: Basic tests on writing to trace_marker 4 + # requires: trace_marker 5 + # flags: instance 6 + 7 + get_buffer_data_size() { 8 + sed -ne 's/^.*data.*size:$[0-9][0-9]*$.*/\1/p' events/header_page 9 + } 10 + 11 + get_buffer_data_offset() { 12 + sed -ne 's/^.*data.*offset:$[0-9][0-9]*$.*/\1/p' events/header_page 13 + } 14 + 15 + get_event_header_size() { 16 + type_len=`sed -ne 's/^.*type_len.*:[^0-9]*$[0-9][0-9]*$.*/\1/p' events/header_event` 17 + time_len=`sed -ne 's/^.*time_delta.*:[^0-9]*$[0-9][0-9]*$.*/\1/p' events/header_event` 18 + array_len=`sed -ne 's/^.*array.*:[^0-9]*$[0-9][0-9]*$.*/\1/p' events/header_event` 19 + total_bits=$((type_len+time_len+array_len)) 20 + total_bits=$((total_bits+7)) 21 + echo $((total_bits/8)) 22 + } 23 + 24 + get_print_event_buf_offset() { 25 + sed -ne 's/^.*buf.*offset:$[0-9][0-9]*$.*/\1/p' events/ftrace/print/format 26 + } 27 + 28 + event_header_size=`get_event_header_size` 29 + print_header_size=`get_print_event_buf_offset` 30 + 31 + data_offset=`get_buffer_data_offset` 32 + 33 + marker_meta=$((event_header_size+print_header_size)) 34 + 35 + make_str() { 36 + cnt=$1 37 + # subtract two for \n\0 as marker adds these 38 + cnt=$((cnt-2)) 39 + printf -- 'X%.0s' $(seq $cnt) 40 + } 41 + 42 + write_buffer() { 43 + size=$1 44 + 45 + str=`make_str $size` 46 + 47 + # clear the buffer 48 + echo > trace 49 + 50 + # write the string into the marker 51 + echo -n $str > trace_marker 52 + 53 + echo $str 54 + } 55 + 56 + test_buffer() { 57 + 58 + size=`get_buffer_data_size` 59 + oneline_size=$((size-marker_meta)) 60 + echo size = $size 61 + echo meta size = $marker_meta 62 + 63 + # Now add a little more the meta data overhead will overflow 64 + 65 + str=`write_buffer $size` 66 + 67 + # Make sure the line was broken 68 + new_str=`awk ' /tracing_mark_write:/ { sub(/^.*tracing_mark_write: /,"");printf "%s", $0; exit}' trace` 69 + 70 + if [ "$new_str" = "$str" ]; then 71 + exit fail; 72 + fi 73 + 74 + # Make sure the entire line can be found 75 + new_str=`awk ' /tracing_mark_write:/ { sub(/^.*tracing_mark_write: /,"");printf "%s", $0; }' trace` 76 + 77 + if [ "$new_str" != "$str" ]; then 78 + exit fail; 79 + fi 80 + } 81 + 82 + test_buffer

Configure Feed

Configure Feed