Merge tag 'trace-sorttable-v6.15' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

+1

arch/arm64/Kconfig

··· 218 218 if DYNAMIC_FTRACE_WITH_ARGS 219 219 select HAVE_SAMPLE_FTRACE_DIRECT 220 220 select HAVE_SAMPLE_FTRACE_DIRECT_MULTI 221 + select HAVE_BUILDTIME_MCOUNT_SORT 221 222 select HAVE_EFFICIENT_UNALIGNED_ACCESS 222 223 select HAVE_GUP_FAST 223 224 select HAVE_FTRACE_GRAPH_FUNC

+50 -5

kernel/trace/ftrace.c

··· 7016 7016 unsigned long *p; 7017 7017 unsigned long addr; 7018 7018 unsigned long flags = 0; /* Shut up gcc */ 7019 + unsigned long pages; 7019 7020 int ret = -ENOMEM; 7020 7021 7021 7022 count = end - start; 7022 7023 7023 7024 if (!count) 7024 7025 return 0; 7026 + 7027 + pages = DIV_ROUND_UP(count, ENTRIES_PER_PAGE); 7025 7028 7026 7029 /* 7027 7030 * Sorting mcount in vmlinux at build time depend on ··· 7070 7067 pg = start_pg; 7071 7068 while (p < end) { 7072 7069 unsigned long end_offset; 7073 - addr = ftrace_call_adjust(*p++); 7070 + 7071 + addr = *p++; 7072 + 7074 7073 /* 7075 7074 * Some architecture linkers will pad between 7076 7075 * the different mcount_loc sections of different ··· 7083 7078 skipped++; 7084 7079 continue; 7085 7080 } 7081 + 7082 + /* 7083 + * If this is core kernel, make sure the address is in core 7084 + * or inittext, as weak functions get zeroed and KASLR can 7085 + * move them to something other than zero. It just will not 7086 + * move it to an area where kernel text is. 7087 + */ 7088 + if (!mod && !(is_kernel_text(addr) || is_kernel_inittext(addr))) { 7089 + skipped++; 7090 + continue; 7091 + } 7092 + 7093 + addr = ftrace_call_adjust(addr); 7086 7094 7087 7095 end_offset = (pg->index+1) * sizeof(pg->records[0]); 7088 7096 if (end_offset > PAGE_SIZE << pg->order) { ··· 7136 7118 7137 7119 /* We should have used all pages unless we skipped some */ 7138 7120 if (pg_unuse) { 7139 - WARN_ON(!skipped); 7121 + unsigned long pg_remaining, remaining = 0; 7122 + unsigned long skip; 7123 + 7124 + /* Count the number of entries unused and compare it to skipped. */ 7125 + pg_remaining = (ENTRIES_PER_PAGE << pg->order) - pg->index; 7126 + 7127 + if (!WARN(skipped < pg_remaining, "Extra allocated pages for ftrace")) { 7128 + 7129 + skip = skipped - pg_remaining; 7130 + 7131 + for (pg = pg_unuse; pg; pg = pg->next) 7132 + remaining += 1 << pg->order; 7133 + 7134 + pages -= remaining; 7135 + 7136 + skip = DIV_ROUND_UP(skip, ENTRIES_PER_PAGE); 7137 + 7138 + /* 7139 + * Check to see if the number of pages remaining would 7140 + * just fit the number of entries skipped. 7141 + */ 7142 + WARN(skip != remaining, "Extra allocated pages for ftrace: %lu with %lu skipped", 7143 + remaining, skipped); 7144 + } 7140 7145 /* Need to synchronize with ftrace_location_range() */ 7141 7146 synchronize_rcu(); 7142 7147 ftrace_free_pages(pg_unuse); 7143 7148 } 7149 + 7150 + if (!mod) { 7151 + count -= skipped; 7152 + pr_info("ftrace: allocating %ld entries in %ld pages\n", 7153 + count, pages); 7154 + } 7155 + 7144 7156 return ret; 7145 7157 } 7146 7158 ··· 7815 7767 pr_info("ftrace: No functions to be traced?\n"); 7816 7768 goto failed; 7817 7769 } 7818 - 7819 - pr_info("ftrace: allocating %ld entries in %ld pages\n", 7820 - count, DIV_ROUND_UP(count, ENTRIES_PER_PAGE)); 7821 7770 7822 7771 ret = ftrace_process_locs(NULL, 7823 7772 __start_mcount_loc,

+3 -1

scripts/link-vmlinux.sh

··· 173 173 174 174 sorttable() 175 175 { 176 - ${objtree}/scripts/sorttable ${1} 176 + ${NM} -S ${1} > .tmp_vmlinux.nm-sort 177 + ${objtree}/scripts/sorttable -s .tmp_vmlinux.nm-sort ${1} 177 178 } 178 179 179 180 cleanup() 180 181 { 181 182 rm -f .btf.* 183 + rm -f .tmp_vmlinux.nm-sort 182 184 rm -f System.map 183 185 rm -f vmlinux 184 186 rm -f vmlinux.map

+403 -8

scripts/sorttable.c

··· 28 28 #include <fcntl.h> 29 29 #include <stdio.h> 30 30 #include <stdlib.h> 31 + #include <stdbool.h> 31 32 #include <string.h> 32 33 #include <unistd.h> 33 34 #include <errno.h> ··· 80 79 Elf64_Sym e64; 81 80 } Elf_Sym; 82 81 82 + typedef union { 83 + Elf32_Rela e32; 84 + Elf64_Rela e64; 85 + } Elf_Rela; 86 + 83 87 static uint32_t (*r)(const uint32_t *); 84 88 static uint16_t (*r2)(const uint16_t *); 85 89 static uint64_t (*r8)(const uint64_t *); 86 90 static void (*w)(uint32_t, uint32_t *); 91 + static void (*w8)(uint64_t, uint64_t *); 87 92 typedef void (*table_sort_t)(char *, int); 88 93 89 94 static struct elf_funcs { ··· 109 102 uint32_t (*sym_name)(Elf_Sym *sym); 110 103 uint64_t (*sym_value)(Elf_Sym *sym); 111 104 uint16_t (*sym_shndx)(Elf_Sym *sym); 105 + uint64_t (*rela_offset)(Elf_Rela *rela); 106 + uint64_t (*rela_info)(Elf_Rela *rela); 107 + uint64_t (*rela_addend)(Elf_Rela *rela); 108 + void (*rela_write_addend)(Elf_Rela *rela, uint64_t val); 112 109 } e; 113 110 114 111 static uint64_t ehdr64_shoff(Elf_Ehdr *ehdr) ··· 273 262 SYM_WORD(name) 274 263 SYM_HALF(shndx) 275 264 265 + #define __maybe_unused __attribute__((__unused__)) 266 + 267 + #define RELA_ADDR(fn_name) \ 268 + static uint64_t rela64_##fn_name(Elf_Rela *rela) \ 269 + { \ 270 + return r8((uint64_t *)&rela->e64.r_##fn_name); \ 271 + } \ 272 + \ 273 + static uint64_t rela32_##fn_name(Elf_Rela *rela) \ 274 + { \ 275 + return r((uint32_t *)&rela->e32.r_##fn_name); \ 276 + } \ 277 + \ 278 + static uint64_t __maybe_unused rela_##fn_name(Elf_Rela *rela) \ 279 + { \ 280 + return e.rela_##fn_name(rela); \ 281 + } 282 + 283 + RELA_ADDR(offset) 284 + RELA_ADDR(info) 285 + RELA_ADDR(addend) 286 + 287 + static void rela64_write_addend(Elf_Rela *rela, uint64_t val) 288 + { 289 + w8(val, (uint64_t *)&rela->e64.r_addend); 290 + } 291 + 292 + static void rela32_write_addend(Elf_Rela *rela, uint64_t val) 293 + { 294 + w(val, (uint32_t *)&rela->e32.r_addend); 295 + } 296 + 276 297 /* 277 298 * Get the whole file as a programming convenience in order to avoid 278 299 * malloc+lseek+read+free of many pieces. If successful, then mmap ··· 384 341 put_unaligned_le32(val, x); 385 342 } 386 343 344 + static void w8be(uint64_t val, uint64_t *x) 345 + { 346 + put_unaligned_be64(val, x); 347 + } 348 + 349 + static void w8le(uint64_t val, uint64_t *x) 350 + { 351 + put_unaligned_le64(val, x); 352 + } 353 + 387 354 /* 388 355 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of 389 356 * the way to -256..-1, to avoid conflicting with real section ··· 451 398 static int extable_ent_size; 452 399 static int long_size; 453 400 401 + #define ERRSTR_MAXSZ 256 454 402 455 403 #ifdef UNWINDER_ORC_ENABLED 456 404 /* ORC unwinder only support X86_64 */ 457 405 #include <asm/orc_types.h> 458 - 459 - #define ERRSTR_MAXSZ 256 460 406 461 407 static char g_err[ERRSTR_MAXSZ]; 462 408 static int *g_orc_ip_table; ··· 551 499 #endif 552 500 553 501 #ifdef MCOUNT_SORT_ENABLED 502 + 503 + static int compare_values_64(const void *a, const void *b) 504 + { 505 + uint64_t av = *(uint64_t *)a; 506 + uint64_t bv = *(uint64_t *)b; 507 + 508 + if (av < bv) 509 + return -1; 510 + return av > bv; 511 + } 512 + 513 + static int compare_values_32(const void *a, const void *b) 514 + { 515 + uint32_t av = *(uint32_t *)a; 516 + uint32_t bv = *(uint32_t *)b; 517 + 518 + if (av < bv) 519 + return -1; 520 + return av > bv; 521 + } 522 + 523 + static int (*compare_values)(const void *a, const void *b); 524 + 525 + /* Only used for sorting mcount table */ 526 + static void rela_write_addend(Elf_Rela *rela, uint64_t val) 527 + { 528 + e.rela_write_addend(rela, val); 529 + } 530 + 531 + struct func_info { 532 + uint64_t addr; 533 + uint64_t size; 534 + }; 535 + 536 + /* List of functions created by: nm -S vmlinux */ 537 + static struct func_info *function_list; 538 + static int function_list_size; 539 + 540 + /* Allocate functions in 1k blocks */ 541 + #define FUNC_BLK_SIZE 1024 542 + #define FUNC_BLK_MASK (FUNC_BLK_SIZE - 1) 543 + 544 + static int add_field(uint64_t addr, uint64_t size) 545 + { 546 + struct func_info *fi; 547 + int fsize = function_list_size; 548 + 549 + if (!(fsize & FUNC_BLK_MASK)) { 550 + fsize += FUNC_BLK_SIZE; 551 + fi = realloc(function_list, fsize * sizeof(struct func_info)); 552 + if (!fi) 553 + return -1; 554 + function_list = fi; 555 + } 556 + fi = &function_list[function_list_size++]; 557 + fi->addr = addr; 558 + fi->size = size; 559 + return 0; 560 + } 561 + 562 + /* Used for when mcount/fentry is before the function entry */ 563 + static int before_func; 564 + 565 + /* Only return match if the address lies inside the function size */ 566 + static int cmp_func_addr(const void *K, const void *A) 567 + { 568 + uint64_t key = *(const uint64_t *)K; 569 + const struct func_info *a = A; 570 + 571 + if (key + before_func < a->addr) 572 + return -1; 573 + return key >= a->addr + a->size; 574 + } 575 + 576 + /* Find the function in function list that is bounded by the function size */ 577 + static int find_func(uint64_t key) 578 + { 579 + return bsearch(&key, function_list, function_list_size, 580 + sizeof(struct func_info), cmp_func_addr) != NULL; 581 + } 582 + 583 + static int cmp_funcs(const void *A, const void *B) 584 + { 585 + const struct func_info *a = A; 586 + const struct func_info *b = B; 587 + 588 + if (a->addr < b->addr) 589 + return -1; 590 + return a->addr > b->addr; 591 + } 592 + 593 + static int parse_symbols(const char *fname) 594 + { 595 + FILE *fp; 596 + char addr_str[20]; /* Only need 17, but round up to next int size */ 597 + char size_str[20]; 598 + char type; 599 + 600 + fp = fopen(fname, "r"); 601 + if (!fp) { 602 + perror(fname); 603 + return -1; 604 + } 605 + 606 + while (fscanf(fp, "%16s %16s %c %*s\n", addr_str, size_str, &type) == 3) { 607 + uint64_t addr; 608 + uint64_t size; 609 + 610 + /* Only care about functions */ 611 + if (type != 't' && type != 'T' && type != 'W') 612 + continue; 613 + 614 + addr = strtoull(addr_str, NULL, 16); 615 + size = strtoull(size_str, NULL, 16); 616 + if (add_field(addr, size) < 0) 617 + return -1; 618 + } 619 + fclose(fp); 620 + 621 + qsort(function_list, function_list_size, sizeof(struct func_info), cmp_funcs); 622 + 623 + return 0; 624 + } 625 + 554 626 static pthread_t mcount_sort_thread; 627 + static bool sort_reloc; 628 + 629 + static long rela_type; 630 + 631 + static char m_err[ERRSTR_MAXSZ]; 555 632 556 633 struct elf_mcount_loc { 557 634 Elf_Ehdr *ehdr; ··· 689 508 uint64_t stop_mcount_loc; 690 509 }; 691 510 511 + /* Fill the array with the content of the relocs */ 512 + static int fill_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 513 + { 514 + Elf_Shdr *shdr_start; 515 + Elf_Rela *rel; 516 + unsigned int shnum; 517 + unsigned int count = 0; 518 + int shentsize; 519 + void *array_end = ptr + size; 520 + 521 + shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 522 + shentsize = ehdr_shentsize(ehdr); 523 + 524 + shnum = ehdr_shnum(ehdr); 525 + if (shnum == SHN_UNDEF) 526 + shnum = shdr_size(shdr_start); 527 + 528 + for (int i = 0; i < shnum; i++) { 529 + Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 530 + void *end; 531 + 532 + if (shdr_type(shdr) != SHT_RELA) 533 + continue; 534 + 535 + rel = (void *)ehdr + shdr_offset(shdr); 536 + end = (void *)rel + shdr_size(shdr); 537 + 538 + for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 539 + uint64_t offset = rela_offset(rel); 540 + 541 + if (offset >= start_loc && offset < start_loc + size) { 542 + if (ptr + long_size > array_end) { 543 + snprintf(m_err, ERRSTR_MAXSZ, 544 + "Too many relocations"); 545 + return -1; 546 + } 547 + 548 + /* Make sure this has the correct type */ 549 + if (rela_info(rel) != rela_type) { 550 + snprintf(m_err, ERRSTR_MAXSZ, 551 + "rela has type %lx but expected %lx\n", 552 + (long)rela_info(rel), rela_type); 553 + return -1; 554 + } 555 + 556 + if (long_size == 4) 557 + *(uint32_t *)ptr = rela_addend(rel); 558 + else 559 + *(uint64_t *)ptr = rela_addend(rel); 560 + ptr += long_size; 561 + count++; 562 + } 563 + } 564 + } 565 + return count; 566 + } 567 + 568 + /* Put the sorted vals back into the relocation elements */ 569 + static void replace_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 570 + { 571 + Elf_Shdr *shdr_start; 572 + Elf_Rela *rel; 573 + unsigned int shnum; 574 + int shentsize; 575 + 576 + shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 577 + shentsize = ehdr_shentsize(ehdr); 578 + 579 + shnum = ehdr_shnum(ehdr); 580 + if (shnum == SHN_UNDEF) 581 + shnum = shdr_size(shdr_start); 582 + 583 + for (int i = 0; i < shnum; i++) { 584 + Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 585 + void *end; 586 + 587 + if (shdr_type(shdr) != SHT_RELA) 588 + continue; 589 + 590 + rel = (void *)ehdr + shdr_offset(shdr); 591 + end = (void *)rel + shdr_size(shdr); 592 + 593 + for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 594 + uint64_t offset = rela_offset(rel); 595 + 596 + if (offset >= start_loc && offset < start_loc + size) { 597 + if (long_size == 4) 598 + rela_write_addend(rel, *(uint32_t *)ptr); 599 + else 600 + rela_write_addend(rel, *(uint64_t *)ptr); 601 + ptr += long_size; 602 + } 603 + } 604 + } 605 + } 606 + 607 + static int fill_addrs(void *ptr, uint64_t size, void *addrs) 608 + { 609 + void *end = ptr + size; 610 + int count = 0; 611 + 612 + for (; ptr < end; ptr += long_size, addrs += long_size, count++) { 613 + if (long_size == 4) 614 + *(uint32_t *)ptr = r(addrs); 615 + else 616 + *(uint64_t *)ptr = r8(addrs); 617 + } 618 + return count; 619 + } 620 + 621 + static void replace_addrs(void *ptr, uint64_t size, void *addrs) 622 + { 623 + void *end = ptr + size; 624 + 625 + for (; ptr < end; ptr += long_size, addrs += long_size) { 626 + if (long_size == 4) 627 + w(*(uint32_t *)ptr, addrs); 628 + else 629 + w8(*(uint64_t *)ptr, addrs); 630 + } 631 + } 632 + 692 633 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 693 634 static void *sort_mcount_loc(void *arg) 694 635 { 695 636 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 696 637 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec) 697 638 + shdr_offset(emloc->init_data_sec); 698 - uint64_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc; 639 + uint64_t size = emloc->stop_mcount_loc - emloc->start_mcount_loc; 699 640 unsigned char *start_loc = (void *)emloc->ehdr + offset; 641 + Elf_Ehdr *ehdr = emloc->ehdr; 642 + void *e_msg = NULL; 643 + void *vals; 644 + int count; 700 645 701 - qsort(start_loc, count/long_size, long_size, compare_extable); 702 - return NULL; 646 + vals = malloc(long_size * size); 647 + if (!vals) { 648 + snprintf(m_err, ERRSTR_MAXSZ, "Failed to allocate sort array"); 649 + pthread_exit(m_err); 650 + } 651 + 652 + if (sort_reloc) { 653 + count = fill_relocs(vals, size, ehdr, emloc->start_mcount_loc); 654 + /* gcc may use relocs to save the addresses, but clang does not. */ 655 + if (!count) { 656 + count = fill_addrs(vals, size, start_loc); 657 + sort_reloc = 0; 658 + } 659 + } else 660 + count = fill_addrs(vals, size, start_loc); 661 + 662 + if (count < 0) { 663 + e_msg = m_err; 664 + goto out; 665 + } 666 + 667 + if (count != size / long_size) { 668 + snprintf(m_err, ERRSTR_MAXSZ, "Expected %u mcount elements but found %u\n", 669 + (int)(size / long_size), count); 670 + e_msg = m_err; 671 + goto out; 672 + } 673 + 674 + /* zero out any locations not found by function list */ 675 + if (function_list_size) { 676 + for (void *ptr = vals; ptr < vals + size; ptr += long_size) { 677 + uint64_t key; 678 + 679 + key = long_size == 4 ? r((uint32_t *)ptr) : r8((uint64_t *)ptr); 680 + if (!find_func(key)) { 681 + if (long_size == 4) 682 + *(uint32_t *)ptr = 0; 683 + else 684 + *(uint64_t *)ptr = 0; 685 + } 686 + } 687 + } 688 + 689 + compare_values = long_size == 4 ? compare_values_32 : compare_values_64; 690 + 691 + qsort(vals, count, long_size, compare_values); 692 + 693 + if (sort_reloc) 694 + replace_relocs(vals, size, ehdr, emloc->start_mcount_loc); 695 + else 696 + replace_addrs(vals, size, start_loc); 697 + 698 + out: 699 + free(vals); 700 + 701 + pthread_exit(e_msg); 703 702 } 704 703 705 704 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ ··· 916 555 return; 917 556 } 918 557 } 558 + #else /* MCOUNT_SORT_ENABLED */ 559 + static inline int parse_symbols(const char *fname) { return 0; } 919 560 #endif 920 561 921 562 static int do_sort(Elf_Ehdr *ehdr, ··· 1229 866 r2 = r2le; 1230 867 r8 = r8le; 1231 868 w = wle; 869 + w8 = w8le; 1232 870 break; 1233 871 case ELFDATA2MSB: 1234 872 r = rbe; 1235 873 r2 = r2be; 1236 874 r8 = r8be; 1237 875 w = wbe; 876 + w8 = w8be; 1238 877 break; 1239 878 default: 1240 879 fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", ··· 1252 887 } 1253 888 1254 889 switch (r2(&ehdr->e32.e_machine)) { 1255 - case EM_386: 1256 890 case EM_AARCH64: 891 + #ifdef MCOUNT_SORT_ENABLED 892 + sort_reloc = true; 893 + rela_type = 0x403; 894 + /* arm64 uses patchable function entry placing before function */ 895 + before_func = 8; 896 + #endif 897 + /* fallthrough */ 898 + case EM_386: 1257 899 case EM_LOONGARCH: 1258 900 case EM_RISCV: 1259 901 case EM_S390: ··· 1304 932 .sym_name = sym32_name, 1305 933 .sym_value = sym32_value, 1306 934 .sym_shndx = sym32_shndx, 935 + .rela_offset = rela32_offset, 936 + .rela_info = rela32_info, 937 + .rela_addend = rela32_addend, 938 + .rela_write_addend = rela32_write_addend, 1307 939 }; 1308 940 1309 941 e = efuncs; ··· 1341 965 .sym_name = sym64_name, 1342 966 .sym_value = sym64_value, 1343 967 .sym_shndx = sym64_shndx, 968 + .rela_offset = rela64_offset, 969 + .rela_info = rela64_info, 970 + .rela_addend = rela64_addend, 971 + .rela_write_addend = rela64_write_addend, 1344 972 }; 1345 973 1346 974 e = efuncs; ··· 1375 995 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */ 1376 996 size_t size = 0; 1377 997 void *addr = NULL; 998 + int c; 1378 999 1379 - if (argc < 2) { 1000 + while ((c = getopt(argc, argv, "s:")) >= 0) { 1001 + switch (c) { 1002 + case 's': 1003 + if (parse_symbols(optarg) < 0) { 1004 + fprintf(stderr, "Could not parse %s\n", optarg); 1005 + return -1; 1006 + } 1007 + break; 1008 + default: 1009 + fprintf(stderr, "usage: sorttable [-s nm-file] vmlinux...\n"); 1010 + return 0; 1011 + } 1012 + } 1013 + 1014 + if ((argc - optind) < 1) { 1380 1015 fprintf(stderr, "usage: sorttable vmlinux...\n"); 1381 1016 return 0; 1382 1017 } 1383 1018 1384 1019 /* Process each file in turn, allowing deep failure. */ 1385 - for (i = 1; i < argc; i++) { 1020 + for (i = optind; i < argc; i++) { 1386 1021 addr = mmap_file(argv[i], &size); 1387 1022 if (!addr) { 1388 1023 ++n_error;

Configure Feed

Configure Feed