tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge branch 'core-objtool-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull objtool updates from Ingo Molnar:
"The biggest changes in this cycle were the vmlinux.o optimizations by
Peter Zijlstra, which are preparatory and optimization work to run
objtool against the much richer vmlinux.o object file, to perform
new, whole-program section based logic. That work exposed a handful
of problems with the existing code, which fixes and optimizations are
merged here. The complete 'vmlinux.o and noinstr' work is still work
in progress, targeted for v5.8.

There's also assorted fixes and enhancements from Josh Poimboeuf.

In particular I'd like to draw attention to commit 644592d328370,
which turns fatal objtool errors into failed kernel builds. This
behavior is IMO now justified on multiple grounds (it's easy currently
to not notice an essentially corrupted kernel build), and the commit
has been in -next testing for several weeks, but there could still be
build failures with old or weird toolchains. Should that be widespread
or high profile enough then I'd suggest a quick revert, to not hold up
the merge window"

* 'core-objtool-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (22 commits)
objtool: Re-arrange validate_functions()
objtool: Optimize find_rela_by_dest_range()
objtool: Delete cleanup()
objtool: Optimize read_sections()
objtool: Optimize find_symbol_by_name()
objtool: Resize insn_hash
objtool: Rename find_containing_func()
objtool: Optimize find_symbol_*() and read_symbols()
objtool: Optimize find_section_by_name()
objtool: Optimize find_section_by_index()
objtool: Add a statistics mode
objtool: Optimize find_symbol_by_index()
x86/kexec: Make relocate_kernel_64.S objtool clean
x86/kexec: Use RIP relative addressing
objtool: Rename func_for_each_insn_all()
objtool: Rename func_for_each_insn()
objtool: Introduce validate_return()
objtool: Improve call destination function detection
objtool: Fix clang switch table edge case
objtool: Add relocation check for alternative sections
...

Linus Torvalds d937a6df 2ce94bc4

+433 -215
-1
arch/x86/kernel/Makefile
··· 28 28 KASAN_SANITIZE_stacktrace.o := n 29 29 KASAN_SANITIZE_paravirt.o := n 30 30 31 - OBJECT_FILES_NON_STANDARD_relocate_kernel_$(BITS).o := y 32 31 OBJECT_FILES_NON_STANDARD_test_nx.o := y 33 32 OBJECT_FILES_NON_STANDARD_paravirt_patch.o := y 34 33
+8 -4
arch/x86/kernel/relocate_kernel_64.S
··· 9 9 #include <asm/kexec.h> 10 10 #include <asm/processor-flags.h> 11 11 #include <asm/pgtable_types.h> 12 + #include <asm/nospec-branch.h> 13 + #include <asm/unwind_hints.h> 12 14 13 15 /* 14 16 * Must be relocatable PIC code callable as a C function ··· 41 39 .align PAGE_SIZE 42 40 .code64 43 41 SYM_CODE_START_NOALIGN(relocate_kernel) 42 + UNWIND_HINT_EMPTY 44 43 /* 45 44 * %rdi indirection_page 46 45 * %rsi page_list ··· 108 105 SYM_CODE_END(relocate_kernel) 109 106 110 107 SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) 108 + UNWIND_HINT_EMPTY 111 109 /* set return address to 0 if not preserving context */ 112 110 pushq $0 113 111 /* store the start address on the stack */ ··· 196 192 1: 197 193 popq %rdx 198 194 leaq PAGE_SIZE(%r10), %rsp 195 + ANNOTATE_RETPOLINE_SAFE 199 196 call *%rdx 200 197 201 198 /* get the re-entry point of the peer system */ 202 199 movq 0(%rsp), %rbp 203 - call 1f 204 - 1: 205 - popq %r8 206 - subq $(1b - relocate_kernel), %r8 200 + leaq relocate_kernel(%rip), %r8 207 201 movq CP_PA_SWAP_PAGE(%r8), %r10 208 202 movq CP_PA_BACKUP_PAGES_MAP(%r8), %rdi 209 203 movq CP_PA_TABLE_PAGE(%r8), %rax ··· 214 212 SYM_CODE_END(identity_mapped) 215 213 216 214 SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) 215 + UNWIND_HINT_EMPTY 217 216 movq RSP(%r8), %rsp 218 217 movq CR4(%r8), %rax 219 218 movq %rax, %cr4 ··· 236 233 237 234 /* Do the copies */ 238 235 SYM_CODE_START_LOCAL_NOALIGN(swap_pages) 236 + UNWIND_HINT_EMPTY 239 237 movq %rdi, %rcx /* Put the page_list in %rcx */ 240 238 xorl %edi, %edi 241 239 xorl %esi, %esi
+5
tools/objtool/Build
··· 11 11 objtool-y += libstring.o 12 12 objtool-y += libctype.o 13 13 objtool-y += str_error_r.o 14 + objtool-y += librbtree.o 14 15 15 16 CFLAGS += -I$(srctree)/tools/lib 16 17 ··· 24 23 $(call if_changed_dep,cc_o_c) 25 24 26 25 $(OUTPUT)str_error_r.o: ../lib/str_error_r.c FORCE 26 + $(call rule_mkdir) 27 + $(call if_changed_dep,cc_o_c) 28 + 29 + $(OUTPUT)librbtree.o: ../lib/rbtree.c FORCE 27 30 $(call rule_mkdir) 28 31 $(call if_changed_dep,cc_o_c)
+2 -1
tools/objtool/builtin-check.c
··· 17 17 #include "builtin.h" 18 18 #include "check.h" 19 19 20 - bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess; 20 + bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats; 21 21 22 22 static const char * const check_usage[] = { 23 23 "objtool check [<options>] file.o", ··· 31 31 OPT_BOOLEAN('m', "module", &module, "Indicates the object will be part of a kernel module"), 32 32 OPT_BOOLEAN('b', "backtrace", &backtrace, "unwind on error"), 33 33 OPT_BOOLEAN('a', "uaccess", &uaccess, "enable uaccess checking"), 34 + OPT_BOOLEAN('s', "stats", &stats, "print statistics"), 34 35 OPT_END(), 35 36 }; 36 37
+1 -1
tools/objtool/builtin.h
··· 8 8 #include <subcmd/parse-options.h> 9 9 10 10 extern const struct option check_options[]; 11 - extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess; 11 + extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats; 12 12 13 13 extern int cmd_check(int argc, const char **argv); 14 14 extern int cmd_orc(int argc, const char **argv);
+161 -113
tools/objtool/check.c
··· 72 72 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 73 73 } 74 74 75 - #define func_for_each_insn_all(file, func, insn) \ 75 + #define func_for_each_insn(file, func, insn) \ 76 76 for (insn = find_insn(file, func->sec, func->offset); \ 77 77 insn; \ 78 78 insn = next_insn_same_func(file, insn)) 79 79 80 - #define func_for_each_insn(file, func, insn) \ 81 - for (insn = find_insn(file, func->sec, func->offset); \ 80 + #define sym_for_each_insn(file, sym, insn) \ 81 + for (insn = find_insn(file, sym->sec, sym->offset); \ 82 82 insn && &insn->list != &file->insn_list && \ 83 - insn->sec == func->sec && \ 84 - insn->offset < func->offset + func->len; \ 83 + insn->sec == sym->sec && \ 84 + insn->offset < sym->offset + sym->len; \ 85 85 insn = list_next_entry(insn, list)) 86 86 87 - #define func_for_each_insn_continue_reverse(file, func, insn) \ 87 + #define sym_for_each_insn_continue_reverse(file, sym, insn) \ 88 88 for (insn = list_prev_entry(insn, list); \ 89 89 &insn->list != &file->insn_list && \ 90 - insn->sec == func->sec && insn->offset >= func->offset; \ 90 + insn->sec == sym->sec && insn->offset >= sym->offset; \ 91 91 insn = list_prev_entry(insn, list)) 92 92 93 93 #define sec_for_each_insn_from(file, insn) \ ··· 97 97 for (insn = next_insn_same_sec(file, insn); insn; \ 98 98 insn = next_insn_same_sec(file, insn)) 99 99 100 + static bool is_static_jump(struct instruction *insn) 101 + { 102 + return insn->type == INSN_JUMP_CONDITIONAL || 103 + insn->type == INSN_JUMP_UNCONDITIONAL; 104 + } 105 + 100 106 static bool is_sibling_call(struct instruction *insn) 101 107 { 102 108 /* An indirect jump is either a sibling call or a jump to a table. */ 103 109 if (insn->type == INSN_JUMP_DYNAMIC) 104 110 return list_empty(&insn->alts); 105 111 106 - if (insn->type != INSN_JUMP_CONDITIONAL && 107 - insn->type != INSN_JUMP_UNCONDITIONAL) 112 + if (!is_static_jump(insn)) 108 113 return false; 109 114 110 115 /* add_jump_destinations() sets insn->call_dest for sibling calls. */ ··· 170 165 if (!insn->func) 171 166 return false; 172 167 173 - func_for_each_insn_all(file, func, insn) { 168 + func_for_each_insn(file, func, insn) { 174 169 empty = false; 175 170 176 171 if (insn->type == INSN_RETURN) ··· 185 180 * case, the function's dead-end status depends on whether the target 186 181 * of the sibling call returns. 187 182 */ 188 - func_for_each_insn_all(file, func, insn) { 183 + func_for_each_insn(file, func, insn) { 189 184 if (is_sibling_call(insn)) { 190 185 struct instruction *dest = insn->jump_dest; 191 186 ··· 239 234 struct symbol *func; 240 235 unsigned long offset; 241 236 struct instruction *insn; 237 + unsigned long nr_insns = 0; 242 238 int ret; 243 239 244 240 for_each_sec(file, sec) { ··· 275 269 276 270 hash_add(file->insn_hash, &insn->hash, insn->offset); 277 271 list_add_tail(&insn->list, &file->insn_list); 272 + nr_insns++; 278 273 } 279 274 280 275 list_for_each_entry(func, &sec->symbol_list, list) { ··· 288 281 return -1; 289 282 } 290 283 291 - func_for_each_insn(file, func, insn) 284 + sym_for_each_insn(file, func, insn) 292 285 insn->func = func; 293 286 } 294 287 } 288 + 289 + if (stats) 290 + printf("nr_insns: %lu\n", nr_insns); 295 291 296 292 return 0; 297 293 ··· 425 415 break; 426 416 427 417 case STT_SECTION: 428 - func = find_symbol_by_offset(rela->sym->sec, rela->addend); 429 - if (!func || func->type != STT_FUNC) 418 + func = find_func_by_offset(rela->sym->sec, rela->addend); 419 + if (!func) 430 420 continue; 431 421 break; 432 422 ··· 435 425 continue; 436 426 } 437 427 438 - func_for_each_insn_all(file, func, insn) 428 + func_for_each_insn(file, func, insn) 439 429 insn->ignore = true; 440 430 } 441 431 } ··· 563 553 unsigned long dest_off; 564 554 565 555 for_each_insn(file, insn) { 566 - if (insn->type != INSN_JUMP_CONDITIONAL && 567 - insn->type != INSN_JUMP_UNCONDITIONAL) 556 + if (!is_static_jump(insn)) 568 557 continue; 569 558 570 559 if (insn->ignore || insn->offset == FAKE_JUMP_OFFSET) 571 560 continue; 572 561 573 - rela = find_rela_by_dest_range(insn->sec, insn->offset, 574 - insn->len); 562 + rela = find_rela_by_dest_range(file->elf, insn->sec, 563 + insn->offset, insn->len); 575 564 if (!rela) { 576 565 dest_sec = insn->sec; 577 566 dest_off = insn->offset + insn->len + insn->immediate; ··· 666 657 if (insn->type != INSN_CALL) 667 658 continue; 668 659 669 - rela = find_rela_by_dest_range(insn->sec, insn->offset, 670 - insn->len); 660 + rela = find_rela_by_dest_range(file->elf, insn->sec, 661 + insn->offset, insn->len); 671 662 if (!rela) { 672 663 dest_off = insn->offset + insn->len + insn->immediate; 673 - insn->call_dest = find_symbol_by_offset(insn->sec, 674 - dest_off); 664 + insn->call_dest = find_func_by_offset(insn->sec, dest_off); 665 + if (!insn->call_dest) 666 + insn->call_dest = find_symbol_by_offset(insn->sec, dest_off); 675 667 676 - if (!insn->call_dest && !insn->ignore) { 668 + if (insn->ignore) 669 + continue; 670 + 671 + if (!insn->call_dest) { 677 672 WARN_FUNC("unsupported intra-function call", 678 673 insn->sec, insn->offset); 679 674 if (retpoline) ··· 685 672 return -1; 686 673 } 687 674 675 + if (insn->func && insn->call_dest->type != STT_FUNC) { 676 + WARN_FUNC("unsupported call to non-function", 677 + insn->sec, insn->offset); 678 + return -1; 679 + } 680 + 688 681 } else if (rela->sym->type == STT_SECTION) { 689 - insn->call_dest = find_symbol_by_offset(rela->sym->sec, 690 - rela->addend+4); 691 - if (!insn->call_dest || 692 - insn->call_dest->type != STT_FUNC) { 682 + insn->call_dest = find_func_by_offset(rela->sym->sec, 683 + rela->addend+4); 684 + if (!insn->call_dest) { 693 685 WARN_FUNC("can't find call dest symbol at %s+0x%x", 694 686 insn->sec, insn->offset, 695 687 rela->sym->sec->name, ··· 782 764 insn->ignore = orig_insn->ignore_alts; 783 765 insn->func = orig_insn->func; 784 766 785 - if (insn->type != INSN_JUMP_CONDITIONAL && 786 - insn->type != INSN_JUMP_UNCONDITIONAL) 767 + /* 768 + * Since alternative replacement code is copy/pasted by the 769 + * kernel after applying relocations, generally such code can't 770 + * have relative-address relocation references to outside the 771 + * .altinstr_replacement section, unless the arch's 772 + * alternatives code can adjust the relative offsets 773 + * accordingly. 774 + * 775 + * The x86 alternatives code adjusts the offsets only when it 776 + * encounters a branch instruction at the very beginning of the 777 + * replacement group. 778 + */ 779 + if ((insn->offset != special_alt->new_off || 780 + (insn->type != INSN_CALL && !is_static_jump(insn))) && 781 + find_rela_by_dest_range(file->elf, insn->sec, insn->offset, insn->len)) { 782 + 783 + WARN_FUNC("unsupported relocation in alternatives section", 784 + insn->sec, insn->offset); 785 + return -1; 786 + } 787 + 788 + if (!is_static_jump(insn)) 787 789 continue; 788 790 789 791 if (!insn->immediate) ··· 1039 1001 struct instruction *insn) 1040 1002 { 1041 1003 struct rela *text_rela, *table_rela; 1042 - struct instruction *orig_insn = insn; 1004 + struct instruction *dest_insn, *orig_insn = insn; 1043 1005 struct section *table_sec; 1044 1006 unsigned long table_offset; 1045 1007 ··· 1066 1028 break; 1067 1029 1068 1030 /* look for a relocation which references .rodata */ 1069 - text_rela = find_rela_by_dest_range(insn->sec, insn->offset, 1070 - insn->len); 1031 + text_rela = find_rela_by_dest_range(file->elf, insn->sec, 1032 + insn->offset, insn->len); 1071 1033 if (!text_rela || text_rela->sym->type != STT_SECTION || 1072 1034 !text_rela->sym->sec->rodata) 1073 1035 continue; ··· 1091 1053 strcmp(table_sec->name, C_JUMP_TABLE_SECTION)) 1092 1054 continue; 1093 1055 1094 - /* Each table entry has a rela associated with it. */ 1095 - table_rela = find_rela_by_dest(table_sec, table_offset); 1056 + /* 1057 + * Each table entry has a rela associated with it. The rela 1058 + * should reference text in the same function as the original 1059 + * instruction. 1060 + */ 1061 + table_rela = find_rela_by_dest(file->elf, table_sec, table_offset); 1096 1062 if (!table_rela) 1063 + continue; 1064 + dest_insn = find_insn(file, table_rela->sym->sec, table_rela->addend); 1065 + if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func) 1097 1066 continue; 1098 1067 1099 1068 /* ··· 1127 1082 struct instruction *insn, *last = NULL; 1128 1083 struct rela *rela; 1129 1084 1130 - func_for_each_insn_all(file, func, insn) { 1085 + func_for_each_insn(file, func, insn) { 1131 1086 if (!last) 1132 1087 last = insn; 1133 1088 ··· 1162 1117 struct instruction *insn; 1163 1118 int ret; 1164 1119 1165 - func_for_each_insn_all(file, func, insn) { 1120 + func_for_each_insn(file, func, insn) { 1166 1121 if (!insn->jump_table) 1167 1122 continue; 1168 1123 ··· 1232 1187 for (i = 0; i < sec->len / sizeof(struct unwind_hint); i++) { 1233 1188 hint = (struct unwind_hint *)sec->data->d_buf + i; 1234 1189 1235 - rela = find_rela_by_dest(sec, i * sizeof(*hint)); 1190 + rela = find_rela_by_dest(file->elf, sec, i * sizeof(*hint)); 1236 1191 if (!rela) { 1237 1192 WARN("can't find rela for unwind_hints[%d]", i); 1238 1193 return -1; ··· 1980 1935 return validate_call(insn, state); 1981 1936 } 1982 1937 1938 + static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 1939 + { 1940 + if (state->uaccess && !func_uaccess_safe(func)) { 1941 + WARN_FUNC("return with UACCESS enabled", 1942 + insn->sec, insn->offset); 1943 + return 1; 1944 + } 1945 + 1946 + if (!state->uaccess && func_uaccess_safe(func)) { 1947 + WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function", 1948 + insn->sec, insn->offset); 1949 + return 1; 1950 + } 1951 + 1952 + if (state->df) { 1953 + WARN_FUNC("return with DF set", 1954 + insn->sec, insn->offset); 1955 + return 1; 1956 + } 1957 + 1958 + if (func && has_modified_stack_frame(state)) { 1959 + WARN_FUNC("return with modified stack frame", 1960 + insn->sec, insn->offset); 1961 + return 1; 1962 + } 1963 + 1964 + if (state->bp_scratch) { 1965 + WARN("%s uses BP as a scratch register", 1966 + func->name); 1967 + return 1; 1968 + } 1969 + 1970 + return 0; 1971 + } 1972 + 1983 1973 /* 1984 1974 * Follow the branch starting at the given instruction, and recursively follow 1985 1975 * any other branches (jumps). Meanwhile, track the frame pointer state at ··· 2069 1989 2070 1990 i = insn; 2071 1991 save_insn = NULL; 2072 - func_for_each_insn_continue_reverse(file, func, i) { 1992 + sym_for_each_insn_continue_reverse(file, func, i) { 2073 1993 if (i->save) { 2074 1994 save_insn = i; 2075 1995 break; ··· 2130 2050 switch (insn->type) { 2131 2051 2132 2052 case INSN_RETURN: 2133 - if (state.uaccess && !func_uaccess_safe(func)) { 2134 - WARN_FUNC("return with UACCESS enabled", sec, insn->offset); 2135 - return 1; 2136 - } 2137 - 2138 - if (!state.uaccess && func_uaccess_safe(func)) { 2139 - WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function", sec, insn->offset); 2140 - return 1; 2141 - } 2142 - 2143 - if (state.df) { 2144 - WARN_FUNC("return with DF set", sec, insn->offset); 2145 - return 1; 2146 - } 2147 - 2148 - if (func && has_modified_stack_frame(&state)) { 2149 - WARN_FUNC("return with modified stack frame", 2150 - sec, insn->offset); 2151 - return 1; 2152 - } 2153 - 2154 - if (state.bp_scratch) { 2155 - WARN("%s uses BP as a scratch register", 2156 - func->name); 2157 - return 1; 2158 - } 2159 - 2160 - return 0; 2053 + return validate_return(func, insn, &state); 2161 2054 2162 2055 case INSN_CALL: 2163 2056 case INSN_CALL_DYNAMIC: ··· 2395 2342 return false; 2396 2343 } 2397 2344 2398 - static int validate_functions(struct objtool_file *file) 2345 + static int validate_section(struct objtool_file *file, struct section *sec) 2399 2346 { 2400 - struct section *sec; 2401 2347 struct symbol *func; 2402 2348 struct instruction *insn; 2403 2349 struct insn_state state; ··· 2409 2357 CFI_NUM_REGS * sizeof(struct cfi_reg)); 2410 2358 state.stack_size = initial_func_cfi.cfa.offset; 2411 2359 2412 - for_each_sec(file, sec) { 2413 - list_for_each_entry(func, &sec->symbol_list, list) { 2414 - if (func->type != STT_FUNC) 2415 - continue; 2360 + list_for_each_entry(func, &sec->symbol_list, list) { 2361 + if (func->type != STT_FUNC) 2362 + continue; 2416 2363 2417 - if (!func->len) { 2418 - WARN("%s() is missing an ELF size annotation", 2419 - func->name); 2420 - warnings++; 2421 - } 2422 - 2423 - if (func->pfunc != func || func->alias != func) 2424 - continue; 2425 - 2426 - insn = find_insn(file, sec, func->offset); 2427 - if (!insn || insn->ignore || insn->visited) 2428 - continue; 2429 - 2430 - state.uaccess = func->uaccess_safe; 2431 - 2432 - ret = validate_branch(file, func, insn, state); 2433 - if (ret && backtrace) 2434 - BT_FUNC("<=== (func)", insn); 2435 - warnings += ret; 2364 + if (!func->len) { 2365 + WARN("%s() is missing an ELF size annotation", 2366 + func->name); 2367 + warnings++; 2436 2368 } 2369 + 2370 + if (func->pfunc != func || func->alias != func) 2371 + continue; 2372 + 2373 + insn = find_insn(file, sec, func->offset); 2374 + if (!insn || insn->ignore || insn->visited) 2375 + continue; 2376 + 2377 + state.uaccess = func->uaccess_safe; 2378 + 2379 + ret = validate_branch(file, func, insn, state); 2380 + if (ret && backtrace) 2381 + BT_FUNC("<=== (func)", insn); 2382 + warnings += ret; 2437 2383 } 2384 + 2385 + return warnings; 2386 + } 2387 + 2388 + static int validate_functions(struct objtool_file *file) 2389 + { 2390 + struct section *sec; 2391 + int warnings = 0; 2392 + 2393 + for_each_sec(file, sec) 2394 + warnings += validate_section(file, sec); 2438 2395 2439 2396 return warnings; 2440 2397 } ··· 2464 2403 } 2465 2404 2466 2405 return 0; 2467 - } 2468 - 2469 - static void cleanup(struct objtool_file *file) 2470 - { 2471 - struct instruction *insn, *tmpinsn; 2472 - struct alternative *alt, *tmpalt; 2473 - 2474 - list_for_each_entry_safe(insn, tmpinsn, &file->insn_list, list) { 2475 - list_for_each_entry_safe(alt, tmpalt, &insn->alts, list) { 2476 - list_del(&alt->list); 2477 - free(alt); 2478 - } 2479 - list_del(&insn->list); 2480 - hash_del(&insn->hash); 2481 - free(insn); 2482 - } 2483 - elf_close(file->elf); 2484 2406 } 2485 2407 2486 2408 static struct objtool_file file; ··· 2533 2489 } 2534 2490 2535 2491 out: 2536 - cleanup(&file); 2492 + if (ret < 0) { 2493 + /* 2494 + * Fatal error. The binary is corrupt or otherwise broken in 2495 + * some way, or objtool itself is broken. Fail the kernel 2496 + * build. 2497 + */ 2498 + return ret; 2499 + } 2537 2500 2538 - /* ignore warnings for now until we get all the code cleaned up */ 2539 - if (ret || warnings) 2540 - return 0; 2541 2501 return 0; 2542 2502 }
+1 -1
tools/objtool/check.h
··· 50 50 struct objtool_file { 51 51 struct elf *elf; 52 52 struct list_head insn_list; 53 - DECLARE_HASHTABLE(insn_hash, 16); 53 + DECLARE_HASHTABLE(insn_hash, 20); 54 54 bool ignore_unreachables, c_file, hints, rodata; 55 55 }; 56 56
+203 -80
tools/objtool/elf.c
··· 15 15 #include <string.h> 16 16 #include <unistd.h> 17 17 #include <errno.h> 18 + #include "builtin.h" 18 19 19 20 #include "elf.h" 20 21 #include "warn.h" 21 22 22 23 #define MAX_NAME_LEN 128 23 24 25 + static inline u32 str_hash(const char *str) 26 + { 27 + return jhash(str, strlen(str), 0); 28 + } 29 + 30 + static void rb_add(struct rb_root *tree, struct rb_node *node, 31 + int (*cmp)(struct rb_node *, const struct rb_node *)) 32 + { 33 + struct rb_node **link = &tree->rb_node; 34 + struct rb_node *parent = NULL; 35 + 36 + while (*link) { 37 + parent = *link; 38 + if (cmp(node, parent) < 0) 39 + link = &parent->rb_left; 40 + else 41 + link = &parent->rb_right; 42 + } 43 + 44 + rb_link_node(node, parent, link); 45 + rb_insert_color(node, tree); 46 + } 47 + 48 + static struct rb_node *rb_find_first(struct rb_root *tree, const void *key, 49 + int (*cmp)(const void *key, const struct rb_node *)) 50 + { 51 + struct rb_node *node = tree->rb_node; 52 + struct rb_node *match = NULL; 53 + 54 + while (node) { 55 + int c = cmp(key, node); 56 + if (c <= 0) { 57 + if (!c) 58 + match = node; 59 + node = node->rb_left; 60 + } else if (c > 0) { 61 + node = node->rb_right; 62 + } 63 + } 64 + 65 + return match; 66 + } 67 + 68 + static struct rb_node *rb_next_match(struct rb_node *node, const void *key, 69 + int (*cmp)(const void *key, const struct rb_node *)) 70 + { 71 + node = rb_next(node); 72 + if (node && cmp(key, node)) 73 + node = NULL; 74 + return node; 75 + } 76 + 77 + #define rb_for_each(tree, node, key, cmp) \ 78 + for ((node) = rb_find_first((tree), (key), (cmp)); \ 79 + (node); (node) = rb_next_match((node), (key), (cmp))) 80 + 81 + static int symbol_to_offset(struct rb_node *a, const struct rb_node *b) 82 + { 83 + struct symbol *sa = rb_entry(a, struct symbol, node); 84 + struct symbol *sb = rb_entry(b, struct symbol, node); 85 + 86 + if (sa->offset < sb->offset) 87 + return -1; 88 + if (sa->offset > sb->offset) 89 + return 1; 90 + 91 + if (sa->len < sb->len) 92 + return -1; 93 + if (sa->len > sb->len) 94 + return 1; 95 + 96 + sa->alias = sb; 97 + 98 + return 0; 99 + } 100 + 101 + static int symbol_by_offset(const void *key, const struct rb_node *node) 102 + { 103 + const struct symbol *s = rb_entry(node, struct symbol, node); 104 + const unsigned long *o = key; 105 + 106 + if (*o < s->offset) 107 + return -1; 108 + if (*o > s->offset + s->len) 109 + return 1; 110 + 111 + return 0; 112 + } 113 + 24 114 struct section *find_section_by_name(struct elf *elf, const char *name) 25 115 { 26 116 struct section *sec; 27 117 28 - list_for_each_entry(sec, &elf->sections, list) 118 + hash_for_each_possible(elf->section_name_hash, sec, name_hash, str_hash(name)) 29 119 if (!strcmp(sec->name, name)) 30 120 return sec; 31 121 ··· 127 37 { 128 38 struct section *sec; 129 39 130 - list_for_each_entry(sec, &elf->sections, list) 40 + hash_for_each_possible(elf->section_hash, sec, hash, idx) 131 41 if (sec->idx == idx) 132 42 return sec; 133 43 ··· 136 46 137 47 static struct symbol *find_symbol_by_index(struct elf *elf, unsigned int idx) 138 48 { 139 - struct section *sec; 140 49 struct symbol *sym; 141 50 142 - list_for_each_entry(sec, &elf->sections, list) 143 - hash_for_each_possible(sec->symbol_hash, sym, hash, idx) 144 - if (sym->idx == idx) 145 - return sym; 51 + hash_for_each_possible(elf->symbol_hash, sym, hash, idx) 52 + if (sym->idx == idx) 53 + return sym; 146 54 147 55 return NULL; 148 56 } 149 57 150 58 struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset) 151 59 { 152 - struct symbol *sym; 60 + struct rb_node *node; 153 61 154 - list_for_each_entry(sym, &sec->symbol_list, list) 155 - if (sym->type != STT_SECTION && 156 - sym->offset == offset) 157 - return sym; 62 + rb_for_each(&sec->symbol_tree, node, &offset, symbol_by_offset) { 63 + struct symbol *s = rb_entry(node, struct symbol, node); 64 + 65 + if (s->offset == offset && s->type != STT_SECTION) 66 + return s; 67 + } 158 68 159 69 return NULL; 160 70 } 161 71 162 - struct symbol *find_symbol_by_name(struct elf *elf, const char *name) 72 + struct symbol *find_func_by_offset(struct section *sec, unsigned long offset) 163 73 { 164 - struct section *sec; 165 - struct symbol *sym; 74 + struct rb_node *node; 166 75 167 - list_for_each_entry(sec, &elf->sections, list) 168 - list_for_each_entry(sym, &sec->symbol_list, list) 169 - if (!strcmp(sym->name, name)) 170 - return sym; 76 + rb_for_each(&sec->symbol_tree, node, &offset, symbol_by_offset) { 77 + struct symbol *s = rb_entry(node, struct symbol, node); 78 + 79 + if (s->offset == offset && s->type == STT_FUNC) 80 + return s; 81 + } 171 82 172 83 return NULL; 173 84 } 174 85 175 86 struct symbol *find_symbol_containing(struct section *sec, unsigned long offset) 176 87 { 88 + struct rb_node *node; 89 + 90 + rb_for_each(&sec->symbol_tree, node, &offset, symbol_by_offset) { 91 + struct symbol *s = rb_entry(node, struct symbol, node); 92 + 93 + if (s->type != STT_SECTION) 94 + return s; 95 + } 96 + 97 + return NULL; 98 + } 99 + 100 + struct symbol *find_func_containing(struct section *sec, unsigned long offset) 101 + { 102 + struct rb_node *node; 103 + 104 + rb_for_each(&sec->symbol_tree, node, &offset, symbol_by_offset) { 105 + struct symbol *s = rb_entry(node, struct symbol, node); 106 + 107 + if (s->type == STT_FUNC) 108 + return s; 109 + } 110 + 111 + return NULL; 112 + } 113 + 114 + struct symbol *find_symbol_by_name(struct elf *elf, const char *name) 115 + { 177 116 struct symbol *sym; 178 117 179 - list_for_each_entry(sym, &sec->symbol_list, list) 180 - if (sym->type != STT_SECTION && 181 - offset >= sym->offset && offset < sym->offset + sym->len) 118 + hash_for_each_possible(elf->symbol_name_hash, sym, name_hash, str_hash(name)) 119 + if (!strcmp(sym->name, name)) 182 120 return sym; 183 121 184 122 return NULL; 185 123 } 186 124 187 - struct rela *find_rela_by_dest_range(struct section *sec, unsigned long offset, 188 - unsigned int len) 125 + struct rela *find_rela_by_dest_range(struct elf *elf, struct section *sec, 126 + unsigned long offset, unsigned int len) 189 127 { 190 - struct rela *rela; 128 + struct rela *rela, *r = NULL; 191 129 unsigned long o; 192 130 193 131 if (!sec->rela) 194 132 return NULL; 195 133 196 - for (o = offset; o < offset + len; o++) 197 - hash_for_each_possible(sec->rela->rela_hash, rela, hash, o) 198 - if (rela->offset == o) 199 - return rela; 134 + sec = sec->rela; 135 + 136 + for_offset_range(o, offset, offset + len) { 137 + hash_for_each_possible(elf->rela_hash, rela, hash, 138 + sec_offset_hash(sec, o)) { 139 + if (rela->sec != sec) 140 + continue; 141 + 142 + if (rela->offset >= offset && rela->offset < offset + len) { 143 + if (!r || rela->offset < r->offset) 144 + r = rela; 145 + } 146 + } 147 + if (r) 148 + return r; 149 + } 200 150 201 151 return NULL; 202 152 } 203 153 204 - struct rela *find_rela_by_dest(struct section *sec, unsigned long offset) 154 + struct rela *find_rela_by_dest(struct elf *elf, struct section *sec, unsigned long offset) 205 155 { 206 - return find_rela_by_dest_range(sec, offset, 1); 207 - } 208 - 209 - struct symbol *find_containing_func(struct section *sec, unsigned long offset) 210 - { 211 - struct symbol *func; 212 - 213 - list_for_each_entry(func, &sec->symbol_list, list) 214 - if (func->type == STT_FUNC && offset >= func->offset && 215 - offset < func->offset + func->len) 216 - return func; 217 - 218 - return NULL; 156 + return find_rela_by_dest_range(elf, sec, offset, 1); 219 157 } 220 158 221 159 static int read_sections(struct elf *elf) ··· 273 155 274 156 INIT_LIST_HEAD(&sec->symbol_list); 275 157 INIT_LIST_HEAD(&sec->rela_list); 276 - hash_init(sec->rela_hash); 277 - hash_init(sec->symbol_hash); 278 - 279 - list_add_tail(&sec->list, &elf->sections); 280 158 281 159 s = elf_getscn(elf->elf, i); 282 160 if (!s) { ··· 307 193 } 308 194 } 309 195 sec->len = sec->sh.sh_size; 196 + 197 + list_add_tail(&sec->list, &elf->sections); 198 + hash_add(elf->section_hash, &sec->hash, sec->idx); 199 + hash_add(elf->section_name_hash, &sec->name_hash, str_hash(sec->name)); 310 200 } 201 + 202 + if (stats) 203 + printf("nr_sections: %lu\n", (unsigned long)sections_nr); 311 204 312 205 /* sanity check, one more call to elf_nextscn() should return NULL */ 313 206 if (elf_nextscn(elf->elf, s)) { ··· 328 207 static int read_symbols(struct elf *elf) 329 208 { 330 209 struct section *symtab, *sec; 331 - struct symbol *sym, *pfunc, *alias; 332 - struct list_head *entry, *tmp; 210 + struct symbol *sym, *pfunc; 211 + struct list_head *entry; 212 + struct rb_node *pnode; 333 213 int symbols_nr, i; 334 214 char *coldstr; 335 215 ··· 349 227 return -1; 350 228 } 351 229 memset(sym, 0, sizeof(*sym)); 352 - alias = sym; 230 + sym->alias = sym; 353 231 354 232 sym->idx = i; 355 233 ··· 387 265 sym->offset = sym->sym.st_value; 388 266 sym->len = sym->sym.st_size; 389 267 390 - /* sorted insert into a per-section list */ 391 - entry = &sym->sec->symbol_list; 392 - list_for_each_prev(tmp, &sym->sec->symbol_list) { 393 - struct symbol *s; 394 - 395 - s = list_entry(tmp, struct symbol, list); 396 - 397 - if (sym->offset > s->offset) { 398 - entry = tmp; 399 - break; 400 - } 401 - 402 - if (sym->offset == s->offset) { 403 - if (sym->len && sym->len == s->len && alias == sym) 404 - alias = s; 405 - 406 - if (sym->len >= s->len) { 407 - entry = tmp; 408 - break; 409 - } 410 - } 411 - } 412 - sym->alias = alias; 268 + rb_add(&sym->sec->symbol_tree, &sym->node, symbol_to_offset); 269 + pnode = rb_prev(&sym->node); 270 + if (pnode) 271 + entry = &rb_entry(pnode, struct symbol, node)->list; 272 + else 273 + entry = &sym->sec->symbol_list; 413 274 list_add(&sym->list, entry); 414 - hash_add(sym->sec->symbol_hash, &sym->hash, sym->idx); 275 + hash_add(elf->symbol_hash, &sym->hash, sym->idx); 276 + hash_add(elf->symbol_name_hash, &sym->name_hash, str_hash(sym->name)); 415 277 } 278 + 279 + if (stats) 280 + printf("nr_symbols: %lu\n", (unsigned long)symbols_nr); 416 281 417 282 /* Create parent/child links for any cold subfunctions */ 418 283 list_for_each_entry(sec, &elf->sections, list) { ··· 462 353 struct rela *rela; 463 354 int i; 464 355 unsigned int symndx; 356 + unsigned long nr_rela, max_rela = 0, tot_rela = 0; 465 357 466 358 list_for_each_entry(sec, &elf->sections, list) { 467 359 if (sec->sh.sh_type != SHT_RELA) ··· 477 367 478 368 sec->base->rela = sec; 479 369 370 + nr_rela = 0; 480 371 for (i = 0; i < sec->sh.sh_size / sec->sh.sh_entsize; i++) { 481 372 rela = malloc(sizeof(*rela)); 482 373 if (!rela) { ··· 504 393 } 505 394 506 395 list_add_tail(&rela->list, &sec->rela_list); 507 - hash_add(sec->rela_hash, &rela->hash, rela->offset); 508 - 396 + hash_add(elf->rela_hash, &rela->hash, rela_hash(rela)); 397 + nr_rela++; 509 398 } 399 + max_rela = max(max_rela, nr_rela); 400 + tot_rela += nr_rela; 401 + } 402 + 403 + if (stats) { 404 + printf("max_rela: %lu\n", max_rela); 405 + printf("tot_rela: %lu\n", tot_rela); 510 406 } 511 407 512 408 return 0; ··· 533 415 } 534 416 memset(elf, 0, sizeof(*elf)); 535 417 418 + hash_init(elf->symbol_hash); 419 + hash_init(elf->symbol_name_hash); 420 + hash_init(elf->section_hash); 421 + hash_init(elf->section_name_hash); 422 + hash_init(elf->rela_hash); 536 423 INIT_LIST_HEAD(&elf->sections); 537 424 538 425 elf->fd = open(name, flags); ··· 598 475 599 476 INIT_LIST_HEAD(&sec->symbol_list); 600 477 INIT_LIST_HEAD(&sec->rela_list); 601 - hash_init(sec->rela_hash); 602 - hash_init(sec->symbol_hash); 603 - 604 - list_add_tail(&sec->list, &elf->sections); 605 478 606 479 s = elf_newscn(elf->elf); 607 480 if (!s) { ··· 674 555 675 556 shstrtab->len += strlen(name) + 1; 676 557 shstrtab->changed = true; 558 + 559 + list_add_tail(&sec->list, &elf->sections); 560 + hash_add(elf->section_hash, &sec->hash, sec->idx); 561 + hash_add(elf->section_name_hash, &sec->name_hash, str_hash(sec->name)); 677 562 678 563 return sec; 679 564 }
+44 -7
tools/objtool/elf.h
··· 10 10 #include <gelf.h> 11 11 #include <linux/list.h> 12 12 #include <linux/hashtable.h> 13 + #include <linux/rbtree.h> 14 + #include <linux/jhash.h> 13 15 14 16 #ifdef LIBELF_USE_DEPRECATED 15 17 # define elf_getshdrnum elf_getshnum ··· 27 25 28 26 struct section { 29 27 struct list_head list; 28 + struct hlist_node hash; 29 + struct hlist_node name_hash; 30 30 GElf_Shdr sh; 31 + struct rb_root symbol_tree; 31 32 struct list_head symbol_list; 32 - DECLARE_HASHTABLE(symbol_hash, 8); 33 33 struct list_head rela_list; 34 - DECLARE_HASHTABLE(rela_hash, 16); 35 34 struct section *base, *rela; 36 35 struct symbol *sym; 37 36 Elf_Data *data; ··· 44 41 45 42 struct symbol { 46 43 struct list_head list; 44 + struct rb_node node; 47 45 struct hlist_node hash; 46 + struct hlist_node name_hash; 48 47 GElf_Sym sym; 49 48 struct section *sec; 50 49 char *name; ··· 76 71 int fd; 77 72 char *name; 78 73 struct list_head sections; 79 - DECLARE_HASHTABLE(rela_hash, 16); 74 + DECLARE_HASHTABLE(symbol_hash, 20); 75 + DECLARE_HASHTABLE(symbol_name_hash, 20); 76 + DECLARE_HASHTABLE(section_hash, 16); 77 + DECLARE_HASHTABLE(section_name_hash, 16); 78 + DECLARE_HASHTABLE(rela_hash, 20); 80 79 }; 81 80 81 + #define OFFSET_STRIDE_BITS 4 82 + #define OFFSET_STRIDE (1UL << OFFSET_STRIDE_BITS) 83 + #define OFFSET_STRIDE_MASK (~(OFFSET_STRIDE - 1)) 84 + 85 + #define for_offset_range(_offset, _start, _end) \ 86 + for (_offset = ((_start) & OFFSET_STRIDE_MASK); \ 87 + _offset <= ((_end) & OFFSET_STRIDE_MASK); \ 88 + _offset += OFFSET_STRIDE) 89 + 90 + static inline u32 sec_offset_hash(struct section *sec, unsigned long offset) 91 + { 92 + u32 ol, oh, idx = sec->idx; 93 + 94 + offset &= OFFSET_STRIDE_MASK; 95 + 96 + ol = offset; 97 + oh = offset >> 32; 98 + 99 + __jhash_mix(ol, oh, idx); 100 + 101 + return ol; 102 + } 103 + 104 + static inline u32 rela_hash(struct rela *rela) 105 + { 106 + return sec_offset_hash(rela->sec, rela->offset); 107 + } 82 108 83 109 struct elf *elf_read(const char *name, int flags); 84 110 struct section *find_section_by_name(struct elf *elf, const char *name); 111 + struct symbol *find_func_by_offset(struct section *sec, unsigned long offset); 85 112 struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset); 86 113 struct symbol *find_symbol_by_name(struct elf *elf, const char *name); 87 114 struct symbol *find_symbol_containing(struct section *sec, unsigned long offset); 88 - struct rela *find_rela_by_dest(struct section *sec, unsigned long offset); 89 - struct rela *find_rela_by_dest_range(struct section *sec, unsigned long offset, 90 - unsigned int len); 91 - struct symbol *find_containing_func(struct section *sec, unsigned long offset); 115 + struct rela *find_rela_by_dest(struct elf *elf, struct section *sec, unsigned long offset); 116 + struct rela *find_rela_by_dest_range(struct elf *elf, struct section *sec, 117 + unsigned long offset, unsigned int len); 118 + struct symbol *find_func_containing(struct section *sec, unsigned long offset); 92 119 struct section *elf_create_section(struct elf *elf, const char *name, size_t 93 120 entsize, int nr); 94 121 struct section *elf_create_rela_section(struct elf *elf, struct section *base);
+5 -4
tools/objtool/orc_gen.c
··· 81 81 return 0; 82 82 } 83 83 84 - static int create_orc_entry(struct section *u_sec, struct section *ip_relasec, 84 + static int create_orc_entry(struct elf *elf, struct section *u_sec, struct section *ip_relasec, 85 85 unsigned int idx, struct section *insn_sec, 86 86 unsigned long insn_off, struct orc_entry *o) 87 87 { ··· 109 109 rela->addend = insn_off; 110 110 rela->type = R_X86_64_PC32; 111 111 rela->offset = idx * sizeof(int); 112 + rela->sec = ip_relasec; 112 113 113 114 list_add_tail(&rela->list, &ip_relasec->rela_list); 114 - hash_add(ip_relasec->rela_hash, &rela->hash, rela->offset); 115 + hash_add(elf->rela_hash, &rela->hash, rela_hash(rela)); 115 116 116 117 return 0; 117 118 } ··· 183 182 if (!prev_insn || memcmp(&insn->orc, &prev_insn->orc, 184 183 sizeof(struct orc_entry))) { 185 184 186 - if (create_orc_entry(u_sec, ip_relasec, idx, 185 + if (create_orc_entry(file->elf, u_sec, ip_relasec, idx, 187 186 insn->sec, insn->offset, 188 187 &insn->orc)) 189 188 return -1; ··· 195 194 196 195 /* section terminator */ 197 196 if (prev_insn) { 198 - if (create_orc_entry(u_sec, ip_relasec, idx, 197 + if (create_orc_entry(file->elf, u_sec, ip_relasec, idx, 199 198 prev_insn->sec, 200 199 prev_insn->offset + prev_insn->len, 201 200 &empty))
+2 -2
tools/objtool/special.c
··· 118 118 } 119 119 } 120 120 121 - orig_rela = find_rela_by_dest(sec, offset + entry->orig); 121 + orig_rela = find_rela_by_dest(elf, sec, offset + entry->orig); 122 122 if (!orig_rela) { 123 123 WARN_FUNC("can't find orig rela", sec, offset + entry->orig); 124 124 return -1; ··· 133 133 alt->orig_off = orig_rela->addend; 134 134 135 135 if (!entry->group || alt->new_len) { 136 - new_rela = find_rela_by_dest(sec, offset + entry->new); 136 + new_rela = find_rela_by_dest(elf, sec, offset + entry->new); 137 137 if (!new_rela) { 138 138 WARN_FUNC("can't find new rela", 139 139 sec, offset + entry->new);
+1 -1
tools/objtool/warn.h
··· 21 21 char *name, *str; 22 22 unsigned long name_off; 23 23 24 - func = find_containing_func(sec, offset); 24 + func = find_func_containing(sec, offset); 25 25 if (func) { 26 26 name = func->name; 27 27 name_off = offset - func->offset;