+2 -1

MAINTAINERS

reviewed

··· 14459 14459 F: Documentation/ABI/testing/sysfs-kernel-livepatch 14460 14460 F: Documentation/livepatch/ 14461 14461 F: arch/powerpc/include/asm/livepatch.h 14462 14462 - F: include/linux/livepatch.h 14462 14462 + F: include/linux/livepatch*.h 14463 14463 F: kernel/livepatch/ 14464 14464 F: kernel/module/livepatch.c 14465 14465 F: samples/livepatch/ 14466 14466 + F: scripts/livepatch/ 14466 14467 F: tools/testing/selftests/livepatch/ 14467 14468 14468 14469 LLC (802.2)

+1 -1

arch/s390/include/asm/nospec-insn.h

reviewed

··· 19 19 #ifdef CONFIG_EXPOLINE_EXTERN 20 20 SYM_CODE_START(\name) 21 21 #else 22 22 - .pushsection .text.\name,"axG",@progbits,\name,comdat 22 22 + .pushsection .text..\name,"axG",@progbits,\name,comdat 23 23 .globl \name 24 24 .hidden \name 25 25 .type \name,@function

+1 -1

arch/s390/kernel/vmlinux.lds.S

reviewed

··· 51 51 IRQENTRY_TEXT 52 52 SOFTIRQENTRY_TEXT 53 53 FTRACE_HOTPATCH_TRAMPOLINES_TEXT 54 54 - *(.text.*_indirect_*) 54 54 + *(.text..*_indirect_*) 55 55 *(.gnu.warning) 56 56 . = ALIGN(PAGE_SIZE); 57 57 _etext = .; /* End of text section */

+1

arch/x86/Kconfig

reviewed

··· 261 261 select HAVE_FUNCTION_ERROR_INJECTION 262 262 select HAVE_KRETPROBES 263 263 select HAVE_RETHOOK 264 264 + select HAVE_KLP_BUILD if X86_64 264 265 select HAVE_LIVEPATCH if X86_64 265 266 select HAVE_MIXED_BREAKPOINTS_REGS 266 267 select HAVE_MOD_ARCH_SPECIFIC

+1 -1

arch/x86/boot/startup/Makefile

reviewed

··· 36 36 # relocations, even if other objtool actions are being deferred. 37 37 # 38 38 $(pi-objs): objtool-enabled = 1 39 39 - $(pi-objs): objtool-args = $(if $(delay-objtool),,$(objtool-args-y)) --noabs 39 39 + $(pi-objs): objtool-args = $(if $(delay-objtool),--dry-run,$(objtool-args-y)) --noabs 40 40 41 41 # 42 42 # Confine the startup code by prefixing all symbols with __pi_ (for position

+4

arch/x86/include/asm/alternative.h

reviewed

··· 198 198 199 199 #define ALTINSTR_ENTRY(ft_flags) \ 200 200 ".pushsection .altinstructions,\"a\"\n" \ 201 201 + ANNOTATE_DATA_SPECIAL \ 201 202 " .long 771b - .\n" /* label */ \ 202 203 " .long 774f - .\n" /* new instruction */ \ 203 204 " .4byte " __stringify(ft_flags) "\n" /* feature + flags */ \ ··· 208 207 209 208 #define ALTINSTR_REPLACEMENT(newinstr) /* replacement */ \ 210 209 ".pushsection .altinstr_replacement, \"ax\"\n" \ 210 210 + ANNOTATE_DATA_SPECIAL \ 211 211 "# ALT: replacement\n" \ 212 212 "774:\n\t" newinstr "\n775:\n" \ 213 213 ".popsection\n" ··· 339 337 * instruction. See apply_alternatives(). 340 338 */ 341 339 .macro altinstr_entry orig alt ft_flags orig_len alt_len 340 340 + ANNOTATE_DATA_SPECIAL 342 341 .long \orig - . 343 342 .long \alt - . 344 343 .4byte \ft_flags ··· 368 365 .popsection ; \ 369 366 .pushsection .altinstr_replacement,"ax" ; \ 370 367 743: \ 368 368 + ANNOTATE_DATA_SPECIAL ; \ 371 369 newinst ; \ 372 370 744: \ 373 371 .popsection ;

+5

arch/x86/include/asm/asm.h

reviewed

··· 2 2 #ifndef _ASM_X86_ASM_H 3 3 #define _ASM_X86_ASM_H 4 4 5 5 + #include <linux/annotate.h> 6 6 + 5 7 #ifdef __ASSEMBLER__ 6 8 # define __ASM_FORM(x, ...) x,## __VA_ARGS__ 7 9 # define __ASM_FORM_RAW(x, ...) x,## __VA_ARGS__ ··· 134 132 # define _ASM_EXTABLE_TYPE(from, to, type) \ 135 133 .pushsection "__ex_table","a" ; \ 136 134 .balign 4 ; \ 135 135 + ANNOTATE_DATA_SPECIAL ; \ 137 136 .long (from) - . ; \ 138 137 .long (to) - . ; \ 139 138 .long type ; \ ··· 182 179 # define _ASM_EXTABLE_TYPE(from, to, type) \ 183 180 " .pushsection \"__ex_table\",\"a\"\n" \ 184 181 " .balign 4\n" \ 182 182 + ANNOTATE_DATA_SPECIAL \ 185 183 " .long (" #from ") - .\n" \ 186 184 " .long (" #to ") - .\n" \ 187 185 " .long " __stringify(type) " \n" \ ··· 191 187 # define _ASM_EXTABLE_TYPE_REG(from, to, type, reg) \ 192 188 " .pushsection \"__ex_table\",\"a\"\n" \ 193 189 " .balign 4\n" \ 190 190 + ANNOTATE_DATA_SPECIAL \ 194 191 " .long (" #from ") - .\n" \ 195 192 " .long (" #to ") - .\n" \ 196 193 DEFINE_EXTABLE_TYPE_REG \

+1

arch/x86/include/asm/bug.h

reviewed

··· 57 57 #define _BUG_FLAGS_ASM(ins, file, line, flags, size, extra) \ 58 58 "1:\t" ins "\n" \ 59 59 ".pushsection __bug_table,\"aw\"\n" \ 60 60 + ANNOTATE_DATA_SPECIAL \ 60 61 __BUG_ENTRY(file, line, flags) \ 61 62 "\t.org 2b + " size "\n" \ 62 63 ".popsection\n" \

+1

arch/x86/include/asm/cpufeature.h

reviewed

··· 101 101 asm goto(ALTERNATIVE_TERNARY("jmp 6f", %c[feature], "", "jmp %l[t_no]") 102 102 ".pushsection .altinstr_aux,\"ax\"\n" 103 103 "6:\n" 104 104 + ANNOTATE_DATA_SPECIAL 104 105 " testb %[bitnum], %a[cap_byte]\n" 105 106 " jnz %l[t_yes]\n" 106 107 " jmp %l[t_no]\n"

+1

arch/x86/include/asm/jump_label.h

reviewed

··· 15 15 #define JUMP_TABLE_ENTRY(key, label) \ 16 16 ".pushsection __jump_table, \"aw\" \n\t" \ 17 17 _ASM_ALIGN "\n\t" \ 18 18 + ANNOTATE_DATA_SPECIAL \ 18 19 ".long 1b - . \n\t" \ 19 20 ".long " label " - . \n\t" \ 20 21 _ASM_PTR " " key " - . \n\t" \

+1 -1

arch/x86/include/asm/smp.h

reviewed

··· 109 109 int native_kick_ap(unsigned int cpu, struct task_struct *tidle); 110 110 int native_cpu_disable(void); 111 111 void __noreturn hlt_play_dead(void); 112 112 - void native_play_dead(void); 112 112 + void __noreturn native_play_dead(void); 113 113 void play_dead_common(void); 114 114 void wbinvd_on_cpu(int cpu); 115 115 void wbinvd_on_all_cpus(void);

+28 -23

arch/x86/kernel/alternative.c

reviewed

··· 2244 2244 * See entry_{32,64}.S for more details. 2245 2245 */ 2246 2246 2247 2247 - /* 2248 2248 - * We define the int3_magic() function in assembly to control the calling 2249 2249 - * convention such that we can 'call' it from assembly. 2250 2250 - */ 2251 2251 - 2252 2252 - extern void int3_magic(unsigned int *ptr); /* defined in asm */ 2247 2247 + extern void int3_selftest_asm(unsigned int *ptr); 2253 2248 2254 2249 asm ( 2255 2250 " .pushsection .init.text, \"ax\", @progbits\n" 2256 2256 - " .type int3_magic, @function\n" 2257 2257 - "int3_magic:\n" 2251 2251 + " .type int3_selftest_asm, @function\n" 2252 2252 + "int3_selftest_asm:\n" 2258 2253 ANNOTATE_NOENDBR 2259 2259 - " movl $1, (%" _ASM_ARG1 ")\n" 2254 2254 + /* 2255 2255 + * INT3 padded with NOP to CALL_INSN_SIZE. The INT3 triggers an 2256 2256 + * exception, then the int3_exception_nb notifier emulates a call to 2257 2257 + * int3_selftest_callee(). 2258 2258 + */ 2259 2259 + " int3; nop; nop; nop; nop\n" 2260 2260 ASM_RET 2261 2261 - " .size int3_magic, .-int3_magic\n" 2261 2261 + " .size int3_selftest_asm, . - int3_selftest_asm\n" 2262 2262 + " .popsection\n" 2263 2263 + ); 2264 2264 + 2265 2265 + extern void int3_selftest_callee(unsigned int *ptr); 2266 2266 + 2267 2267 + asm ( 2268 2268 + " .pushsection .init.text, \"ax\", @progbits\n" 2269 2269 + " .type int3_selftest_callee, @function\n" 2270 2270 + "int3_selftest_callee:\n" 2271 2271 + ANNOTATE_NOENDBR 2272 2272 + " movl $0x1234, (%" _ASM_ARG1 ")\n" 2273 2273 + ASM_RET 2274 2274 + " .size int3_selftest_callee, . - int3_selftest_callee\n" 2262 2275 " .popsection\n" 2263 2276 ); 2264 2277 ··· 2280 2267 static int __init 2281 2268 int3_exception_notify(struct notifier_block *self, unsigned long val, void *data) 2282 2269 { 2283 2283 - unsigned long selftest = (unsigned long)&int3_selftest_ip; 2270 2270 + unsigned long selftest = (unsigned long)&int3_selftest_asm; 2284 2271 struct die_args *args = data; 2285 2272 struct pt_regs *regs = args->regs; 2286 2273 ··· 2295 2282 if (regs->ip - INT3_INSN_SIZE != selftest) 2296 2283 return NOTIFY_DONE; 2297 2284 2298 2298 - int3_emulate_call(regs, (unsigned long)&int3_magic); 2285 2285 + int3_emulate_call(regs, (unsigned long)&int3_selftest_callee); 2299 2286 return NOTIFY_STOP; 2300 2287 } 2301 2288 ··· 2311 2298 BUG_ON(register_die_notifier(&int3_exception_nb)); 2312 2299 2313 2300 /* 2314 2314 - * Basically: int3_magic(&val); but really complicated :-) 2315 2315 - * 2316 2316 - * INT3 padded with NOP to CALL_INSN_SIZE. The int3_exception_nb 2317 2317 - * notifier above will emulate CALL for us. 2301 2301 + * Basically: int3_selftest_callee(&val); but really complicated :-) 2318 2302 */ 2319 2319 - asm volatile ("int3_selftest_ip:\n\t" 2320 2320 - ANNOTATE_NOENDBR 2321 2321 - " int3; nop; nop; nop; nop\n\t" 2322 2322 - : ASM_CALL_CONSTRAINT 2323 2323 - : __ASM_SEL_RAW(a, D) (&val) 2324 2324 - : "memory"); 2303 2303 + int3_selftest_asm(&val); 2325 2304 2326 2326 - BUG_ON(val != 1); 2305 2305 + BUG_ON(val != 0x1234); 2327 2306 2328 2307 unregister_die_notifier(&int3_exception_nb); 2329 2308 }

-4

arch/x86/kernel/kprobes/opt.c

reviewed

··· 103 103 104 104 asm ( 105 105 ".pushsection .rodata\n" 106 106 - "optprobe_template_func:\n" 107 106 ".global optprobe_template_entry\n" 108 107 "optprobe_template_entry:\n" 109 108 #ifdef CONFIG_X86_64 ··· 158 159 ".global optprobe_template_end\n" 159 160 "optprobe_template_end:\n" 160 161 ".popsection\n"); 161 161 - 162 162 - void optprobe_template_func(void); 163 163 - STACK_FRAME_NON_STANDARD(optprobe_template_func); 164 162 165 163 #define TMPL_CLAC_IDX \ 166 164 ((long)optprobe_template_clac - (long)optprobe_template_entry)

+9 -6

arch/x86/kernel/module.c

reviewed

··· 97 97 DEBUGP("%s relocate section %u to %u\n", 98 98 apply ? "Applying" : "Clearing", 99 99 relsec, sechdrs[relsec].sh_info); 100 100 + 100 101 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 101 102 size_t size; 102 103 ··· 163 162 164 163 if (apply) { 165 164 if (memcmp(loc, &zero, size)) { 166 166 - pr_err("x86/modules: Invalid relocation target, existing value is nonzero for type %d, loc %p, val %Lx\n", 167 167 - (int)ELF64_R_TYPE(rel[i].r_info), loc, val); 165 165 + pr_err("x86/modules: Invalid relocation target, existing value is nonzero for sec %u, idx %u, type %d, loc %lx, val %llx\n", 166 166 + relsec, i, (int)ELF64_R_TYPE(rel[i].r_info), 167 167 + (unsigned long)loc, val); 168 168 return -ENOEXEC; 169 169 } 170 170 write(loc, &val, size); 171 171 } else { 172 172 if (memcmp(loc, &val, size)) { 173 173 - pr_warn("x86/modules: Invalid relocation target, existing value does not match expected value for type %d, loc %p, val %Lx\n", 174 174 - (int)ELF64_R_TYPE(rel[i].r_info), loc, val); 173 173 + pr_warn("x86/modules: Invalid relocation target, existing value does not match expected value for sec %u, idx %u, type %d, loc %lx, val %llx\n", 174 174 + relsec, i, (int)ELF64_R_TYPE(rel[i].r_info), 175 175 + (unsigned long)loc, val); 175 176 return -ENOEXEC; 176 177 } 177 178 write(loc, &zero, size); ··· 182 179 return 0; 183 180 184 181 overflow: 185 185 - pr_err("overflow in relocation type %d val %Lx\n", 186 186 - (int)ELF64_R_TYPE(rel[i].r_info), val); 182 182 + pr_err("overflow in relocation type %d val %llx sec %u idx %d\n", 183 183 + (int)ELF64_R_TYPE(rel[i].r_info), val, relsec, i); 187 184 pr_err("`%s' likely not compiled with -mcmodel=kernel\n", 188 185 me->name); 189 186 return -ENOEXEC;

+2 -6

arch/x86/kernel/smpboot.c

reviewed

··· 1328 1328 native_halt(); 1329 1329 } 1330 1330 1331 1331 - /* 1332 1332 - * native_play_dead() is essentially a __noreturn function, but it can't 1333 1333 - * be marked as such as the compiler may complain about it. 1334 1334 - */ 1335 1335 - void native_play_dead(void) 1331 1331 + void __noreturn native_play_dead(void) 1336 1332 { 1337 1333 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) 1338 1334 __update_spec_ctrl(0); ··· 1347 1351 return -ENOSYS; 1348 1352 } 1349 1353 1350 1350 - void native_play_dead(void) 1354 1354 + void __noreturn native_play_dead(void) 1351 1355 { 1352 1356 BUG(); 1353 1357 }

+4

drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.h

reviewed

··· 53 53 usnic_uiom_interval_tree_remove(struct usnic_uiom_interval_node *node, 54 54 struct rb_root_cached *root); 55 55 extern struct usnic_uiom_interval_node * 56 56 + usnic_uiom_interval_tree_subtree_search(struct usnic_uiom_interval_node *node, 57 57 + unsigned long start, 58 58 + unsigned long last); 59 59 + extern struct usnic_uiom_interval_node * 56 60 usnic_uiom_interval_tree_iter_first(struct rb_root_cached *root, 57 61 unsigned long start, 58 62 unsigned long last);

+2 -2

drivers/staging/media/atomisp/i2c/atomisp-gc2235.c

reviewed

··· 491 491 return ret; 492 492 } 493 493 494 494 - static int startup(struct v4l2_subdev *sd) 494 494 + static int gc2235_startup(struct v4l2_subdev *sd) 495 495 { 496 496 struct gc2235_device *dev = to_gc2235_sensor(sd); 497 497 struct i2c_client *client = v4l2_get_subdevdata(sd); ··· 556 556 return 0; 557 557 } 558 558 559 559 - ret = startup(sd); 559 559 + ret = gc2235_startup(sd); 560 560 if (ret) { 561 561 dev_err(&client->dev, "gc2235 startup err\n"); 562 562 goto err;

+3 -3

drivers/staging/media/atomisp/i2c/atomisp-ov2722.c

reviewed

··· 600 600 } 601 601 602 602 /* TODO: remove it. */ 603 603 - static int startup(struct v4l2_subdev *sd) 603 603 + static int ov2722_startup(struct v4l2_subdev *sd) 604 604 { 605 605 struct ov2722_device *dev = to_ov2722_sensor(sd); 606 606 struct i2c_client *client = v4l2_get_subdevdata(sd); ··· 662 662 dev->pixels_per_line = dev->res->pixels_per_line; 663 663 dev->lines_per_frame = dev->res->lines_per_frame; 664 664 665 665 - ret = startup(sd); 665 665 + ret = ov2722_startup(sd); 666 666 if (ret) { 667 667 int i = 0; 668 668 ··· 677 677 dev_err(&client->dev, "power up failed, continue\n"); 678 678 continue; 679 679 } 680 680 - ret = startup(sd); 680 680 + ret = ov2722_startup(sd); 681 681 if (ret) { 682 682 dev_err(&client->dev, " startup FAILED!\n"); 683 683 } else {

+7 -7

drivers/tty/amiserial.c

reviewed

··· 438 438 * --------------------------------------------------------------- 439 439 */ 440 440 441 441 - static int startup(struct tty_struct *tty, struct serial_state *info) 441 441 + static int rs_startup(struct tty_struct *tty, struct serial_state *info) 442 442 { 443 443 struct tty_port *port = &info->tport; 444 444 unsigned long flags; ··· 513 513 * This routine will shutdown a serial port; interrupts are disabled, and 514 514 * DTR is dropped if the hangup on close termio flag is on. 515 515 */ 516 516 - static void shutdown(struct tty_struct *tty, struct serial_state *info) 516 516 + static void rs_shutdown(struct tty_struct *tty, struct serial_state *info) 517 517 { 518 518 unsigned long flags; 519 519 ··· 975 975 change_speed(tty, state, NULL); 976 976 } 977 977 } else 978 978 - retval = startup(tty, state); 978 978 + retval = rs_startup(tty, state); 979 979 tty_unlock(tty); 980 980 return retval; 981 981 } ··· 1251 1251 */ 1252 1252 rs_wait_until_sent(tty, state->timeout); 1253 1253 } 1254 1254 - shutdown(tty, state); 1254 1254 + rs_shutdown(tty, state); 1255 1255 rs_flush_buffer(tty); 1256 1256 - 1256 1256 + 1257 1257 tty_ldisc_flush(tty); 1258 1258 port->tty = NULL; 1259 1259 ··· 1325 1325 struct serial_state *info = tty->driver_data; 1326 1326 1327 1327 rs_flush_buffer(tty); 1328 1328 - shutdown(tty, info); 1328 1328 + rs_shutdown(tty, info); 1329 1329 info->tport.count = 0; 1330 1330 tty_port_set_active(&info->tport, false); 1331 1331 info->tport.tty = NULL; ··· 1349 1349 port->tty = tty; 1350 1350 tty->driver_data = info; 1351 1351 1352 1352 - retval = startup(tty, info); 1352 1352 + retval = rs_startup(tty, info); 1353 1353 if (retval) { 1354 1354 return retval; 1355 1355 }

+4 -4

drivers/tty/serial/icom.c

reviewed

··· 760 760 dma_free_coherent(&dev->dev, 4096, new_page, temp_pci); 761 761 } 762 762 763 763 - static int startup(struct icom_port *icom_port) 763 763 + static int icom_startup(struct icom_port *icom_port) 764 764 { 765 765 unsigned long temp; 766 766 unsigned char cable_id, raw_cable_id; ··· 832 832 return 0; 833 833 } 834 834 835 835 - static void shutdown(struct icom_port *icom_port) 835 835 + static void icom_shutdown(struct icom_port *icom_port) 836 836 { 837 837 unsigned long temp; 838 838 unsigned char cmdReg; ··· 1311 1311 int retval; 1312 1312 1313 1313 kref_get(&icom_port->adapter->kref); 1314 1314 - retval = startup(icom_port); 1314 1314 + retval = icom_startup(icom_port); 1315 1315 1316 1316 if (retval) { 1317 1317 kref_put(&icom_port->adapter->kref, icom_kref_release); ··· 1333 1333 cmdReg = readb(&icom_port->dram->CmdReg); 1334 1334 writeb(cmdReg & ~CMD_RCV_ENABLE, &icom_port->dram->CmdReg); 1335 1335 1336 1336 - shutdown(icom_port); 1336 1336 + icom_shutdown(icom_port); 1337 1337 1338 1338 kref_put(&icom_port->adapter->kref, icom_kref_release); 1339 1339 }

+10 -10

drivers/tty/synclink_gt.c

reviewed

··· 407 407 408 408 static void msc_set_vcr(struct slgt_info *info); 409 409 410 410 - static int startup(struct slgt_info *info); 410 410 + static int startup_hw(struct slgt_info *info); 411 411 static int block_til_ready(struct tty_struct *tty, struct file * filp,struct slgt_info *info); 412 412 - static void shutdown(struct slgt_info *info); 412 412 + static void shutdown_hw(struct slgt_info *info); 413 413 static void program_hw(struct slgt_info *info); 414 414 static void change_params(struct slgt_info *info); 415 415 ··· 622 622 623 623 if (info->port.count == 1) { 624 624 /* 1st open on this device, init hardware */ 625 625 - retval = startup(info); 625 625 + retval = startup_hw(info); 626 626 if (retval < 0) { 627 627 mutex_unlock(&info->port.mutex); 628 628 goto cleanup; ··· 666 666 flush_buffer(tty); 667 667 tty_ldisc_flush(tty); 668 668 669 669 - shutdown(info); 669 669 + shutdown_hw(info); 670 670 mutex_unlock(&info->port.mutex); 671 671 672 672 tty_port_close_end(&info->port, tty); ··· 687 687 flush_buffer(tty); 688 688 689 689 mutex_lock(&info->port.mutex); 690 690 - shutdown(info); 690 690 + shutdown_hw(info); 691 691 692 692 spin_lock_irqsave(&info->port.lock, flags); 693 693 info->port.count = 0; ··· 1445 1445 spin_unlock_irqrestore(&info->netlock, flags); 1446 1446 1447 1447 /* claim resources and init adapter */ 1448 1448 - if ((rc = startup(info)) != 0) { 1448 1448 + if ((rc = startup_hw(info)) != 0) { 1449 1449 spin_lock_irqsave(&info->netlock, flags); 1450 1450 info->netcount=0; 1451 1451 spin_unlock_irqrestore(&info->netlock, flags); ··· 1455 1455 /* generic HDLC layer open processing */ 1456 1456 rc = hdlc_open(dev); 1457 1457 if (rc) { 1458 1458 - shutdown(info); 1458 1458 + shutdown_hw(info); 1459 1459 spin_lock_irqsave(&info->netlock, flags); 1460 1460 info->netcount = 0; 1461 1461 spin_unlock_irqrestore(&info->netlock, flags); ··· 1499 1499 netif_stop_queue(dev); 1500 1500 1501 1501 /* shutdown adapter and release resources */ 1502 1502 - shutdown(info); 1502 1502 + shutdown_hw(info); 1503 1503 1504 1504 hdlc_close(dev); 1505 1505 ··· 2328 2328 return IRQ_HANDLED; 2329 2329 } 2330 2330 2331 2331 - static int startup(struct slgt_info *info) 2331 2331 + static int startup_hw(struct slgt_info *info) 2332 2332 { 2333 2333 DBGINFO(("%s startup\n", info->device_name)); 2334 2334 ··· 2361 2361 /* 2362 2362 * called by close() and hangup() to shutdown hardware 2363 2363 */ 2364 2364 - static void shutdown(struct slgt_info *info) 2364 2364 + static void shutdown_hw(struct slgt_info *info) 2365 2365 { 2366 2366 unsigned long flags; 2367 2367

+2 -2

drivers/xen/xenbus/xenbus_xs.c

reviewed

··· 410 410 return (!buffer) ? ERR_PTR(-ENOMEM) : buffer; 411 411 } 412 412 413 413 - static char **split(char *strings, unsigned int len, unsigned int *num) 413 413 + static char **split_strings(char *strings, unsigned int len, unsigned int *num) 414 414 { 415 415 char *p, **ret; 416 416 ··· 448 448 if (IS_ERR(strings)) 449 449 return ERR_CAST(strings); 450 450 451 451 - return split(strings, len, num); 451 451 + return split_strings(strings, len, num); 452 452 } 453 453 EXPORT_SYMBOL_GPL(xenbus_directory); 454 454

+46 -30

include/asm-generic/vmlinux.lds.h

reviewed

··· 87 87 #define ALIGN_FUNCTION() . = ALIGN(CONFIG_FUNCTION_ALIGNMENT) 88 88 89 89 /* 90 90 - * LD_DEAD_CODE_DATA_ELIMINATION option enables -fdata-sections, which 91 91 - * generates .data.identifier sections, which need to be pulled in with 92 92 - * .data. We don't want to pull in .data..other sections, which Linux 93 93 - * has defined. Same for text and bss. 90 90 + * Support -ffunction-sections by matching .text and .text.*, 91 91 + * but exclude '.text..*', .text.startup[.*], and .text.exit[.*]. 94 92 * 95 95 - * With LTO_CLANG, the linker also splits sections by default, so we need 96 96 - * these macros to combine the sections during the final link. 93 93 + * .text.startup and .text.startup.* are matched later by INIT_TEXT, and 94 94 + * .text.exit and .text.exit.* are matched later by EXIT_TEXT, so they must be 95 95 + * explicitly excluded here. 97 96 * 98 98 - * With AUTOFDO_CLANG and PROPELLER_CLANG, by default, the linker splits 99 99 - * text sections and regroups functions into subsections. 97 97 + * Other .text.* sections that are typically grouped separately, such as 98 98 + * .text.unlikely or .text.hot, must be matched explicitly before using 99 99 + * TEXT_MAIN. 100 100 * 101 101 - * RODATA_MAIN is not used because existing code already defines .rodata.x 102 102 - * sections to be brought in with rodata. 101 101 + * NOTE: builds *with* and *without* -ffunction-sections are both supported by 102 102 + * this single macro. Even with -ffunction-sections, there may be some objects 103 103 + * NOT compiled with the flag due to the use of a specific Makefile override 104 104 + * like cflags-y or AUTOFDO_PROFILE_foo.o. So this single catchall rule is 105 105 + * needed to support mixed object builds. 106 106 + * 107 107 + * One implication is that functions named startup(), exit(), split(), 108 108 + * unlikely(), hot(), and unknown() are not allowed in the kernel due to the 109 109 + * ambiguity of their section names with -ffunction-sections. For example, 110 110 + * .text.startup could be __attribute__((constructor)) code in a *non* 111 111 + * ffunction-sections object, which should be placed in .init.text; or it could 112 112 + * be an actual function named startup() in an ffunction-sections object, which 113 113 + * should be placed in .text. The build will detect and complain about any such 114 114 + * ambiguously named functions. 103 115 */ 104 104 - #if defined(CONFIG_LD_DEAD_CODE_DATA_ELIMINATION) || defined(CONFIG_LTO_CLANG) || \ 105 105 - defined(CONFIG_AUTOFDO_CLANG) || defined(CONFIG_PROPELLER_CLANG) 106 106 - #define TEXT_MAIN .text .text.[0-9a-zA-Z_]* 107 107 - #else 108 108 - #define TEXT_MAIN .text 109 109 - #endif 110 110 - #if defined(CONFIG_LD_DEAD_CODE_DATA_ELIMINATION) || defined(CONFIG_LTO_CLANG) 116 116 + #define TEXT_MAIN \ 117 117 + .text \ 118 118 + .text.[_0-9A-Za-df-rt-z]* \ 119 119 + .text.s[_0-9A-Za-su-z]* .text.s .text.s.* \ 120 120 + .text.st[_0-9A-Zb-z]* .text.st .text.st.* \ 121 121 + .text.sta[_0-9A-Za-qs-z]* .text.sta .text.sta.* \ 122 122 + .text.star[_0-9A-Za-su-z]* .text.star .text.star.* \ 123 123 + .text.start[_0-9A-Za-tv-z]* .text.start .text.start.* \ 124 124 + .text.startu[_0-9A-Za-oq-z]* .text.startu .text.startu.* \ 125 125 + .text.startup[_0-9A-Za-z]* \ 126 126 + .text.e[_0-9A-Za-wy-z]* .text.e .text.e.* \ 127 127 + .text.ex[_0-9A-Za-hj-z]* .text.ex .text.ex.* \ 128 128 + .text.exi[_0-9A-Za-su-z]* .text.exi .text.exi.* \ 129 129 + .text.exit[_0-9A-Za-z]* 130 130 + 131 131 + /* 132 132 + * Support -fdata-sections by matching .data, .data.*, and others, 133 133 + * but exclude '.data..*'. 134 134 + */ 111 135 #define DATA_MAIN .data .data.[0-9a-zA-Z_]* .data.rel.* .data..L* .data..compoundliteral* .data.$__unnamed_* .data.$L* 112 136 #define SDATA_MAIN .sdata .sdata.[0-9a-zA-Z_]* 113 137 #define RODATA_MAIN .rodata .rodata.[0-9a-zA-Z_]* .rodata..L* 114 138 #define BSS_MAIN .bss .bss.[0-9a-zA-Z_]* .bss..L* .bss..compoundliteral* 115 139 #define SBSS_MAIN .sbss .sbss.[0-9a-zA-Z_]* 116 116 - #else 117 117 - #define DATA_MAIN .data .data.rel .data.rel.local 118 118 - #define SDATA_MAIN .sdata 119 119 - #define RODATA_MAIN .rodata 120 120 - #define BSS_MAIN .bss 121 121 - #define SBSS_MAIN .sbss 122 122 - #endif 123 140 124 141 /* 125 142 * GCC 4.5 and later have a 32 bytes section alignment for structures. ··· 598 581 * during second ld run in second ld pass when generating System.map 599 582 * 600 583 * TEXT_MAIN here will match symbols with a fixed pattern (for example, 601 601 - * .text.hot or .text.unlikely) if dead code elimination or 602 602 - * function-section is enabled. Match these symbols first before 603 603 - * TEXT_MAIN to ensure they are grouped together. 584 584 + * .text.hot or .text.unlikely). Match those before TEXT_MAIN to ensure 585 585 + * they get grouped together. 604 586 * 605 587 * Also placing .text.hot section at the beginning of a page, this 606 588 * would help the TLB performance. ··· 745 729 746 730 #define INIT_TEXT \ 747 731 *(.init.text .init.text.*) \ 748 748 - *(.text.startup) 732 732 + *(.text.startup .text.startup.*) 749 733 750 734 #define EXIT_DATA \ 751 735 *(.exit.data .exit.data.*) \ 752 736 *(.fini_array .fini_array.*) \ 753 753 - *(.dtors .dtors.*) \ 737 737 + *(.dtors .dtors.*) 754 738 755 739 #define EXIT_TEXT \ 756 740 *(.exit.text) \ 757 757 - *(.text.exit) \ 741 741 + *(.text.exit .text.exit.*) 758 742 759 743 #define EXIT_CALL \ 760 744 *(.exitcall.exit)

+134

include/linux/annotate.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 + #ifndef _LINUX_ANNOTATE_H 3 3 + #define _LINUX_ANNOTATE_H 4 4 + 5 5 + #include <linux/objtool_types.h> 6 6 + 7 7 + #ifdef CONFIG_OBJTOOL 8 8 + 9 9 + #ifndef __ASSEMBLY__ 10 10 + 11 11 + #define __ASM_ANNOTATE(section, label, type) \ 12 12 + ".pushsection " section ",\"M\", @progbits, 8\n\t" \ 13 13 + ".long " __stringify(label) " - .\n\t" \ 14 14 + ".long " __stringify(type) "\n\t" \ 15 15 + ".popsection\n\t" 16 16 + 17 17 + #define ASM_ANNOTATE_LABEL(label, type) \ 18 18 + __ASM_ANNOTATE(".discard.annotate_insn", label, type) 19 19 + 20 20 + #define ASM_ANNOTATE(type) \ 21 21 + "911:\n\t" \ 22 22 + ASM_ANNOTATE_LABEL(911b, type) 23 23 + 24 24 + #define ASM_ANNOTATE_DATA(type) \ 25 25 + "912:\n\t" \ 26 26 + __ASM_ANNOTATE(".discard.annotate_data", 912b, type) 27 27 + 28 28 + #else /* __ASSEMBLY__ */ 29 29 + 30 30 + .macro __ANNOTATE section, type 31 31 + .Lhere_\@: 32 32 + .pushsection \section, "M", @progbits, 8 33 33 + .long .Lhere_\@ - . 34 34 + .long \type 35 35 + .popsection 36 36 + .endm 37 37 + 38 38 + .macro ANNOTATE type 39 39 + __ANNOTATE ".discard.annotate_insn", \type 40 40 + .endm 41 41 + 42 42 + .macro ANNOTATE_DATA type 43 43 + __ANNOTATE ".discard.annotate_data", \type 44 44 + .endm 45 45 + 46 46 + #endif /* __ASSEMBLY__ */ 47 47 + 48 48 + #else /* !CONFIG_OBJTOOL */ 49 49 + #ifndef __ASSEMBLY__ 50 50 + #define ASM_ANNOTATE_LABEL(label, type) "" 51 51 + #define ASM_ANNOTATE(type) 52 52 + #define ASM_ANNOTATE_DATA(type) 53 53 + #else /* __ASSEMBLY__ */ 54 54 + .macro ANNOTATE type 55 55 + .endm 56 56 + .macro ANNOTATE_DATA type 57 57 + .endm 58 58 + #endif /* __ASSEMBLY__ */ 59 59 + #endif /* !CONFIG_OBJTOOL */ 60 60 + 61 61 + #ifndef __ASSEMBLY__ 62 62 + 63 63 + /* 64 64 + * Annotate away the various 'relocation to !ENDBR` complaints; knowing that 65 65 + * these relocations will never be used for indirect calls. 66 66 + */ 67 67 + #define ANNOTATE_NOENDBR ASM_ANNOTATE(ANNOTYPE_NOENDBR) 68 68 + #define ANNOTATE_NOENDBR_SYM(sym) asm(ASM_ANNOTATE_LABEL(sym, ANNOTYPE_NOENDBR)) 69 69 + 70 70 + /* 71 71 + * This should be used immediately before an indirect jump/call. It tells 72 72 + * objtool the subsequent indirect jump/call is vouched safe for retpoline 73 73 + * builds. 74 74 + */ 75 75 + #define ANNOTATE_RETPOLINE_SAFE ASM_ANNOTATE(ANNOTYPE_RETPOLINE_SAFE) 76 76 + /* 77 77 + * See linux/instrumentation.h 78 78 + */ 79 79 + #define ANNOTATE_INSTR_BEGIN(label) ASM_ANNOTATE_LABEL(label, ANNOTYPE_INSTR_BEGIN) 80 80 + #define ANNOTATE_INSTR_END(label) ASM_ANNOTATE_LABEL(label, ANNOTYPE_INSTR_END) 81 81 + /* 82 82 + * objtool annotation to ignore the alternatives and only consider the original 83 83 + * instruction(s). 84 84 + */ 85 85 + #define ANNOTATE_IGNORE_ALTERNATIVE ASM_ANNOTATE(ANNOTYPE_IGNORE_ALTS) 86 86 + /* 87 87 + * This macro indicates that the following intra-function call is valid. 88 88 + * Any non-annotated intra-function call will cause objtool to issue a warning. 89 89 + */ 90 90 + #define ANNOTATE_INTRA_FUNCTION_CALL ASM_ANNOTATE(ANNOTYPE_INTRA_FUNCTION_CALL) 91 91 + /* 92 92 + * Use objtool to validate the entry requirement that all code paths do 93 93 + * VALIDATE_UNRET_END before RET. 94 94 + * 95 95 + * NOTE: The macro must be used at the beginning of a global symbol, otherwise 96 96 + * it will be ignored. 97 97 + */ 98 98 + #define ANNOTATE_UNRET_BEGIN ASM_ANNOTATE(ANNOTYPE_UNRET_BEGIN) 99 99 + /* 100 100 + * This should be used to refer to an instruction that is considered 101 101 + * terminating, like a noreturn CALL or UD2 when we know they are not -- eg 102 102 + * WARN using UD2. 103 103 + */ 104 104 + #define ANNOTATE_REACHABLE(label) ASM_ANNOTATE_LABEL(label, ANNOTYPE_REACHABLE) 105 105 + /* 106 106 + * This should not be used; it annotates away CFI violations. There are a few 107 107 + * valid use cases like kexec handover to the next kernel image, and there is 108 108 + * no security concern there. 109 109 + * 110 110 + * There are also a few real issues annotated away, like EFI because we can't 111 111 + * control the EFI code. 112 112 + */ 113 113 + #define ANNOTATE_NOCFI_SYM(sym) asm(ASM_ANNOTATE_LABEL(sym, ANNOTYPE_NOCFI)) 114 114 + 115 115 + /* 116 116 + * Annotate a special section entry. This emables livepatch module generation 117 117 + * to find and extract individual special section entries as needed. 118 118 + */ 119 119 + #define ANNOTATE_DATA_SPECIAL ASM_ANNOTATE_DATA(ANNOTYPE_DATA_SPECIAL) 120 120 + 121 121 + #else /* __ASSEMBLY__ */ 122 122 + #define ANNOTATE_NOENDBR ANNOTATE type=ANNOTYPE_NOENDBR 123 123 + #define ANNOTATE_RETPOLINE_SAFE ANNOTATE type=ANNOTYPE_RETPOLINE_SAFE 124 124 + /* ANNOTATE_INSTR_BEGIN ANNOTATE type=ANNOTYPE_INSTR_BEGIN */ 125 125 + /* ANNOTATE_INSTR_END ANNOTATE type=ANNOTYPE_INSTR_END */ 126 126 + #define ANNOTATE_IGNORE_ALTERNATIVE ANNOTATE type=ANNOTYPE_IGNORE_ALTS 127 127 + #define ANNOTATE_INTRA_FUNCTION_CALL ANNOTATE type=ANNOTYPE_INTRA_FUNCTION_CALL 128 128 + #define ANNOTATE_UNRET_BEGIN ANNOTATE type=ANNOTYPE_UNRET_BEGIN 129 129 + #define ANNOTATE_REACHABLE ANNOTATE type=ANNOTYPE_REACHABLE 130 130 + #define ANNOTATE_NOCFI_SYM ANNOTATE type=ANNOTYPE_NOCFI 131 131 + #define ANNOTATE_DATA_SPECIAL ANNOTATE_DATA type=ANNOTYPE_DATA_SPECIAL 132 132 + #endif /* __ASSEMBLY__ */ 133 133 + 134 134 + #endif /* _LINUX_ANNOTATE_H */

+6 -2

include/linux/compiler.h

reviewed

··· 163 163 __asm__ ("" : "=r" (var) : "0" (var)) 164 164 #endif 165 165 166 166 - #define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __COUNTER__) 166 166 + /* Format: __UNIQUE_ID_<name>_<__COUNTER__> */ 167 167 + #define __UNIQUE_ID(name) \ 168 168 + __PASTE(__UNIQUE_ID_, \ 169 169 + __PASTE(name, \ 170 170 + __PASTE(_, __COUNTER__))) 167 171 168 172 /** 169 173 * data_race - mark an expression as containing intentional data races ··· 287 283 */ 288 284 #define ___ADDRESSABLE(sym, __attrs) \ 289 285 static void * __used __attrs \ 290 290 - __UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)(uintptr_t)&sym; 286 286 + __UNIQUE_ID(__PASTE(addressable_, sym)) = (void *)(uintptr_t)&sym; 291 287 292 288 #define __ADDRESSABLE(sym) \ 293 289 ___ADDRESSABLE(sym, __section(".discard.addressable"))

+5 -8

include/linux/elfnote.h

reviewed

··· 60 60 61 61 #else /* !__ASSEMBLER__ */ 62 62 #include <uapi/linux/elf.h> 63 63 + #include <linux/compiler.h> 63 64 /* 64 65 * Use an anonymous structure which matches the shape of 65 66 * Elf{32,64}_Nhdr, but includes the name and desc data. The size and 66 67 * type of name and desc depend on the macro arguments. "name" must 67 67 - * be a literal string, and "desc" must be passed by value. You may 68 68 - * only define one note per line, since __LINE__ is used to generate 69 69 - * unique symbols. 68 68 + * be a literal string, and "desc" must be passed by value. 70 69 */ 71 71 - #define _ELFNOTE_PASTE(a,b) a##b 72 72 - #define _ELFNOTE(size, name, unique, type, desc) \ 70 70 + #define ELFNOTE(size, name, type, desc) \ 73 71 static const struct { \ 74 72 struct elf##size##_note _nhdr; \ 75 73 unsigned char _name[sizeof(name)] \ 76 74 __attribute__((aligned(sizeof(Elf##size##_Word)))); \ 77 75 typeof(desc) _desc \ 78 76 __attribute__((aligned(sizeof(Elf##size##_Word)))); \ 79 79 - } _ELFNOTE_PASTE(_note_, unique) \ 77 77 + } __UNIQUE_ID(note) \ 80 78 __used \ 81 79 __attribute__((section(".note." name), \ 82 80 aligned(sizeof(Elf##size##_Word)), \ ··· 87 89 name, \ 88 90 desc \ 89 91 } 90 90 - #define ELFNOTE(size, name, type, desc) \ 91 91 - _ELFNOTE(size, name, __LINE__, type, desc) 92 92 93 93 #define ELFNOTE32(name, type, desc) ELFNOTE(32, name, type, desc) 94 94 #define ELFNOTE64(name, type, desc) ELFNOTE(64, name, type, desc) 95 95 + 95 96 #endif /* __ASSEMBLER__ */ 96 97 97 98 #endif /* _LINUX_ELFNOTE_H */

+2 -1

include/linux/init.h

reviewed

··· 200 200 201 201 /* Format: <modname>__<counter>_<line>_<fn> */ 202 202 #define __initcall_id(fn) \ 203 203 + __PASTE(kmod_, \ 203 204 __PASTE(__KBUILD_MODNAME, \ 204 205 __PASTE(__, \ 205 206 __PASTE(__COUNTER__, \ 206 207 __PASTE(_, \ 207 208 __PASTE(__LINE__, \ 208 208 - __PASTE(_, fn)))))) 209 209 + __PASTE(_, fn))))))) 209 210 210 211 /* Format: __<prefix>__<iid><id> */ 211 212 #define __initcall_name(prefix, __iid, id) \

+4

include/linux/interval_tree.h

reviewed

··· 20 20 struct rb_root_cached *root); 21 21 22 22 extern struct interval_tree_node * 23 23 + interval_tree_subtree_search(struct interval_tree_node *node, 24 24 + unsigned long start, unsigned long last); 25 25 + 26 26 + extern struct interval_tree_node * 23 27 interval_tree_iter_first(struct rb_root_cached *root, 24 28 unsigned long start, unsigned long last); 25 29

+1 -1

include/linux/interval_tree_generic.h

reviewed

··· 77 77 * Cond2: start <= ITLAST(node) \ 78 78 */ \ 79 79 \ 80 80 - static ITSTRUCT * \ 80 80 + ITSTATIC ITSTRUCT * \ 81 81 ITPREFIX ## _subtree_search(ITSTRUCT *node, ITTYPE start, ITTYPE last) \ 82 82 { \ 83 83 while (true) { \

+1 -24

include/linux/livepatch.h

reviewed

··· 13 13 #include <linux/ftrace.h> 14 14 #include <linux/completion.h> 15 15 #include <linux/list.h> 16 16 + #include <linux/livepatch_external.h> 16 17 #include <linux/livepatch_sched.h> 17 18 18 19 #if IS_ENABLED(CONFIG_LIVEPATCH) ··· 76 75 bool nop; 77 76 bool patched; 78 77 bool transition; 79 79 - }; 80 80 - 81 81 - struct klp_object; 82 82 - 83 83 - /** 84 84 - * struct klp_callbacks - pre/post live-(un)patch callback structure 85 85 - * @pre_patch: executed before code patching 86 86 - * @post_patch: executed after code patching 87 87 - * @pre_unpatch: executed before code unpatching 88 88 - * @post_unpatch: executed after code unpatching 89 89 - * @post_unpatch_enabled: flag indicating if post-unpatch callback 90 90 - * should run 91 91 - * 92 92 - * All callbacks are optional. Only the pre-patch callback, if provided, 93 93 - * will be unconditionally executed. If the parent klp_object fails to 94 94 - * patch for any reason, including a non-zero error status returned from 95 95 - * the pre-patch callback, no further callbacks will be executed. 96 96 - */ 97 97 - struct klp_callbacks { 98 98 - int (*pre_patch)(struct klp_object *obj); 99 99 - void (*post_patch)(struct klp_object *obj); 100 100 - void (*pre_unpatch)(struct klp_object *obj); 101 101 - void (*post_unpatch)(struct klp_object *obj); 102 102 - bool post_unpatch_enabled; 103 78 }; 104 79 105 80 /**

+76

include/linux/livepatch_external.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 + /* 3 3 + * External livepatch interfaces for patch creation tooling 4 4 + */ 5 5 + 6 6 + #ifndef _LINUX_LIVEPATCH_EXTERNAL_H_ 7 7 + #define _LINUX_LIVEPATCH_EXTERNAL_H_ 8 8 + 9 9 + #include <linux/types.h> 10 10 + 11 11 + #define KLP_RELOC_SEC_PREFIX ".klp.rela." 12 12 + #define KLP_SYM_PREFIX ".klp.sym." 13 13 + 14 14 + #define __KLP_PRE_PATCH_PREFIX __klp_pre_patch_callback_ 15 15 + #define __KLP_POST_PATCH_PREFIX __klp_post_patch_callback_ 16 16 + #define __KLP_PRE_UNPATCH_PREFIX __klp_pre_unpatch_callback_ 17 17 + #define __KLP_POST_UNPATCH_PREFIX __klp_post_unpatch_callback_ 18 18 + 19 19 + #define KLP_PRE_PATCH_PREFIX __stringify(__KLP_PRE_PATCH_PREFIX) 20 20 + #define KLP_POST_PATCH_PREFIX __stringify(__KLP_POST_PATCH_PREFIX) 21 21 + #define KLP_PRE_UNPATCH_PREFIX __stringify(__KLP_PRE_UNPATCH_PREFIX) 22 22 + #define KLP_POST_UNPATCH_PREFIX __stringify(__KLP_POST_UNPATCH_PREFIX) 23 23 + 24 24 + struct klp_object; 25 25 + 26 26 + typedef int (*klp_pre_patch_t)(struct klp_object *obj); 27 27 + typedef void (*klp_post_patch_t)(struct klp_object *obj); 28 28 + typedef void (*klp_pre_unpatch_t)(struct klp_object *obj); 29 29 + typedef void (*klp_post_unpatch_t)(struct klp_object *obj); 30 30 + 31 31 + /** 32 32 + * struct klp_callbacks - pre/post live-(un)patch callback structure 33 33 + * @pre_patch: executed before code patching 34 34 + * @post_patch: executed after code patching 35 35 + * @pre_unpatch: executed before code unpatching 36 36 + * @post_unpatch: executed after code unpatching 37 37 + * @post_unpatch_enabled: flag indicating if post-unpatch callback 38 38 + * should run 39 39 + * 40 40 + * All callbacks are optional. Only the pre-patch callback, if provided, 41 41 + * will be unconditionally executed. If the parent klp_object fails to 42 42 + * patch for any reason, including a non-zero error status returned from 43 43 + * the pre-patch callback, no further callbacks will be executed. 44 44 + */ 45 45 + struct klp_callbacks { 46 46 + klp_pre_patch_t pre_patch; 47 47 + klp_post_patch_t post_patch; 48 48 + klp_pre_unpatch_t pre_unpatch; 49 49 + klp_post_unpatch_t post_unpatch; 50 50 + bool post_unpatch_enabled; 51 51 + }; 52 52 + 53 53 + /* 54 54 + * 'struct klp_{func,object}_ext' are compact "external" representations of 55 55 + * 'struct klp_{func,object}'. They are used by objtool for livepatch 56 56 + * generation. The structs are then read by the livepatch module and converted 57 57 + * to the real structs before calling klp_enable_patch(). 58 58 + * 59 59 + * TODO make these the official API for klp_enable_patch(). That should 60 60 + * simplify livepatch's interface as well as its data structure lifetime 61 61 + * management. 62 62 + */ 63 63 + struct klp_func_ext { 64 64 + const char *old_name; 65 65 + void *new_func; 66 66 + unsigned long sympos; 67 67 + }; 68 68 + 69 69 + struct klp_object_ext { 70 70 + const char *name; 71 71 + struct klp_func_ext *funcs; 72 72 + struct klp_callbacks callbacks; 73 73 + unsigned int nr_funcs; 74 74 + }; 75 75 + 76 76 + #endif /* _LINUX_LIVEPATCH_EXTERNAL_H_ */

+77

include/linux/livepatch_helpers.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 + #ifndef _LINUX_LIVEPATCH_HELPERS_H 3 3 + #define _LINUX_LIVEPATCH_HELPERS_H 4 4 + 5 5 + /* 6 6 + * Interfaces for use by livepatch patches 7 7 + */ 8 8 + 9 9 + #include <linux/syscalls.h> 10 10 + #include <linux/livepatch.h> 11 11 + 12 12 + #ifdef MODULE 13 13 + #define KLP_OBJNAME __KBUILD_MODNAME 14 14 + #else 15 15 + #define KLP_OBJNAME vmlinux 16 16 + #endif 17 17 + 18 18 + /* Livepatch callback registration */ 19 19 + 20 20 + #define KLP_CALLBACK_PTRS ".discard.klp_callback_ptrs" 21 21 + 22 22 + #define KLP_PRE_PATCH_CALLBACK(func) \ 23 23 + klp_pre_patch_t __used __section(KLP_CALLBACK_PTRS) \ 24 24 + __PASTE(__KLP_PRE_PATCH_PREFIX, KLP_OBJNAME) = func 25 25 + 26 26 + #define KLP_POST_PATCH_CALLBACK(func) \ 27 27 + klp_post_patch_t __used __section(KLP_CALLBACK_PTRS) \ 28 28 + __PASTE(__KLP_POST_PATCH_PREFIX, KLP_OBJNAME) = func 29 29 + 30 30 + #define KLP_PRE_UNPATCH_CALLBACK(func) \ 31 31 + klp_pre_unpatch_t __used __section(KLP_CALLBACK_PTRS) \ 32 32 + __PASTE(__KLP_PRE_UNPATCH_PREFIX, KLP_OBJNAME) = func 33 33 + 34 34 + #define KLP_POST_UNPATCH_CALLBACK(func) \ 35 35 + klp_post_unpatch_t __used __section(KLP_CALLBACK_PTRS) \ 36 36 + __PASTE(__KLP_POST_UNPATCH_PREFIX, KLP_OBJNAME) = func 37 37 + 38 38 + /* 39 39 + * Replace static_call() usage with this macro when create-diff-object 40 40 + * recommends it due to the original static call key living in a module. 41 41 + * 42 42 + * This converts the static call to a regular indirect call. 43 43 + */ 44 44 + #define KLP_STATIC_CALL(name) \ 45 45 + ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func)) 46 46 + 47 47 + /* Syscall patching */ 48 48 + 49 49 + #define KLP_SYSCALL_DEFINE1(name, ...) KLP_SYSCALL_DEFINEx(1, _##name, __VA_ARGS__) 50 50 + #define KLP_SYSCALL_DEFINE2(name, ...) KLP_SYSCALL_DEFINEx(2, _##name, __VA_ARGS__) 51 51 + #define KLP_SYSCALL_DEFINE3(name, ...) KLP_SYSCALL_DEFINEx(3, _##name, __VA_ARGS__) 52 52 + #define KLP_SYSCALL_DEFINE4(name, ...) KLP_SYSCALL_DEFINEx(4, _##name, __VA_ARGS__) 53 53 + #define KLP_SYSCALL_DEFINE5(name, ...) KLP_SYSCALL_DEFINEx(5, _##name, __VA_ARGS__) 54 54 + #define KLP_SYSCALL_DEFINE6(name, ...) KLP_SYSCALL_DEFINEx(6, _##name, __VA_ARGS__) 55 55 + 56 56 + #define KLP_SYSCALL_DEFINEx(x, sname, ...) \ 57 57 + __KLP_SYSCALL_DEFINEx(x, sname, __VA_ARGS__) 58 58 + 59 59 + #ifdef CONFIG_X86_64 60 60 + // TODO move this to arch/x86/include/asm/syscall_wrapper.h and share code 61 61 + #define __KLP_SYSCALL_DEFINEx(x, name, ...) \ 62 62 + static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \ 63 63 + static inline long __klp_do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__));\ 64 64 + __X64_SYS_STUBx(x, name, __VA_ARGS__) \ 65 65 + __IA32_SYS_STUBx(x, name, __VA_ARGS__) \ 66 66 + static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \ 67 67 + { \ 68 68 + long ret = __klp_do_sys##name(__MAP(x,__SC_CAST,__VA_ARGS__));\ 69 69 + __MAP(x,__SC_TEST,__VA_ARGS__); \ 70 70 + __PROTECT(x, ret,__MAP(x,__SC_ARGS,__VA_ARGS__)); \ 71 71 + return ret; \ 72 72 + } \ 73 73 + static inline long __klp_do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) 74 74 + 75 75 + #endif 76 76 + 77 77 + #endif /* _LINUX_LIVEPATCH_HELPERS_H */

+2

include/linux/mm.h

reviewed

··· 3376 3376 struct rb_root_cached *root); 3377 3377 void vma_interval_tree_remove(struct vm_area_struct *node, 3378 3378 struct rb_root_cached *root); 3379 3379 + struct vm_area_struct *vma_interval_tree_subtree_search(struct vm_area_struct *node, 3380 3380 + unsigned long start, unsigned long last); 3379 3381 struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root, 3380 3382 unsigned long start, unsigned long last); 3381 3383 struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,

+2 -1

include/linux/module.h

reviewed

··· 251 251 */ 252 252 #define __mod_device_table(type, name) \ 253 253 __PASTE(__mod_device_table__, \ 254 254 + __PASTE(kmod_, \ 254 255 __PASTE(__KBUILD_MODNAME, \ 255 256 __PASTE(__, \ 256 257 __PASTE(type, \ 257 257 - __PASTE(__, name))))) 258 258 + __PASTE(__, name)))))) 258 259 259 260 /* Creates an alias so file2alias.c can find device table. */ 260 261 #define MODULE_DEVICE_TABLE(type, name) \

+5 -91

include/linux/objtool.h

reviewed

··· 3 3 #define _LINUX_OBJTOOL_H 4 4 5 5 #include <linux/objtool_types.h> 6 6 + #include <linux/annotate.h> 6 7 7 8 #ifdef CONFIG_OBJTOOL 8 9 9 9 - #include <asm/asm.h> 10 10 - 11 10 #ifndef __ASSEMBLY__ 12 11 13 13 - #define UNWIND_HINT(type, sp_reg, sp_offset, signal) \ 12 12 + #define UNWIND_HINT(type, sp_reg, sp_offset, signal) \ 14 13 "987: \n\t" \ 15 14 ".pushsection .discard.unwind_hints\n\t" \ 15 15 + ANNOTATE_DATA_SPECIAL \ 16 16 /* struct unwind_hint */ \ 17 17 ".long 987b - .\n\t" \ 18 18 ".short " __stringify(sp_offset) "\n\t" \ ··· 53 53 54 54 #define __ASM_BREF(label) label ## b 55 55 56 56 - #define __ASM_ANNOTATE(label, type) \ 57 57 - ".pushsection .discard.annotate_insn,\"M\",@progbits,8\n\t" \ 58 58 - ".long " __stringify(label) " - .\n\t" \ 59 59 - ".long " __stringify(type) "\n\t" \ 60 60 - ".popsection\n\t" 61 61 - 62 62 - #define ASM_ANNOTATE(type) \ 63 63 - "911:\n\t" \ 64 64 - __ASM_ANNOTATE(911b, type) 65 65 - 66 56 #else /* __ASSEMBLY__ */ 67 57 68 58 /* ··· 79 89 .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 80 90 .Lhere_\@: 81 91 .pushsection .discard.unwind_hints 92 92 + ANNOTATE_DATA_SPECIAL 82 93 /* struct unwind_hint */ 83 94 .long .Lhere_\@ - . 84 95 .short \sp_offset ··· 92 101 93 102 .macro STACK_FRAME_NON_STANDARD func:req 94 103 .pushsection .discard.func_stack_frame_non_standard, "aw" 95 95 - .long \func - . 104 104 + .quad \func 96 105 .popsection 97 106 .endm 98 107 ··· 100 109 #ifdef CONFIG_FRAME_POINTER 101 110 STACK_FRAME_NON_STANDARD \func 102 111 #endif 103 103 - .endm 104 104 - 105 105 - .macro ANNOTATE type:req 106 106 - .Lhere_\@: 107 107 - .pushsection .discard.annotate_insn,"M",@progbits,8 108 108 - .long .Lhere_\@ - . 109 109 - .long \type 110 110 - .popsection 111 112 .endm 112 113 113 114 #endif /* __ASSEMBLY__ */ ··· 111 128 #define UNWIND_HINT(type, sp_reg, sp_offset, signal) "\n\t" 112 129 #define STACK_FRAME_NON_STANDARD(func) 113 130 #define STACK_FRAME_NON_STANDARD_FP(func) 114 114 - #define __ASM_ANNOTATE(label, type) "" 115 115 - #define ASM_ANNOTATE(type) 116 131 #else 117 132 .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 118 133 .endm 119 134 .macro STACK_FRAME_NON_STANDARD func:req 120 135 .endm 121 121 - .macro ANNOTATE type:req 122 122 - .endm 123 136 #endif 124 137 125 138 #endif /* CONFIG_OBJTOOL */ 126 126 - 127 127 - #ifndef __ASSEMBLY__ 128 128 - /* 129 129 - * Annotate away the various 'relocation to !ENDBR` complaints; knowing that 130 130 - * these relocations will never be used for indirect calls. 131 131 - */ 132 132 - #define ANNOTATE_NOENDBR ASM_ANNOTATE(ANNOTYPE_NOENDBR) 133 133 - #define ANNOTATE_NOENDBR_SYM(sym) asm(__ASM_ANNOTATE(sym, ANNOTYPE_NOENDBR)) 134 134 - 135 135 - /* 136 136 - * This should be used immediately before an indirect jump/call. It tells 137 137 - * objtool the subsequent indirect jump/call is vouched safe for retpoline 138 138 - * builds. 139 139 - */ 140 140 - #define ANNOTATE_RETPOLINE_SAFE ASM_ANNOTATE(ANNOTYPE_RETPOLINE_SAFE) 141 141 - /* 142 142 - * See linux/instrumentation.h 143 143 - */ 144 144 - #define ANNOTATE_INSTR_BEGIN(label) __ASM_ANNOTATE(label, ANNOTYPE_INSTR_BEGIN) 145 145 - #define ANNOTATE_INSTR_END(label) __ASM_ANNOTATE(label, ANNOTYPE_INSTR_END) 146 146 - /* 147 147 - * objtool annotation to ignore the alternatives and only consider the original 148 148 - * instruction(s). 149 149 - */ 150 150 - #define ANNOTATE_IGNORE_ALTERNATIVE ASM_ANNOTATE(ANNOTYPE_IGNORE_ALTS) 151 151 - /* 152 152 - * This macro indicates that the following intra-function call is valid. 153 153 - * Any non-annotated intra-function call will cause objtool to issue a warning. 154 154 - */ 155 155 - #define ANNOTATE_INTRA_FUNCTION_CALL ASM_ANNOTATE(ANNOTYPE_INTRA_FUNCTION_CALL) 156 156 - /* 157 157 - * Use objtool to validate the entry requirement that all code paths do 158 158 - * VALIDATE_UNRET_END before RET. 159 159 - * 160 160 - * NOTE: The macro must be used at the beginning of a global symbol, otherwise 161 161 - * it will be ignored. 162 162 - */ 163 163 - #define ANNOTATE_UNRET_BEGIN ASM_ANNOTATE(ANNOTYPE_UNRET_BEGIN) 164 164 - /* 165 165 - * This should be used to refer to an instruction that is considered 166 166 - * terminating, like a noreturn CALL or UD2 when we know they are not -- eg 167 167 - * WARN using UD2. 168 168 - */ 169 169 - #define ANNOTATE_REACHABLE(label) __ASM_ANNOTATE(label, ANNOTYPE_REACHABLE) 170 170 - /* 171 171 - * This should not be used; it annotates away CFI violations. There are a few 172 172 - * valid use cases like kexec handover to the next kernel image, and there is 173 173 - * no security concern there. 174 174 - * 175 175 - * There are also a few real issues annotated away, like EFI because we can't 176 176 - * control the EFI code. 177 177 - */ 178 178 - #define ANNOTATE_NOCFI_SYM(sym) asm(__ASM_ANNOTATE(sym, ANNOTYPE_NOCFI)) 179 179 - 180 180 - #else 181 181 - #define ANNOTATE_NOENDBR ANNOTATE type=ANNOTYPE_NOENDBR 182 182 - #define ANNOTATE_RETPOLINE_SAFE ANNOTATE type=ANNOTYPE_RETPOLINE_SAFE 183 183 - /* ANNOTATE_INSTR_BEGIN ANNOTATE type=ANNOTYPE_INSTR_BEGIN */ 184 184 - /* ANNOTATE_INSTR_END ANNOTATE type=ANNOTYPE_INSTR_END */ 185 185 - #define ANNOTATE_IGNORE_ALTERNATIVE ANNOTATE type=ANNOTYPE_IGNORE_ALTS 186 186 - #define ANNOTATE_INTRA_FUNCTION_CALL ANNOTATE type=ANNOTYPE_INTRA_FUNCTION_CALL 187 187 - #define ANNOTATE_UNRET_BEGIN ANNOTATE type=ANNOTYPE_UNRET_BEGIN 188 188 - #define ANNOTATE_REACHABLE ANNOTATE type=ANNOTYPE_REACHABLE 189 189 - #define ANNOTATE_NOCFI_SYM ANNOTATE type=ANNOTYPE_NOCFI 190 190 - #endif 191 139 192 140 #if defined(CONFIG_NOINSTR_VALIDATION) && \ 193 141 (defined(CONFIG_MITIGATION_UNRET_ENTRY) || defined(CONFIG_MITIGATION_SRSO))

+2

include/linux/objtool_types.h

reviewed

··· 67 67 #define ANNOTYPE_REACHABLE 8 68 68 #define ANNOTYPE_NOCFI 9 69 69 70 70 + #define ANNOTYPE_DATA_SPECIAL 1 71 71 + 70 72 #endif /* _LINUX_OBJTOOL_TYPES_H */

+12

kernel/livepatch/Kconfig

reviewed

··· 18 18 module uses the interface provided by this option to register 19 19 a patch, causing calls to patched functions to be redirected 20 20 to new function code contained in the patch module. 21 21 + 22 22 + config HAVE_KLP_BUILD 23 23 + bool 24 24 + help 25 25 + Arch supports klp-build 26 26 + 27 27 + config KLP_BUILD 28 28 + def_bool y 29 29 + depends on LIVEPATCH && HAVE_KLP_BUILD 30 30 + select OBJTOOL 31 31 + help 32 32 + Enable klp-build support

+4 -4

kernel/livepatch/core.c

reviewed

··· 217 217 for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) { 218 218 sym = (Elf_Sym *)sechdrs[symndx].sh_addr + ELF_R_SYM(relas[i].r_info); 219 219 if (sym->st_shndx != SHN_LIVEPATCH) { 220 220 - pr_err("symbol %s is not marked as a livepatch symbol\n", 221 221 - strtab + sym->st_name); 220 220 + pr_err("symbol %s at rela sec %u idx %d is not marked as a livepatch symbol\n", 221 221 + strtab + sym->st_name, symndx, i); 222 222 return -EINVAL; 223 223 } 224 224 225 225 /* Format: .klp.sym.sym_objname.sym_name,sympos */ 226 226 cnt = sscanf(strtab + sym->st_name, 227 227 - ".klp.sym.%55[^.].%511[^,],%lu", 227 227 + KLP_SYM_PREFIX "%55[^.].%511[^,],%lu", 228 228 sym_objname, sym_name, &sympos); 229 229 if (cnt != 3) { 230 230 pr_err("symbol %s has an incorrectly formatted name\n", ··· 303 303 * See comment in klp_resolve_symbols() for an explanation 304 304 * of the selected field width value. 305 305 */ 306 306 - cnt = sscanf(shstrtab + sec->sh_name, ".klp.rela.%55[^.]", 306 306 + cnt = sscanf(shstrtab + sec->sh_name, KLP_RELOC_SEC_PREFIX "%55[^.]", 307 307 sec_objname); 308 308 if (cnt != 1) { 309 309 pr_err("section %s has an incorrectly formatted name\n",

+1

lib/interval_tree.c

reviewed

··· 13 13 14 14 EXPORT_SYMBOL_GPL(interval_tree_insert); 15 15 EXPORT_SYMBOL_GPL(interval_tree_remove); 16 16 + EXPORT_SYMBOL_GPL(interval_tree_subtree_search); 16 17 EXPORT_SYMBOL_GPL(interval_tree_iter_first); 17 18 EXPORT_SYMBOL_GPL(interval_tree_iter_next); 18 19

+3 -3

scripts/Makefile.lib

reviewed

··· 20 20 name-fix = $(call stringify,$(call name-fix-token,$1)) 21 21 basename_flags = -DKBUILD_BASENAME=$(call name-fix,$(basetarget)) 22 22 modname_flags = -DKBUILD_MODNAME=$(call name-fix,$(modname)) \ 23 23 - -D__KBUILD_MODNAME=kmod_$(call name-fix-token,$(modname)) 23 23 + -D__KBUILD_MODNAME=$(call name-fix-token,$(modname)) 24 24 modfile_flags = -DKBUILD_MODFILE=$(call stringify,$(modfile)) 25 25 26 26 _c_flags = $(filter-out $(CFLAGS_REMOVE_$(target-stem).o), \ ··· 191 191 objtool-args-$(CONFIG_HAVE_UACCESS_VALIDATION) += --uaccess 192 192 objtool-args-$(or $(CONFIG_GCOV_KERNEL),$(CONFIG_KCOV)) += --no-unreachable 193 193 objtool-args-$(CONFIG_PREFIX_SYMBOLS) += --prefix=$(CONFIG_FUNCTION_PADDING_BYTES) 194 194 - objtool-args-$(CONFIG_OBJTOOL_WERROR) += --Werror 194 194 + objtool-args-$(CONFIG_OBJTOOL_WERROR) += --werror 195 195 196 196 objtool-args = $(objtool-args-y) \ 197 197 $(if $(delay-objtool), --link) \ 198 198 $(if $(part-of-module), --module) 199 199 200 200 - delay-objtool := $(or $(CONFIG_LTO_CLANG),$(CONFIG_X86_KERNEL_IBT)) 200 200 + delay-objtool := $(or $(CONFIG_LTO_CLANG),$(CONFIG_X86_KERNEL_IBT),$(CONFIG_KLP_BUILD)) 201 201 202 202 cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool-args) $@) 203 203 cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(objtool))' ; } >> $(dot-target).cmd)

+5 -1

scripts/Makefile.vmlinux_o

reviewed

··· 41 41 ifeq ($(delay-objtool),y) 42 42 vmlinux-objtool-args-y += $(objtool-args-y) 43 43 else 44 44 - vmlinux-objtool-args-$(CONFIG_OBJTOOL_WERROR) += --Werror 44 44 + vmlinux-objtool-args-$(CONFIG_OBJTOOL_WERROR) += --werror 45 45 endif 46 46 47 47 vmlinux-objtool-args-$(CONFIG_NOINSTR_VALIDATION) += --noinstr \ ··· 63 63 --start-group $(KBUILD_VMLINUX_LIBS) --end-group \ 64 64 $(cmd_objtool) 65 65 66 66 + cmd_check_function_names = $(srctree)/scripts/check-function-names.sh $@ 67 67 + 66 68 define rule_ld_vmlinux.o 67 69 $(call cmd_and_savecmd,ld_vmlinux.o) 68 70 $(call cmd,gen_objtooldep) 71 71 + $(call cmd,check_function_names) 69 72 endef 73 73 + 70 74 71 75 vmlinux.o: $(initcalls-lds) vmlinux.a $(KBUILD_VMLINUX_LIBS) FORCE 72 76 $(call if_changed_rule,ld_vmlinux.o)

+25

scripts/check-function-names.sh

reviewed

··· 1 1 + #!/bin/sh 2 2 + # SPDX-License-Identifier: GPL-2.0 3 3 + # 4 4 + # Certain function names are disallowed due to section name ambiguities 5 5 + # introduced by -ffunction-sections. 6 6 + # 7 7 + # See the comment above TEXT_MAIN in include/asm-generic/vmlinux.lds.h. 8 8 + 9 9 + objfile="$1" 10 10 + 11 11 + if [ ! -f "$objfile" ]; then 12 12 + echo "usage: $0 <file.o>" >&2 13 13 + exit 1 14 14 + fi 15 15 + 16 16 + bad_symbols=$(nm "$objfile" | awk '$2 ~ /^[TtWw]$/ {print $3}' | grep -E '^(startup|exit|split|unlikely|hot|unknown)(\.|$)') 17 17 + 18 18 + if [ -n "$bad_symbols" ]; then 19 19 + echo "$bad_symbols" | while read -r sym; do 20 20 + echo "$objfile: error: $sym() function name creates ambiguity with -ffunction-sections" >&2 21 21 + done 22 22 + exit 1 23 23 + fi 24 24 + 25 25 + exit 0

+14 -5

scripts/faddr2line

reviewed

··· 1 1 - #!/bin/bash 1 1 + #!/usr/bin/env bash 2 2 # SPDX-License-Identifier: GPL-2.0 3 3 # 4 4 # Translate stack dump function offsets. ··· 76 76 AWK="awk" 77 77 GREP="grep" 78 78 79 79 + # Enforce ASCII-only output from tools like readelf 80 80 + # ensuring sed processes strings correctly. 81 81 + export LANG=C 82 82 + 79 83 command -v ${AWK} >/dev/null 2>&1 || die "${AWK} isn't installed" 80 84 command -v ${READELF} >/dev/null 2>&1 || die "${READELF} isn't installed" 81 85 command -v ${ADDR2LINE} >/dev/null 2>&1 || die "${ADDR2LINE} isn't installed" ··· 111 107 112 108 run_readelf() { 113 109 local objfile=$1 114 114 - local out=$(${READELF} --file-header --section-headers --symbols --wide $objfile) 110 110 + local tmpfile 111 111 + tmpfile=$(mktemp) 112 112 + 113 113 + ${READELF} --file-header --section-headers --symbols --wide "$objfile" > "$tmpfile" 115 114 116 115 # This assumes that readelf first prints the file header, then the section headers, then the symbols. 117 116 # Note: It seems that GNU readelf does not prefix section headers with the "There are X section headers" 118 117 # line when multiple options are given, so let's also match with the "Section Headers:" line. 119 119 - ELF_FILEHEADER=$(echo "${out}" | sed -n '/There are [0-9]* section headers, starting at offset\|Section Headers:/q;p') 120 120 - ELF_SECHEADERS=$(echo "${out}" | sed -n '/There are [0-9]* section headers, starting at offset\|Section Headers:/,$p' | sed -n '/Symbol table .* contains [0-9]* entries:/q;p') 121 121 - ELF_SYMS=$(echo "${out}" | sed -n '/Symbol table .* contains [0-9]* entries:/,$p') 118 118 + ELF_FILEHEADER=$(sed -n '/There are [0-9]* section headers, starting at offset\|Section Headers:/q;p' "$tmpfile") 119 119 + ELF_SECHEADERS=$(sed -n '/There are [0-9]* section headers, starting at offset\|Section Headers:/,$p' "$tmpfile" | sed -n '/Symbol table .* contains [0-9]* entries:/q;p') 120 120 + ELF_SYMS=$(sed -n '/Symbol table .* contains [0-9]* entries:/,$p' "$tmpfile") 121 121 + 122 122 + rm -f -- "$tmpfile" 122 123 } 123 124 124 125 check_vmlinux() {

+2 -1

scripts/link-vmlinux.sh

reviewed

··· 60 60 # skip output file argument 61 61 shift 62 62 63 63 - if is_enabled CONFIG_LTO_CLANG || is_enabled CONFIG_X86_KERNEL_IBT; then 63 63 + if is_enabled CONFIG_LTO_CLANG || is_enabled CONFIG_X86_KERNEL_IBT || 64 64 + is_enabled CONFIG_KLP_BUILD; then 64 65 # Use vmlinux.o instead of performing the slow LTO link again. 65 66 objs=vmlinux.o 66 67 libs=

+79

scripts/livepatch/fix-patch-lines

reviewed

··· 1 1 + #!/usr/bin/awk -f 2 2 + # SPDX-License-Identifier: GPL-2.0 3 3 + # 4 4 + # Use #line directives to preserve original __LINE__ numbers across patches to 5 5 + # avoid unwanted compilation changes. 6 6 + 7 7 + BEGIN { 8 8 + in_hunk = 0 9 9 + skip = 0 10 10 + } 11 11 + 12 12 + /^--- / { 13 13 + skip = $2 !~ /\.(c|h)$/ 14 14 + print 15 15 + next 16 16 + } 17 17 + 18 18 + /^@@/ { 19 19 + if (skip) { 20 20 + print 21 21 + next 22 22 + } 23 23 + 24 24 + in_hunk = 1 25 25 + 26 26 + # @@ -1,3 +1,4 @@: 27 27 + # 1: line number in old file 28 28 + # 3: how many lines the hunk covers in old file 29 29 + # 1: line number in new file 30 30 + # 4: how many lines the hunk covers in new file 31 31 + 32 32 + match($0, /^@@ -([0-9]+)(,([0-9]+))? \+([0-9]+)(,([0-9]+))? @@/, m) 33 33 + 34 34 + # Set 'cur' to the old file's line number at the start of the hunk. It 35 35 + # gets incremented for every context line and every line removal, so 36 36 + # that it always represents the old file's current line number. 37 37 + cur = m[1] 38 38 + 39 39 + # last = last line number of current hunk 40 40 + last = cur + (m[3] ? m[3] : 1) - 1 41 41 + 42 42 + need_line_directive = 0 43 43 + 44 44 + print 45 45 + next 46 46 + } 47 47 + 48 48 + { 49 49 + if (skip || !in_hunk || $0 ~ /^\\ No newline at end of file/) { 50 50 + print 51 51 + next 52 52 + } 53 53 + 54 54 + # change line 55 55 + if ($0 ~ /^[+-]/) { 56 56 + # inject #line after this group of changes 57 57 + need_line_directive = 1 58 58 + 59 59 + if ($0 ~ /^-/) 60 60 + cur++ 61 61 + 62 62 + print 63 63 + next 64 64 + } 65 65 + 66 66 + # If this is the first context line after a group of changes, inject 67 67 + # the #line directive to force the compiler to correct the line 68 68 + # numbering to match the original file. 69 69 + if (need_line_directive) { 70 70 + print "+#line " cur 71 71 + need_line_directive = 0 72 72 + } 73 73 + 74 74 + if (cur == last) 75 75 + in_hunk = 0 76 76 + 77 77 + cur++ 78 78 + print 79 79 + }

+108

scripts/livepatch/init.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* 3 3 + * Init code for a livepatch kernel module 4 4 + */ 5 5 + 6 6 + #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 7 + 8 8 + #include <linux/kernel.h> 9 9 + #include <linux/slab.h> 10 10 + #include <linux/livepatch.h> 11 11 + 12 12 + extern struct klp_object_ext __start_klp_objects[]; 13 13 + extern struct klp_object_ext __stop_klp_objects[]; 14 14 + 15 15 + static struct klp_patch *patch; 16 16 + 17 17 + static int __init livepatch_mod_init(void) 18 18 + { 19 19 + struct klp_object *objs; 20 20 + unsigned int nr_objs; 21 21 + int ret; 22 22 + 23 23 + nr_objs = __stop_klp_objects - __start_klp_objects; 24 24 + 25 25 + if (!nr_objs) { 26 26 + pr_err("nothing to patch!\n"); 27 27 + ret = -EINVAL; 28 28 + goto err; 29 29 + } 30 30 + 31 31 + patch = kzalloc(sizeof(*patch), GFP_KERNEL); 32 32 + if (!patch) { 33 33 + ret = -ENOMEM; 34 34 + goto err; 35 35 + } 36 36 + 37 37 + objs = kzalloc(sizeof(struct klp_object) * (nr_objs + 1), GFP_KERNEL); 38 38 + if (!objs) { 39 39 + ret = -ENOMEM; 40 40 + goto err_free_patch; 41 41 + } 42 42 + 43 43 + for (int i = 0; i < nr_objs; i++) { 44 44 + struct klp_object_ext *obj_ext = __start_klp_objects + i; 45 45 + struct klp_func_ext *funcs_ext = obj_ext->funcs; 46 46 + unsigned int nr_funcs = obj_ext->nr_funcs; 47 47 + struct klp_func *funcs = objs[i].funcs; 48 48 + struct klp_object *obj = objs + i; 49 49 + 50 50 + funcs = kzalloc(sizeof(struct klp_func) * (nr_funcs + 1), GFP_KERNEL); 51 51 + if (!funcs) { 52 52 + ret = -ENOMEM; 53 53 + for (int j = 0; j < i; j++) 54 54 + kfree(objs[i].funcs); 55 55 + goto err_free_objs; 56 56 + } 57 57 + 58 58 + for (int j = 0; j < nr_funcs; j++) { 59 59 + funcs[j].old_name = funcs_ext[j].old_name; 60 60 + funcs[j].new_func = funcs_ext[j].new_func; 61 61 + funcs[j].old_sympos = funcs_ext[j].sympos; 62 62 + } 63 63 + 64 64 + obj->name = obj_ext->name; 65 65 + obj->funcs = funcs; 66 66 + 67 67 + memcpy(&obj->callbacks, &obj_ext->callbacks, sizeof(struct klp_callbacks)); 68 68 + } 69 69 + 70 70 + patch->mod = THIS_MODULE; 71 71 + patch->objs = objs; 72 72 + 73 73 + /* TODO patch->states */ 74 74 + 75 75 + #ifdef KLP_NO_REPLACE 76 76 + patch->replace = false; 77 77 + #else 78 78 + patch->replace = true; 79 79 + #endif 80 80 + 81 81 + return klp_enable_patch(patch); 82 82 + 83 83 + err_free_objs: 84 84 + kfree(objs); 85 85 + err_free_patch: 86 86 + kfree(patch); 87 87 + err: 88 88 + return ret; 89 89 + } 90 90 + 91 91 + static void __exit livepatch_mod_exit(void) 92 92 + { 93 93 + unsigned int nr_objs; 94 94 + 95 95 + nr_objs = __stop_klp_objects - __start_klp_objects; 96 96 + 97 97 + for (int i = 0; i < nr_objs; i++) 98 98 + kfree(patch->objs[i].funcs); 99 99 + 100 100 + kfree(patch->objs); 101 101 + kfree(patch); 102 102 + } 103 103 + 104 104 + module_init(livepatch_mod_init); 105 105 + module_exit(livepatch_mod_exit); 106 106 + MODULE_LICENSE("GPL"); 107 107 + MODULE_INFO(livepatch, "Y"); 108 108 + MODULE_DESCRIPTION("Livepatch module");

+831

scripts/livepatch/klp-build

reviewed

··· 1 1 + #!/bin/bash 2 2 + # SPDX-License-Identifier: GPL-2.0 3 3 + # 4 4 + # Build a livepatch module 5 5 + 6 6 + # shellcheck disable=SC1090,SC2155 7 7 + 8 8 + if (( BASH_VERSINFO[0] < 4 || \ 9 9 + (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] < 4) )); then 10 10 + echo "error: this script requires bash 4.4+" >&2 11 11 + exit 1 12 12 + fi 13 13 + 14 14 + set -o errexit 15 15 + set -o errtrace 16 16 + set -o pipefail 17 17 + set -o nounset 18 18 + 19 19 + # Allow doing 'cmd | mapfile -t array' instead of 'mapfile -t array < <(cmd)'. 20 20 + # This helps keep execution in pipes so pipefail+errexit can catch errors. 21 21 + shopt -s lastpipe 22 22 + 23 23 + unset DEBUG_CLONE DIFF_CHECKSUM SKIP_CLEANUP XTRACE 24 24 + 25 25 + REPLACE=1 26 26 + SHORT_CIRCUIT=0 27 27 + JOBS="$(getconf _NPROCESSORS_ONLN)" 28 28 + VERBOSE="-s" 29 29 + shopt -o xtrace | grep -q 'on' && XTRACE=1 30 30 + 31 31 + # Avoid removing the previous $TMP_DIR until args have been fully processed. 32 32 + KEEP_TMP=1 33 33 + 34 34 + SCRIPT="$(basename "$0")" 35 35 + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 36 36 + FIX_PATCH_LINES="$SCRIPT_DIR/fix-patch-lines" 37 37 + 38 38 + SRC="$(pwd)" 39 39 + OBJ="$(pwd)" 40 40 + 41 41 + CONFIG="$OBJ/.config" 42 42 + TMP_DIR="$OBJ/klp-tmp" 43 43 + 44 44 + ORIG_DIR="$TMP_DIR/orig" 45 45 + PATCHED_DIR="$TMP_DIR/patched" 46 46 + DIFF_DIR="$TMP_DIR/diff" 47 47 + KMOD_DIR="$TMP_DIR/kmod" 48 48 + 49 49 + STASH_DIR="$TMP_DIR/stash" 50 50 + TIMESTAMP="$TMP_DIR/timestamp" 51 51 + PATCH_TMP_DIR="$TMP_DIR/tmp" 52 52 + 53 53 + KLP_DIFF_LOG="$DIFF_DIR/diff.log" 54 54 + 55 55 + grep0() { 56 56 + command grep "$@" || true 57 57 + } 58 58 + 59 59 + status() { 60 60 + echo "$*" 61 61 + } 62 62 + 63 63 + warn() { 64 64 + echo "error: $SCRIPT: $*" >&2 65 65 + } 66 66 + 67 67 + die() { 68 68 + warn "$@" 69 69 + exit 1 70 70 + } 71 71 + 72 72 + declare -a STASHED_FILES 73 73 + 74 74 + stash_file() { 75 75 + local file="$1" 76 76 + local rel_file="${file#"$SRC"/}" 77 77 + 78 78 + [[ ! -e "$file" ]] && die "no file to stash: $file" 79 79 + 80 80 + mkdir -p "$STASH_DIR/$(dirname "$rel_file")" 81 81 + cp -f "$file" "$STASH_DIR/$rel_file" 82 82 + 83 83 + STASHED_FILES+=("$rel_file") 84 84 + } 85 85 + 86 86 + restore_files() { 87 87 + local file 88 88 + 89 89 + for file in "${STASHED_FILES[@]}"; do 90 90 + mv -f "$STASH_DIR/$file" "$SRC/$file" || warn "can't restore file: $file" 91 91 + done 92 92 + 93 93 + STASHED_FILES=() 94 94 + } 95 95 + 96 96 + cleanup() { 97 97 + set +o nounset 98 98 + revert_patches "--recount" 99 99 + restore_files 100 100 + [[ "$KEEP_TMP" -eq 0 ]] && rm -rf "$TMP_DIR" 101 101 + return 0 102 102 + } 103 103 + 104 104 + trap_err() { 105 105 + warn "line ${BASH_LINENO[0]}: '$BASH_COMMAND'" 106 106 + } 107 107 + 108 108 + trap cleanup EXIT INT TERM HUP 109 109 + trap trap_err ERR 110 110 + 111 111 + __usage() { 112 112 + cat <<EOF 113 113 + Usage: $SCRIPT [OPTIONS] PATCH_FILE(s) 114 114 + Generate a livepatch module. 115 115 + 116 116 + Options: 117 117 + -f, --show-first-changed Show address of first changed instruction 118 118 + -j, --jobs=<jobs> Build jobs to run simultaneously [default: $JOBS] 119 119 + -o, --output=<file.ko> Output file [default: livepatch-<patch-name>.ko] 120 120 + --no-replace Disable livepatch atomic replace 121 121 + -v, --verbose Pass V=1 to kernel/module builds 122 122 + 123 123 + Advanced Options: 124 124 + -d, --debug Show symbol/reloc cloning decisions 125 125 + -S, --short-circuit=STEP Start at build step (requires prior --keep-tmp) 126 126 + 1|orig Build original kernel (default) 127 127 + 2|patched Build patched kernel 128 128 + 3|diff Diff objects 129 129 + 4|kmod Build patch module 130 130 + -T, --keep-tmp Preserve tmp dir on exit 131 131 + 132 132 + EOF 133 133 + } 134 134 + 135 135 + usage() { 136 136 + __usage >&2 137 137 + } 138 138 + 139 139 + process_args() { 140 140 + local keep_tmp=0 141 141 + local short 142 142 + local long 143 143 + local args 144 144 + 145 145 + short="hfj:o:vdS:T" 146 146 + long="help,show-first-changed,jobs:,output:,no-replace,verbose,debug,short-circuit:,keep-tmp" 147 147 + 148 148 + args=$(getopt --options "$short" --longoptions "$long" -- "$@") || { 149 149 + echo; usage; exit 150 150 + } 151 151 + eval set -- "$args" 152 152 + 153 153 + while true; do 154 154 + case "$1" in 155 155 + -h | --help) 156 156 + usage 157 157 + exit 0 158 158 + ;; 159 159 + -f | --show-first-changed) 160 160 + DIFF_CHECKSUM=1 161 161 + shift 162 162 + ;; 163 163 + -j | --jobs) 164 164 + JOBS="$2" 165 165 + shift 2 166 166 + ;; 167 167 + -o | --output) 168 168 + [[ "$2" != *.ko ]] && die "output filename should end with .ko" 169 169 + OUTFILE="$2" 170 170 + NAME="$(basename "$OUTFILE")" 171 171 + NAME="${NAME%.ko}" 172 172 + NAME="$(module_name_string "$NAME")" 173 173 + shift 2 174 174 + ;; 175 175 + --no-replace) 176 176 + REPLACE=0 177 177 + shift 178 178 + ;; 179 179 + -v | --verbose) 180 180 + VERBOSE="V=1" 181 181 + shift 182 182 + ;; 183 183 + -d | --debug) 184 184 + DEBUG_CLONE=1 185 185 + keep_tmp=1 186 186 + shift 187 187 + ;; 188 188 + -S | --short-circuit) 189 189 + [[ ! -d "$TMP_DIR" ]] && die "--short-circuit requires preserved klp-tmp dir" 190 190 + keep_tmp=1 191 191 + case "$2" in 192 192 + 1 | orig) SHORT_CIRCUIT=1; ;; 193 193 + 2 | patched) SHORT_CIRCUIT=2; ;; 194 194 + 3 | diff) SHORT_CIRCUIT=3; ;; 195 195 + 4 | mod) SHORT_CIRCUIT=4; ;; 196 196 + *) die "invalid short-circuit step '$2'" ;; 197 197 + esac 198 198 + shift 2 199 199 + ;; 200 200 + -T | --keep-tmp) 201 201 + keep_tmp=1 202 202 + shift 203 203 + ;; 204 204 + --) 205 205 + shift 206 206 + break 207 207 + ;; 208 208 + *) 209 209 + usage 210 210 + exit 1 211 211 + ;; 212 212 + esac 213 213 + done 214 214 + 215 215 + if [[ $# -eq 0 ]]; then 216 216 + usage 217 217 + exit 1 218 218 + fi 219 219 + 220 220 + KEEP_TMP="$keep_tmp" 221 221 + PATCHES=("$@") 222 222 + } 223 223 + 224 224 + # temporarily disable xtrace for especially verbose code 225 225 + xtrace_save() { 226 226 + [[ -v XTRACE ]] && set +x 227 227 + return 0 228 228 + } 229 229 + 230 230 + xtrace_restore() { 231 231 + [[ -v XTRACE ]] && set -x 232 232 + return 0 233 233 + } 234 234 + 235 235 + validate_config() { 236 236 + xtrace_save "reading .config" 237 237 + source "$CONFIG" || die "no .config file in $(dirname "$CONFIG")" 238 238 + xtrace_restore 239 239 + 240 240 + [[ -v CONFIG_LIVEPATCH ]] || \ 241 241 + die "CONFIG_LIVEPATCH not enabled" 242 242 + 243 243 + [[ -v CONFIG_KLP_BUILD ]] || \ 244 244 + die "CONFIG_KLP_BUILD not enabled" 245 245 + 246 246 + [[ -v CONFIG_GCC_PLUGIN_LATENT_ENTROPY ]] && \ 247 247 + die "kernel option 'CONFIG_GCC_PLUGIN_LATENT_ENTROPY' not supported" 248 248 + 249 249 + [[ -v CONFIG_GCC_PLUGIN_RANDSTRUCT ]] && \ 250 250 + die "kernel option 'CONFIG_GCC_PLUGIN_RANDSTRUCT' not supported" 251 251 + 252 252 + return 0 253 253 + } 254 254 + 255 255 + # Only allow alphanumerics and '_' and '-' in the module name. Everything else 256 256 + # is replaced with '-'. Also truncate to 55 chars so the full name + NUL 257 257 + # terminator fits in the kernel's 56-byte module name array. 258 258 + module_name_string() { 259 259 + echo "${1//[^a-zA-Z0-9_-]/-}" | cut -c 1-55 260 260 + } 261 261 + 262 262 + # If the module name wasn't specified on the cmdline with --output, give it a 263 263 + # name based on the patch name. 264 264 + set_module_name() { 265 265 + [[ -v NAME ]] && return 0 266 266 + 267 267 + if [[ "${#PATCHES[@]}" -eq 1 ]]; then 268 268 + NAME="$(basename "${PATCHES[0]}")" 269 269 + NAME="${NAME%.*}" 270 270 + else 271 271 + NAME="patch" 272 272 + fi 273 273 + 274 274 + NAME="livepatch-$NAME" 275 275 + NAME="$(module_name_string "$NAME")" 276 276 + 277 277 + OUTFILE="$NAME.ko" 278 278 + } 279 279 + 280 280 + # Hardcode the value printed by the localversion script to prevent patch 281 281 + # application from appending it with '+' due to a dirty git working tree. 282 282 + set_kernelversion() { 283 283 + local file="$SRC/scripts/setlocalversion" 284 284 + local localversion 285 285 + 286 286 + stash_file "$file" 287 287 + 288 288 + localversion="$(cd "$SRC" && make --no-print-directory kernelversion)" 289 289 + localversion="$(cd "$SRC" && KERNELVERSION="$localversion" ./scripts/setlocalversion)" 290 290 + [[ -z "$localversion" ]] && die "setlocalversion failed" 291 291 + 292 292 + sed -i "2i echo $localversion; exit 0" scripts/setlocalversion 293 293 + } 294 294 + 295 295 + get_patch_files() { 296 296 + local patch="$1" 297 297 + 298 298 + grep0 -E '^(--- |\+\+\+ )' "$patch" \ 299 299 + | gawk '{print $2}' \ 300 300 + | sed 's|^[^/]*/||' \ 301 301 + | sort -u 302 302 + } 303 303 + 304 304 + # Make sure git re-stats the changed files 305 305 + git_refresh() { 306 306 + local patch="$1" 307 307 + local files=() 308 308 + 309 309 + [[ ! -e "$SRC/.git" ]] && return 310 310 + 311 311 + get_patch_files "$patch" | mapfile -t files 312 312 + 313 313 + ( 314 314 + cd "$SRC" 315 315 + git update-index -q --refresh -- "${files[@]}" 316 316 + ) 317 317 + } 318 318 + 319 319 + check_unsupported_patches() { 320 320 + local patch 321 321 + 322 322 + for patch in "${PATCHES[@]}"; do 323 323 + local files=() 324 324 + 325 325 + get_patch_files "$patch" | mapfile -t files 326 326 + 327 327 + for file in "${files[@]}"; do 328 328 + case "$file" in 329 329 + lib/*|*.S) 330 330 + die "unsupported patch to $file" 331 331 + ;; 332 332 + esac 333 333 + done 334 334 + done 335 335 + } 336 336 + 337 337 + apply_patch() { 338 338 + local patch="$1" 339 339 + shift 340 340 + local extra_args=("$@") 341 341 + 342 342 + [[ ! -f "$patch" ]] && die "$patch doesn't exist" 343 343 + 344 344 + ( 345 345 + cd "$SRC" 346 346 + 347 347 + # The sed strips the version signature from 'git format-patch', 348 348 + # otherwise 'git apply --recount' warns. 349 349 + sed -n '/^-- /q;p' "$patch" | 350 350 + git apply "${extra_args[@]}" 351 351 + ) 352 352 + 353 353 + APPLIED_PATCHES+=("$patch") 354 354 + } 355 355 + 356 356 + revert_patch() { 357 357 + local patch="$1" 358 358 + shift 359 359 + local extra_args=("$@") 360 360 + local tmp=() 361 361 + 362 362 + ( 363 363 + cd "$SRC" 364 364 + 365 365 + sed -n '/^-- /q;p' "$patch" | 366 366 + git apply --reverse "${extra_args[@]}" 367 367 + ) 368 368 + git_refresh "$patch" 369 369 + 370 370 + for p in "${APPLIED_PATCHES[@]}"; do 371 371 + [[ "$p" == "$patch" ]] && continue 372 372 + tmp+=("$p") 373 373 + done 374 374 + 375 375 + APPLIED_PATCHES=("${tmp[@]}") 376 376 + } 377 377 + 378 378 + apply_patches() { 379 379 + local patch 380 380 + 381 381 + for patch in "${PATCHES[@]}"; do 382 382 + apply_patch "$patch" 383 383 + done 384 384 + } 385 385 + 386 386 + revert_patches() { 387 387 + local extra_args=("$@") 388 388 + local patches=("${APPLIED_PATCHES[@]}") 389 389 + 390 390 + for (( i=${#patches[@]}-1 ; i>=0 ; i-- )) ; do 391 391 + revert_patch "${patches[$i]}" "${extra_args[@]}" 392 392 + done 393 393 + 394 394 + APPLIED_PATCHES=() 395 395 + } 396 396 + 397 397 + validate_patches() { 398 398 + check_unsupported_patches 399 399 + apply_patches 400 400 + revert_patches 401 401 + } 402 402 + 403 403 + do_init() { 404 404 + # We're not yet smart enough to handle anything other than in-tree 405 405 + # builds in pwd. 406 406 + [[ ! "$SRC" -ef "$SCRIPT_DIR/../.." ]] && die "please run from the kernel root directory" 407 407 + [[ ! "$OBJ" -ef "$SCRIPT_DIR/../.." ]] && die "please run from the kernel root directory" 408 408 + 409 409 + (( SHORT_CIRCUIT <= 1 )) && rm -rf "$TMP_DIR" 410 410 + mkdir -p "$TMP_DIR" 411 411 + 412 412 + APPLIED_PATCHES=() 413 413 + 414 414 + [[ -x "$FIX_PATCH_LINES" ]] || die "can't find fix-patch-lines" 415 415 + 416 416 + validate_config 417 417 + set_module_name 418 418 + set_kernelversion 419 419 + } 420 420 + 421 421 + # Refresh the patch hunk headers, specifically the line numbers and counts. 422 422 + refresh_patch() { 423 423 + local patch="$1" 424 424 + local tmpdir="$PATCH_TMP_DIR" 425 425 + local files=() 426 426 + 427 427 + rm -rf "$tmpdir" 428 428 + mkdir -p "$tmpdir/a" 429 429 + mkdir -p "$tmpdir/b" 430 430 + 431 431 + # Get all source files affected by the patch 432 432 + get_patch_files "$patch" | mapfile -t files 433 433 + 434 434 + # Copy orig source files to 'a' 435 435 + ( cd "$SRC" && echo "${files[@]}" | xargs cp --parents --target-directory="$tmpdir/a" ) 436 436 + 437 437 + # Copy patched source files to 'b' 438 438 + apply_patch "$patch" --recount 439 439 + ( cd "$SRC" && echo "${files[@]}" | xargs cp --parents --target-directory="$tmpdir/b" ) 440 440 + revert_patch "$patch" --recount 441 441 + 442 442 + # Diff 'a' and 'b' to make a clean patch 443 443 + ( cd "$tmpdir" && git diff --no-index --no-prefix a b > "$patch" ) || true 444 444 + } 445 445 + 446 446 + # Copy the patches to a temporary directory, fix their lines so as not to 447 447 + # affect the __LINE__ macro for otherwise unchanged functions further down the 448 448 + # file, and update $PATCHES to point to the fixed patches. 449 449 + fix_patches() { 450 450 + local idx 451 451 + local i 452 452 + 453 453 + rm -f "$TMP_DIR"/*.patch 454 454 + 455 455 + idx=0001 456 456 + for i in "${!PATCHES[@]}"; do 457 457 + local old_patch="${PATCHES[$i]}" 458 458 + local tmp_patch="$TMP_DIR/tmp.patch" 459 459 + local patch="${PATCHES[$i]}" 460 460 + local new_patch 461 461 + 462 462 + new_patch="$TMP_DIR/$idx-fixed-$(basename "$patch")" 463 463 + 464 464 + cp -f "$old_patch" "$tmp_patch" 465 465 + refresh_patch "$tmp_patch" 466 466 + "$FIX_PATCH_LINES" "$tmp_patch" > "$new_patch" 467 467 + refresh_patch "$new_patch" 468 468 + 469 469 + PATCHES[i]="$new_patch" 470 470 + 471 471 + rm -f "$tmp_patch" 472 472 + idx=$(printf "%04d" $(( 10#$idx + 1 ))) 473 473 + done 474 474 + } 475 475 + 476 476 + clean_kernel() { 477 477 + local cmd=() 478 478 + 479 479 + cmd=("make") 480 480 + cmd+=("--silent") 481 481 + cmd+=("-j$JOBS") 482 482 + cmd+=("clean") 483 483 + 484 484 + ( 485 485 + cd "$SRC" 486 486 + "${cmd[@]}" 487 487 + ) 488 488 + } 489 489 + 490 490 + build_kernel() { 491 491 + local log="$TMP_DIR/build.log" 492 492 + local objtool_args=() 493 493 + local cmd=() 494 494 + 495 495 + objtool_args=("--checksum") 496 496 + 497 497 + cmd=("make") 498 498 + 499 499 + # When a patch to a kernel module references a newly created unexported 500 500 + # symbol which lives in vmlinux or another kernel module, the patched 501 501 + # kernel build fails with the following error: 502 502 + # 503 503 + # ERROR: modpost: "klp_string" [fs/xfs/xfs.ko] undefined! 504 504 + # 505 505 + # The undefined symbols are working as designed in that case. They get 506 506 + # resolved later when the livepatch module build link pulls all the 507 507 + # disparate objects together into the same kernel module. 508 508 + # 509 509 + # It would be good to have a way to tell modpost to skip checking for 510 510 + # undefined symbols altogether. For now, just convert the error to a 511 511 + # warning with KBUILD_MODPOST_WARN, and grep out the warning to avoid 512 512 + # confusing the user. 513 513 + # 514 514 + cmd+=("KBUILD_MODPOST_WARN=1") 515 515 + 516 516 + cmd+=("$VERBOSE") 517 517 + cmd+=("-j$JOBS") 518 518 + cmd+=("KCFLAGS=-ffunction-sections -fdata-sections") 519 519 + cmd+=("OBJTOOL_ARGS=${objtool_args[*]}") 520 520 + cmd+=("vmlinux") 521 521 + cmd+=("modules") 522 522 + 523 523 + ( 524 524 + cd "$SRC" 525 525 + "${cmd[@]}" \ 526 526 + 1> >(tee -a "$log") \ 527 527 + 2> >(tee -a "$log" | grep0 -v "modpost.*undefined!" >&2) 528 528 + ) 529 529 + } 530 530 + 531 531 + find_objects() { 532 532 + local opts=("$@") 533 533 + 534 534 + # Find root-level vmlinux.o and non-root-level .ko files, 535 535 + # excluding klp-tmp/ and .git/ 536 536 + find "$OBJ" \( -path "$TMP_DIR" -o -path "$OBJ/.git" -o -regex "$OBJ/[^/][^/]*\.ko" \) -prune -o \ 537 537 + -type f "${opts[@]}" \ 538 538 + \( -name "*.ko" -o -path "$OBJ/vmlinux.o" \) \ 539 539 + -printf '%P\n' 540 540 + } 541 541 + 542 542 + # Copy all .o archives to $ORIG_DIR 543 543 + copy_orig_objects() { 544 544 + local files=() 545 545 + 546 546 + rm -rf "$ORIG_DIR" 547 547 + mkdir -p "$ORIG_DIR" 548 548 + 549 549 + find_objects | mapfile -t files 550 550 + 551 551 + xtrace_save "copying orig objects" 552 552 + for _file in "${files[@]}"; do 553 553 + local rel_file="${_file/.ko/.o}" 554 554 + local file="$OBJ/$rel_file" 555 555 + local file_dir="$(dirname "$file")" 556 556 + local orig_file="$ORIG_DIR/$rel_file" 557 557 + local orig_dir="$(dirname "$orig_file")" 558 558 + local cmd_file="$file_dir/.$(basename "$file").cmd" 559 559 + 560 560 + [[ ! -f "$file" ]] && die "missing $(basename "$file") for $_file" 561 561 + 562 562 + mkdir -p "$orig_dir" 563 563 + cp -f "$file" "$orig_dir" 564 564 + [[ -e "$cmd_file" ]] && cp -f "$cmd_file" "$orig_dir" 565 565 + done 566 566 + xtrace_restore 567 567 + 568 568 + mv -f "$TMP_DIR/build.log" "$ORIG_DIR" 569 569 + touch "$TIMESTAMP" 570 570 + } 571 571 + 572 572 + # Copy all changed objects to $PATCHED_DIR 573 573 + copy_patched_objects() { 574 574 + local files=() 575 575 + local opts=() 576 576 + local found=0 577 577 + 578 578 + rm -rf "$PATCHED_DIR" 579 579 + mkdir -p "$PATCHED_DIR" 580 580 + 581 581 + # Note this doesn't work with some configs, thus the 'cmp' below. 582 582 + opts=("-newer") 583 583 + opts+=("$TIMESTAMP") 584 584 + 585 585 + find_objects "${opts[@]}" | mapfile -t files 586 586 + 587 587 + xtrace_save "copying changed objects" 588 588 + for _file in "${files[@]}"; do 589 589 + local rel_file="${_file/.ko/.o}" 590 590 + local file="$OBJ/$rel_file" 591 591 + local orig_file="$ORIG_DIR/$rel_file" 592 592 + local patched_file="$PATCHED_DIR/$rel_file" 593 593 + local patched_dir="$(dirname "$patched_file")" 594 594 + 595 595 + [[ ! -f "$file" ]] && die "missing $(basename "$file") for $_file" 596 596 + 597 597 + cmp -s "$orig_file" "$file" && continue 598 598 + 599 599 + mkdir -p "$patched_dir" 600 600 + cp -f "$file" "$patched_dir" 601 601 + found=1 602 602 + done 603 603 + xtrace_restore 604 604 + 605 605 + (( found == 0 )) && die "no changes detected" 606 606 + 607 607 + mv -f "$TMP_DIR/build.log" "$PATCHED_DIR" 608 608 + } 609 609 + 610 610 + # Diff changed objects, writing output object to $DIFF_DIR 611 611 + diff_objects() { 612 612 + local log="$KLP_DIFF_LOG" 613 613 + local files=() 614 614 + local opts=() 615 615 + 616 616 + rm -rf "$DIFF_DIR" 617 617 + mkdir -p "$DIFF_DIR" 618 618 + 619 619 + find "$PATCHED_DIR" -type f -name "*.o" | mapfile -t files 620 620 + [[ ${#files[@]} -eq 0 ]] && die "no changes detected" 621 621 + 622 622 + [[ -v DEBUG_CLONE ]] && opts=("--debug") 623 623 + 624 624 + # Diff all changed objects 625 625 + for file in "${files[@]}"; do 626 626 + local rel_file="${file#"$PATCHED_DIR"/}" 627 627 + local orig_file="$rel_file" 628 628 + local patched_file="$PATCHED_DIR/$rel_file" 629 629 + local out_file="$DIFF_DIR/$rel_file" 630 630 + local filter=() 631 631 + local cmd=() 632 632 + 633 633 + mkdir -p "$(dirname "$out_file")" 634 634 + 635 635 + cmd=("$SRC/tools/objtool/objtool") 636 636 + cmd+=("klp") 637 637 + cmd+=("diff") 638 638 + (( ${#opts[@]} > 0 )) && cmd+=("${opts[@]}") 639 639 + cmd+=("$orig_file") 640 640 + cmd+=("$patched_file") 641 641 + cmd+=("$out_file") 642 642 + 643 643 + if [[ -v DIFF_CHECKSUM ]]; then 644 644 + filter=("grep0") 645 645 + filter+=("-Ev") 646 646 + filter+=("DEBUG: .*checksum: ") 647 647 + else 648 648 + filter=("cat") 649 649 + fi 650 650 + 651 651 + ( 652 652 + cd "$ORIG_DIR" 653 653 + "${cmd[@]}" \ 654 654 + 1> >(tee -a "$log") \ 655 655 + 2> >(tee -a "$log" | "${filter[@]}" >&2) || \ 656 656 + die "objtool klp diff failed" 657 657 + ) 658 658 + done 659 659 + } 660 660 + 661 661 + # For each changed object, run objtool with --debug-checksum to get the 662 662 + # per-instruction checksums, and then diff those to find the first changed 663 663 + # instruction for each function. 664 664 + diff_checksums() { 665 665 + local orig_log="$ORIG_DIR/checksum.log" 666 666 + local patched_log="$PATCHED_DIR/checksum.log" 667 667 + local -A funcs 668 668 + local cmd=() 669 669 + local line 670 670 + local file 671 671 + local func 672 672 + 673 673 + gawk '/\.o: changed function: / { 674 674 + sub(/:$/, "", $1) 675 675 + print $1, $NF 676 676 + }' "$KLP_DIFF_LOG" | mapfile -t lines 677 677 + 678 678 + for line in "${lines[@]}"; do 679 679 + read -r file func <<< "$line" 680 680 + if [[ ! -v funcs["$file"] ]]; then 681 681 + funcs["$file"]="$func" 682 682 + else 683 683 + funcs["$file"]+=" $func" 684 684 + fi 685 685 + done 686 686 + 687 687 + cmd=("$SRC/tools/objtool/objtool") 688 688 + cmd+=("--checksum") 689 689 + cmd+=("--link") 690 690 + cmd+=("--dry-run") 691 691 + 692 692 + for file in "${!funcs[@]}"; do 693 693 + local opt="--debug-checksum=${funcs[$file]// /,}" 694 694 + 695 695 + ( 696 696 + cd "$ORIG_DIR" 697 697 + "${cmd[@]}" "$opt" "$file" &> "$orig_log" || \ 698 698 + ( cat "$orig_log" >&2; die "objtool --debug-checksum failed" ) 699 699 + 700 700 + cd "$PATCHED_DIR" 701 701 + "${cmd[@]}" "$opt" "$file" &> "$patched_log" || \ 702 702 + ( cat "$patched_log" >&2; die "objtool --debug-checksum failed" ) 703 703 + ) 704 704 + 705 705 + for func in ${funcs[$file]}; do 706 706 + diff <( grep0 -E "^DEBUG: .*checksum: $func " "$orig_log" | sed "s|$ORIG_DIR/||") \ 707 707 + <( grep0 -E "^DEBUG: .*checksum: $func " "$patched_log" | sed "s|$PATCHED_DIR/||") \ 708 708 + | gawk '/^< DEBUG: / { 709 709 + gsub(/:/, "") 710 710 + printf "%s: %s: %s\n", $3, $5, $6 711 711 + exit 712 712 + }' || true 713 713 + done 714 714 + done 715 715 + } 716 716 + 717 717 + # Build and post-process livepatch module in $KMOD_DIR 718 718 + build_patch_module() { 719 719 + local makefile="$KMOD_DIR/Kbuild" 720 720 + local log="$KMOD_DIR/build.log" 721 721 + local kmod_file 722 722 + local cflags=() 723 723 + local files=() 724 724 + local cmd=() 725 725 + 726 726 + rm -rf "$KMOD_DIR" 727 727 + mkdir -p "$KMOD_DIR" 728 728 + 729 729 + cp -f "$SRC/scripts/livepatch/init.c" "$KMOD_DIR" 730 730 + 731 731 + echo "obj-m := $NAME.o" > "$makefile" 732 732 + echo -n "$NAME-y := init.o" >> "$makefile" 733 733 + 734 734 + find "$DIFF_DIR" -type f -name "*.o" | mapfile -t files 735 735 + [[ ${#files[@]} -eq 0 ]] && die "no changes detected" 736 736 + 737 737 + for file in "${files[@]}"; do 738 738 + local rel_file="${file#"$DIFF_DIR"/}" 739 739 + local orig_file="$ORIG_DIR/$rel_file" 740 740 + local orig_dir="$(dirname "$orig_file")" 741 741 + local kmod_file="$KMOD_DIR/$rel_file" 742 742 + local kmod_dir="$(dirname "$kmod_file")" 743 743 + local cmd_file="$orig_dir/.$(basename "$file").cmd" 744 744 + 745 745 + mkdir -p "$kmod_dir" 746 746 + cp -f "$file" "$kmod_dir" 747 747 + [[ -e "$cmd_file" ]] && cp -f "$cmd_file" "$kmod_dir" 748 748 + 749 749 + # Tell kbuild this is a prebuilt object 750 750 + cp -f "$file" "${kmod_file}_shipped" 751 751 + 752 752 + echo -n " $rel_file" >> "$makefile" 753 753 + done 754 754 + 755 755 + echo >> "$makefile" 756 756 + 757 757 + cflags=("-ffunction-sections") 758 758 + cflags+=("-fdata-sections") 759 759 + [[ $REPLACE -eq 0 ]] && cflags+=("-DKLP_NO_REPLACE") 760 760 + 761 761 + cmd=("make") 762 762 + cmd+=("$VERBOSE") 763 763 + cmd+=("-j$JOBS") 764 764 + cmd+=("--directory=.") 765 765 + cmd+=("M=$KMOD_DIR") 766 766 + cmd+=("KCFLAGS=${cflags[*]}") 767 767 + 768 768 + # Build a "normal" kernel module with init.c and the diffed objects 769 769 + ( 770 770 + cd "$SRC" 771 771 + "${cmd[@]}" \ 772 772 + 1> >(tee -a "$log") \ 773 773 + 2> >(tee -a "$log" >&2) 774 774 + ) 775 775 + 776 776 + kmod_file="$KMOD_DIR/$NAME.ko" 777 777 + 778 778 + # Save off the intermediate binary for debugging 779 779 + cp -f "$kmod_file" "$kmod_file.orig" 780 780 + 781 781 + # Work around issue where slight .config change makes corrupt BTF 782 782 + objcopy --remove-section=.BTF "$kmod_file" 783 783 + 784 784 + # Fix (and work around) linker wreckage for klp syms / relocs 785 785 + "$SRC/tools/objtool/objtool" klp post-link "$kmod_file" || die "objtool klp post-link failed" 786 786 + 787 787 + cp -f "$kmod_file" "$OUTFILE" 788 788 + } 789 789 + 790 790 + 791 791 + ################################################################################ 792 792 + 793 793 + process_args "$@" 794 794 + do_init 795 795 + 796 796 + if (( SHORT_CIRCUIT <= 1 )); then 797 797 + status "Validating patch(es)" 798 798 + validate_patches 799 799 + status "Building original kernel" 800 800 + clean_kernel 801 801 + build_kernel 802 802 + status "Copying original object files" 803 803 + copy_orig_objects 804 804 + fi 805 805 + 806 806 + if (( SHORT_CIRCUIT <= 2 )); then 807 807 + status "Fixing patch(es)" 808 808 + fix_patches 809 809 + apply_patches 810 810 + status "Building patched kernel" 811 811 + build_kernel 812 812 + revert_patches 813 813 + status "Copying patched object files" 814 814 + copy_patched_objects 815 815 + fi 816 816 + 817 817 + if (( SHORT_CIRCUIT <= 3 )); then 818 818 + status "Diffing objects" 819 819 + diff_objects 820 820 + if [[ -v DIFF_CHECKSUM ]]; then 821 821 + status "Finding first changed instructions" 822 822 + diff_checksums 823 823 + fi 824 824 + fi 825 825 + 826 826 + if (( SHORT_CIRCUIT <= 4 )); then 827 827 + status "Building patch module: $OUTFILE" 828 828 + build_patch_module 829 829 + fi 830 830 + 831 831 + status "SUCCESS"

+5

scripts/mod/modpost.c

reviewed

··· 606 606 strstarts(symname, "_savevr_") || 607 607 strcmp(symname, ".TOC.") == 0) 608 608 return 1; 609 609 + 610 610 + /* ignore linker-created section bounds variables */ 611 611 + if (strstarts(symname, "__start_") || strstarts(symname, "__stop_")) 612 612 + return 1; 613 613 + 609 614 /* Do not ignore this symbol */ 610 615 return 0; 611 616 }

+14 -8

scripts/module.lds.S

reviewed

··· 34 34 35 35 __patchable_function_entries : { *(__patchable_function_entries) } 36 36 37 37 + __klp_funcs 0: ALIGN(8) { KEEP(*(__klp_funcs)) } 38 38 + 39 39 + __klp_objects 0: ALIGN(8) { 40 40 + __start_klp_objects = .; 41 41 + KEEP(*(__klp_objects)) 42 42 + __stop_klp_objects = .; 43 43 + } 44 44 + 37 45 #ifdef CONFIG_ARCH_USES_CFI_TRAPS 38 38 - __kcfi_traps : { KEEP(*(.kcfi_traps)) } 46 46 + __kcfi_traps : { KEEP(*(.kcfi_traps)) } 39 47 #endif 40 48 41 41 - #ifdef CONFIG_LTO_CLANG 42 42 - /* 43 43 - * With CONFIG_LTO_CLANG, LLD always enables -fdata-sections and 44 44 - * -ffunction-sections, which increases the size of the final module. 45 45 - * Merge the split sections in the final binary. 46 46 - */ 49 49 + .text : { 50 50 + *(.text .text.[0-9a-zA-Z_]*) 51 51 + } 52 52 + 47 53 .bss : { 48 54 *(.bss .bss.[0-9a-zA-Z_]*) 49 55 *(.bss..L*) ··· 64 58 *(.rodata .rodata.[0-9a-zA-Z_]*) 65 59 *(.rodata..L*) 66 60 } 67 67 - #endif 61 61 + 68 62 MOD_SEPARATE_CODETAG_SECTIONS() 69 63 } 70 64

+34

tools/arch/x86/tools/gen-cpu-feature-names-x86.awk

reviewed

··· 1 1 + #!/bin/awk -f 2 2 + # SPDX-License-Identifier: GPL-2.0 3 3 + # 4 4 + # Copyright (c) 2025, Oracle and/or its affiliates. 5 5 + # 6 6 + # Usage: awk -f gen-cpu-feature-names-x86.awk cpufeatures.h > cpu-feature-names.c 7 7 + # 8 8 + 9 9 + BEGIN { 10 10 + print "/* cpu feature name array generated from cpufeatures.h */" 11 11 + print "/* Do not change this code. */" 12 12 + print 13 13 + print "static const char *cpu_feature_names[(NCAPINTS+NBUGINTS)*32] = {" 14 14 + 15 15 + value_expr = "\\([0-9*+ ]+\\)" 16 16 + } 17 17 + 18 18 + /^#define X86_FEATURE_/ { 19 19 + if (match($0, value_expr)) { 20 20 + value = substr($0, RSTART + 1, RLENGTH - 2) 21 21 + print "\t[" value "] = \"" $2 "\"," 22 22 + } 23 23 + } 24 24 + 25 25 + /^#define X86_BUG_/ { 26 26 + if (match($0, value_expr)) { 27 27 + value = substr($0, RSTART + 1, RLENGTH - 2) 28 28 + print "\t[NCAPINTS*32+(" value ")] = \"" $2 "\"," 29 29 + } 30 30 + } 31 31 + 32 32 + END { 33 33 + print "};" 34 34 + }

+2

tools/build/Build

reviewed

··· 1 1 + hostprogs := fixdep 2 2 + fixdep-y := fixdep.o

+19 -2

tools/build/Makefile

reviewed

··· 37 37 $(Q)$(MAKE) -C feature OUTPUT=$(TMP_O) clean >/dev/null 38 38 endif 39 39 40 40 - $(OUTPUT)fixdep: $(srctree)/tools/build/fixdep.c 41 41 - $(QUIET_CC)$(HOSTCC) $(KBUILD_HOSTCFLAGS) $(KBUILD_HOSTLDFLAGS) -o $@ $< 40 40 + FIXDEP := $(OUTPUT)fixdep 41 41 + FIXDEP_IN := $(OUTPUT)fixdep-in.o 42 42 + 43 43 + # To track fixdep's dependencies properly, fixdep needs to run on itself. 44 44 + # Build it twice the first time. 45 45 + $(FIXDEP_IN): FORCE 46 46 + $(Q)if [ ! -f $(FIXDEP) ]; then \ 47 47 + $(MAKE) $(build)=fixdep HOSTCFLAGS="$(KBUILD_HOSTCFLAGS)"; \ 48 48 + rm -f $(FIXDEP).o; \ 49 49 + fi 50 50 + $(Q)$(MAKE) $(build)=fixdep HOSTCFLAGS="$(KBUILD_HOSTCFLAGS)" 51 51 + 52 52 + 53 53 + $(FIXDEP): $(FIXDEP_IN) 54 54 + $(QUIET_LINK)$(HOSTCC) $(FIXDEP_IN) $(KBUILD_HOSTLDFLAGS) -o $@ 55 55 + 56 56 + FORCE: 57 57 + 58 58 + .PHONY: FORCE

+3 -1

tools/build/Makefile.feature

reviewed

··· 315 315 316 316 ifeq ($(FEATURE_DISPLAY_DEFERRED),) 317 317 $(call feature_display_entries) 318 318 - $(info ) 318 318 + ifeq ($(feature_display),1) 319 319 + $(info ) 320 320 + endif 319 321 endif

+3 -7

tools/include/linux/interval_tree_generic.h

reviewed

··· 77 77 * Cond2: start <= ITLAST(node) \ 78 78 */ \ 79 79 \ 80 80 - static ITSTRUCT * \ 80 80 + ITSTATIC ITSTRUCT * \ 81 81 ITPREFIX ## _subtree_search(ITSTRUCT *node, ITTYPE start, ITTYPE last) \ 82 82 { \ 83 83 while (true) { \ ··· 104 104 if (ITSTART(node) <= last) { /* Cond1 */ \ 105 105 if (start <= ITLAST(node)) /* Cond2 */ \ 106 106 return node; /* node is leftmost match */ \ 107 107 - if (node->ITRB.rb_right) { \ 108 108 - node = rb_entry(node->ITRB.rb_right, \ 109 109 - ITSTRUCT, ITRB); \ 110 110 - if (start <= node->ITSUBTREE) \ 111 111 - continue; \ 112 112 - } \ 107 107 + node = rb_entry(node->ITRB.rb_right, ITSTRUCT, ITRB); \ 108 108 + continue; \ 113 109 } \ 114 110 return NULL; /* No match */ \ 115 111 } \

+76

tools/include/linux/livepatch_external.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 + /* 3 3 + * External livepatch interfaces for patch creation tooling 4 4 + */ 5 5 + 6 6 + #ifndef _LINUX_LIVEPATCH_EXTERNAL_H_ 7 7 + #define _LINUX_LIVEPATCH_EXTERNAL_H_ 8 8 + 9 9 + #include <linux/types.h> 10 10 + 11 11 + #define KLP_RELOC_SEC_PREFIX ".klp.rela." 12 12 + #define KLP_SYM_PREFIX ".klp.sym." 13 13 + 14 14 + #define __KLP_PRE_PATCH_PREFIX __klp_pre_patch_callback_ 15 15 + #define __KLP_POST_PATCH_PREFIX __klp_post_patch_callback_ 16 16 + #define __KLP_PRE_UNPATCH_PREFIX __klp_pre_unpatch_callback_ 17 17 + #define __KLP_POST_UNPATCH_PREFIX __klp_post_unpatch_callback_ 18 18 + 19 19 + #define KLP_PRE_PATCH_PREFIX __stringify(__KLP_PRE_PATCH_PREFIX) 20 20 + #define KLP_POST_PATCH_PREFIX __stringify(__KLP_POST_PATCH_PREFIX) 21 21 + #define KLP_PRE_UNPATCH_PREFIX __stringify(__KLP_PRE_UNPATCH_PREFIX) 22 22 + #define KLP_POST_UNPATCH_PREFIX __stringify(__KLP_POST_UNPATCH_PREFIX) 23 23 + 24 24 + struct klp_object; 25 25 + 26 26 + typedef int (*klp_pre_patch_t)(struct klp_object *obj); 27 27 + typedef void (*klp_post_patch_t)(struct klp_object *obj); 28 28 + typedef void (*klp_pre_unpatch_t)(struct klp_object *obj); 29 29 + typedef void (*klp_post_unpatch_t)(struct klp_object *obj); 30 30 + 31 31 + /** 32 32 + * struct klp_callbacks - pre/post live-(un)patch callback structure 33 33 + * @pre_patch: executed before code patching 34 34 + * @post_patch: executed after code patching 35 35 + * @pre_unpatch: executed before code unpatching 36 36 + * @post_unpatch: executed after code unpatching 37 37 + * @post_unpatch_enabled: flag indicating if post-unpatch callback 38 38 + * should run 39 39 + * 40 40 + * All callbacks are optional. Only the pre-patch callback, if provided, 41 41 + * will be unconditionally executed. If the parent klp_object fails to 42 42 + * patch for any reason, including a non-zero error status returned from 43 43 + * the pre-patch callback, no further callbacks will be executed. 44 44 + */ 45 45 + struct klp_callbacks { 46 46 + klp_pre_patch_t pre_patch; 47 47 + klp_post_patch_t post_patch; 48 48 + klp_pre_unpatch_t pre_unpatch; 49 49 + klp_post_unpatch_t post_unpatch; 50 50 + bool post_unpatch_enabled; 51 51 + }; 52 52 + 53 53 + /* 54 54 + * 'struct klp_{func,object}_ext' are compact "external" representations of 55 55 + * 'struct klp_{func,object}'. They are used by objtool for livepatch 56 56 + * generation. The structs are then read by the livepatch module and converted 57 57 + * to the real structs before calling klp_enable_patch(). 58 58 + * 59 59 + * TODO make these the official API for klp_enable_patch(). That should 60 60 + * simplify livepatch's interface as well as its data structure lifetime 61 61 + * management. 62 62 + */ 63 63 + struct klp_func_ext { 64 64 + const char *old_name; 65 65 + void *new_func; 66 66 + unsigned long sympos; 67 67 + }; 68 68 + 69 69 + struct klp_object_ext { 70 70 + const char *name; 71 71 + struct klp_func_ext *funcs; 72 72 + struct klp_callbacks callbacks; 73 73 + unsigned int nr_funcs; 74 74 + }; 75 75 + 76 76 + #endif /* _LINUX_LIVEPATCH_EXTERNAL_H_ */

+2

tools/include/linux/objtool_types.h

reviewed

··· 67 67 #define ANNOTYPE_REACHABLE 8 68 68 #define ANNOTYPE_NOCFI 9 69 69 70 70 + #define ANNOTYPE_DATA_SPECIAL 1 71 71 + 70 72 #endif /* _LINUX_OBJTOOL_TYPES_H */

+14

tools/include/linux/string.h

reviewed

··· 44 44 return strncmp(str, prefix, strlen(prefix)) == 0; 45 45 } 46 46 47 47 + /* 48 48 + * Checks if a string ends with another. 49 49 + */ 50 50 + static inline bool str_ends_with(const char *str, const char *substr) 51 51 + { 52 52 + size_t len = strlen(str); 53 53 + size_t sublen = strlen(substr); 54 54 + 55 55 + if (sublen > len) 56 56 + return false; 57 57 + 58 58 + return !strcmp(str + len - sublen, substr); 59 59 + } 60 60 + 47 61 extern char * __must_check skip_spaces(const char *); 48 62 49 63 extern char *strim(char *);

+3

tools/objtool/.gitignore

reviewed

··· 1 1 # SPDX-License-Identifier: GPL-2.0-only 2 2 + arch/x86/lib/cpu-feature-names.c 2 3 arch/x86/lib/inat-tables.c 3 4 /objtool 5 5 + feature 6 6 + FEATURE-DUMP.objtool 4 7 fixdep 5 8 libsubcmd/

+5 -2

tools/objtool/Build

reviewed

··· 8 8 objtool-y += elf.o 9 9 objtool-y += objtool.o 10 10 11 11 - objtool-$(BUILD_ORC) += orc_gen.o 12 12 - objtool-$(BUILD_ORC) += orc_dump.o 11 11 + objtool-$(BUILD_DISAS) += disas.o 12 12 + objtool-$(BUILD_DISAS) += trace.o 13 13 + 14 14 + objtool-$(BUILD_ORC) += orc_gen.o orc_dump.o 15 15 + objtool-$(BUILD_KLP) += builtin-klp.o klp-diff.o klp-post-link.o 13 16 14 17 objtool-y += libstring.o 15 18 objtool-y += libctype.o

+58 -16

tools/objtool/Makefile

reviewed

··· 2 2 include ../scripts/Makefile.include 3 3 include ../scripts/Makefile.arch 4 4 5 5 + ifeq ($(SRCARCH),x86) 6 6 + BUILD_ORC := y 7 7 + ARCH_HAS_KLP := y 8 8 + endif 9 9 + 10 10 + ifeq ($(SRCARCH),loongarch) 11 11 + BUILD_ORC := y 12 12 + endif 13 13 + 14 14 + ifeq ($(ARCH_HAS_KLP),y) 15 15 + HAVE_XXHASH = $(shell printf "$(pound)include <xxhash.h>\nXXH3_state_t *state;int main() {}" | \ 16 16 + $(HOSTCC) -xc - -o /dev/null -lxxhash 2> /dev/null && echo y || echo n) 17 17 + ifeq ($(HAVE_XXHASH),y) 18 18 + BUILD_KLP := y 19 19 + LIBXXHASH_CFLAGS := $(shell $(HOSTPKG_CONFIG) libxxhash --cflags 2>/dev/null) \ 20 20 + -DBUILD_KLP 21 21 + LIBXXHASH_LIBS := $(shell $(HOSTPKG_CONFIG) libxxhash --libs 2>/dev/null || echo -lxxhash) 22 22 + endif 23 23 + endif 24 24 + 25 25 + export BUILD_ORC BUILD_KLP 26 26 + 5 27 ifeq ($(srctree),) 6 28 srctree := $(patsubst %/,%,$(dir $(CURDIR))) 7 29 srctree := $(patsubst %/,%,$(dir $(srctree))) ··· 45 23 46 24 all: $(OBJTOOL) 47 25 26 26 + WARNINGS := -Werror -Wall -Wextra -Wmissing-prototypes \ 27 27 + -Wmissing-declarations -Wwrite-strings \ 28 28 + -Wno-implicit-fallthrough -Wno-sign-compare \ 29 29 + -Wno-unused-parameter 30 30 + 48 31 INCLUDES := -I$(srctree)/tools/include \ 49 32 -I$(srctree)/tools/include/uapi \ 50 33 -I$(srctree)/tools/arch/$(HOSTARCH)/include/uapi \ ··· 57 30 -I$(srctree)/tools/objtool/include \ 58 31 -I$(srctree)/tools/objtool/arch/$(SRCARCH)/include \ 59 32 -I$(LIBSUBCMD_OUTPUT)/include 60 60 - # Note, EXTRA_WARNINGS here was determined for CC and not HOSTCC, it 61 61 - # is passed here to match a legacy behavior. 62 62 - WARNINGS := $(EXTRA_WARNINGS) -Wno-switch-default -Wno-switch-enum -Wno-packed -Wno-nested-externs 63 63 - OBJTOOL_CFLAGS := -Werror $(WARNINGS) $(KBUILD_HOSTCFLAGS) -g $(INCLUDES) $(LIBELF_FLAGS) 64 64 - OBJTOOL_LDFLAGS := $(LIBELF_LIBS) $(LIBSUBCMD) $(KBUILD_HOSTLDFLAGS) 33 33 + 34 34 + OBJTOOL_CFLAGS := -std=gnu11 -fomit-frame-pointer -O2 -g $(WARNINGS) \ 35 35 + $(INCLUDES) $(LIBELF_FLAGS) $(LIBXXHASH_CFLAGS) $(HOSTCFLAGS) 36 36 + 37 37 + OBJTOOL_LDFLAGS := $(LIBSUBCMD) $(LIBELF_LIBS) $(LIBXXHASH_LIBS) $(HOSTLDFLAGS) 65 38 66 39 # Allow old libelf to be used: 67 40 elfshdr := $(shell echo '$(pound)include <libelf.h>' | $(HOSTCC) $(OBJTOOL_CFLAGS) -x c -E - 2>/dev/null | grep elf_getshdr) ··· 70 43 # Always want host compilation. 71 44 HOST_OVERRIDES := CC="$(HOSTCC)" LD="$(HOSTLD)" AR="$(HOSTAR)" 72 45 46 46 + # 47 47 + # To support disassembly, objtool needs libopcodes which is provided 48 48 + # with libbdf (binutils-dev or binutils-devel package). 49 49 + # 50 50 + FEATURE_USER = .objtool 51 51 + FEATURE_TESTS = libbfd disassembler-init-styled 52 52 + FEATURE_DISPLAY = 53 53 + include $(srctree)/tools/build/Makefile.feature 54 54 + 55 55 + ifeq ($(feature-disassembler-init-styled), 1) 56 56 + OBJTOOL_CFLAGS += -DDISASM_INIT_STYLED 57 57 + endif 58 58 + 59 59 + BUILD_DISAS := n 60 60 + 61 61 + ifeq ($(feature-libbfd),1) 62 62 + BUILD_DISAS := y 63 63 + OBJTOOL_CFLAGS += -DDISAS -DPACKAGE="objtool" 64 64 + OBJTOOL_LDFLAGS += -lopcodes 65 65 + endif 66 66 + 67 67 + export BUILD_DISAS 68 68 + 73 69 AWK = awk 74 70 MKDIR = mkdir 75 71 76 76 - BUILD_ORC := n 77 77 - 78 78 - ifeq ($(SRCARCH),x86) 79 79 - BUILD_ORC := y 80 80 - endif 81 81 - 82 82 - ifeq ($(SRCARCH),loongarch) 83 83 - BUILD_ORC := y 84 84 - endif 85 85 - 86 86 - export BUILD_ORC 87 72 export srctree OUTPUT CFLAGS SRCARCH AWK 88 73 include $(srctree)/tools/build/Makefile.include 89 74 ··· 125 86 clean: $(LIBSUBCMD)-clean 126 87 $(call QUIET_CLEAN, objtool) $(RM) $(OBJTOOL) 127 88 $(Q)find $(OUTPUT) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete 89 89 + $(Q)$(RM) $(OUTPUT)arch/x86/lib/cpu-feature-names.c $(OUTPUT)fixdep 128 90 $(Q)$(RM) $(OUTPUT)arch/x86/lib/inat-tables.c $(OUTPUT)fixdep 91 91 + $(Q)$(RM) -- $(OUTPUT)FEATURE-DUMP.objtool 92 92 + $(Q)$(RM) -r -- $(OUTPUT)feature 129 93 130 94 FORCE: 131 95

+26 -3

tools/objtool/arch/loongarch/decode.c

reviewed

··· 1 1 // SPDX-License-Identifier: GPL-2.0-or-later 2 2 #include <string.h> 3 3 #include <objtool/check.h> 4 4 + #include <objtool/disas.h> 4 5 #include <objtool/warn.h> 5 6 #include <asm/inst.h> 6 7 #include <asm/orc_types.h> 7 8 #include <linux/objtool_types.h> 8 9 #include <arch/elf.h> 9 10 10 10 - int arch_ftrace_match(char *name) 11 11 + const char *arch_reg_name[CFI_NUM_REGS] = { 12 12 + "zero", "ra", "tp", "sp", 13 13 + "a0", "a1", "a2", "a3", 14 14 + "a4", "a5", "a6", "a7", 15 15 + "t0", "t1", "t2", "t3", 16 16 + "t4", "t5", "t6", "t7", 17 17 + "t8", "u0", "fp", "s0", 18 18 + "s1", "s2", "s3", "s4", 19 19 + "s5", "s6", "s7", "s8" 20 20 + }; 21 21 + 22 22 + int arch_ftrace_match(const char *name) 11 23 { 12 24 return !strcmp(name, "_mcount"); 13 25 } ··· 29 17 return insn->offset + (insn->immediate << 2); 30 18 } 31 19 32 32 - unsigned long arch_dest_reloc_offset(int addend) 20 20 + s64 arch_insn_adjusted_addend(struct instruction *insn, struct reloc *reloc) 33 21 { 34 34 - return addend; 22 22 + return reloc_addend(reloc); 35 23 } 36 24 37 25 bool arch_pc_relative_reloc(struct reloc *reloc) ··· 426 414 return reloc->sym->offset + reloc_addend(reloc); 427 415 } 428 416 } 417 417 + 418 418 + #ifdef DISAS 419 419 + 420 420 + int arch_disas_info_init(struct disassemble_info *dinfo) 421 421 + { 422 422 + return disas_info_init(dinfo, bfd_arch_loongarch, 423 423 + bfd_mach_loongarch32, bfd_mach_loongarch64, 424 424 + NULL); 425 425 + } 426 426 + 427 427 + #endif /* DISAS */

-1

tools/objtool/arch/loongarch/orc.c

reviewed

··· 5 5 #include <objtool/check.h> 6 6 #include <objtool/orc.h> 7 7 #include <objtool/warn.h> 8 8 - #include <objtool/endianness.h> 9 8 10 9 int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, struct instruction *insn) 11 10 {

+5

tools/objtool/arch/loongarch/special.c

reviewed

··· 194 194 195 195 return rodata_reloc; 196 196 } 197 197 + 198 198 + const char *arch_cpu_feature_name(int feature_number) 199 199 + { 200 200 + return NULL; 201 201 + }

+27 -4

tools/objtool/arch/powerpc/decode.c

reviewed

··· 3 3 #include <stdio.h> 4 4 #include <stdlib.h> 5 5 #include <objtool/check.h> 6 6 + #include <objtool/disas.h> 6 7 #include <objtool/elf.h> 7 8 #include <objtool/arch.h> 8 9 #include <objtool/warn.h> 9 10 #include <objtool/builtin.h> 10 10 - #include <objtool/endianness.h> 11 11 12 12 - int arch_ftrace_match(char *name) 12 12 + const char *arch_reg_name[CFI_NUM_REGS] = { 13 13 + "r0", "sp", "r2", "r3", 14 14 + "r4", "r5", "r6", "r7", 15 15 + "r8", "r9", "r10", "r11", 16 16 + "r12", "r13", "r14", "r15", 17 17 + "r16", "r17", "r18", "r19", 18 18 + "r20", "r21", "r22", "r23", 19 19 + "r24", "r25", "r26", "r27", 20 20 + "r28", "r29", "r30", "r31", 21 21 + "ra" 22 22 + }; 23 23 + 24 24 + int arch_ftrace_match(const char *name) 13 25 { 14 26 return !strcmp(name, "_mcount"); 15 27 } 16 28 17 17 - unsigned long arch_dest_reloc_offset(int addend) 29 29 + s64 arch_insn_adjusted_addend(struct instruction *insn, struct reloc *reloc) 18 30 { 19 19 - return addend; 31 31 + return reloc_addend(reloc); 20 32 } 21 33 22 34 bool arch_callee_saved_reg(unsigned char reg) ··· 140 128 return 8; 141 129 } 142 130 } 131 131 + 132 132 + #ifdef DISAS 133 133 + 134 134 + int arch_disas_info_init(struct disassemble_info *dinfo) 135 135 + { 136 136 + return disas_info_init(dinfo, bfd_arch_powerpc, 137 137 + bfd_mach_ppc, bfd_mach_ppc64, 138 138 + NULL); 139 139 + } 140 140 + 141 141 + #endif /* DISAS */

+5

tools/objtool/arch/powerpc/special.c

reviewed

··· 18 18 { 19 19 exit(-1); 20 20 } 21 21 + 22 22 + const char *arch_cpu_feature_name(int feature_number) 23 23 + { 24 24 + return NULL; 25 25 + }

+12 -1

tools/objtool/arch/x86/Build

reviewed

··· 1 1 - objtool-y += special.o 2 1 objtool-y += decode.o 2 2 + objtool-y += special.o 3 3 objtool-y += orc.o 4 4 5 5 inat_tables_script = ../arch/x86/tools/gen-insn-attr-x86.awk ··· 12 12 $(OUTPUT)arch/x86/decode.o: $(OUTPUT)arch/x86/lib/inat-tables.c 13 13 14 14 CFLAGS_decode.o += -I$(OUTPUT)arch/x86/lib 15 15 + 16 16 + cpu_features = ../arch/x86/include/asm/cpufeatures.h 17 17 + cpu_features_script = ../arch/x86/tools/gen-cpu-feature-names-x86.awk 18 18 + 19 19 + $(OUTPUT)arch/x86/lib/cpu-feature-names.c: $(cpu_features_script) $(cpu_features) 20 20 + $(call rule_mkdir) 21 21 + $(Q)$(call echo-cmd,gen)$(AWK) -f $(cpu_features_script) $(cpu_features) > $@ 22 22 + 23 23 + $(OUTPUT)arch/x86/special.o: $(OUTPUT)arch/x86/lib/cpu-feature-names.c 24 24 + 25 25 + CFLAGS_special.o += -I$(OUTPUT)arch/x86/lib

+94 -17

tools/objtool/arch/x86/decode.c

reviewed

··· 16 16 17 17 #include <asm/orc_types.h> 18 18 #include <objtool/check.h> 19 19 + #include <objtool/disas.h> 19 20 #include <objtool/elf.h> 20 21 #include <objtool/arch.h> 21 22 #include <objtool/warn.h> 22 22 - #include <objtool/endianness.h> 23 23 #include <objtool/builtin.h> 24 24 #include <arch/elf.h> 25 25 26 26 - int arch_ftrace_match(char *name) 26 26 + const char *arch_reg_name[CFI_NUM_REGS] = { 27 27 + "rax", "rcx", "rdx", "rbx", 28 28 + "rsp", "rbp", "rsi", "rdi", 29 29 + "r8", "r9", "r10", "r11", 30 30 + "r12", "r13", "r14", "r15", 31 31 + "ra" 32 32 + }; 33 33 + 34 34 + int arch_ftrace_match(const char *name) 27 35 { 28 36 return !strcmp(name, "__fentry__"); 29 37 } ··· 76 68 } 77 69 } 78 70 79 79 - unsigned long arch_dest_reloc_offset(int addend) 71 71 + /* Undo the effects of __pa_symbol() if necessary */ 72 72 + static unsigned long phys_to_virt(unsigned long pa) 80 73 { 81 81 - return addend + 4; 74 74 + s64 va = pa; 75 75 + 76 76 + if (va > 0) 77 77 + va &= ~(0x80000000); 78 78 + 79 79 + return va; 80 80 + } 81 81 + 82 82 + s64 arch_insn_adjusted_addend(struct instruction *insn, struct reloc *reloc) 83 83 + { 84 84 + s64 addend = reloc_addend(reloc); 85 85 + 86 86 + if (arch_pc_relative_reloc(reloc)) 87 87 + addend += insn->offset + insn->len - reloc_offset(reloc); 88 88 + 89 89 + return phys_to_virt(addend); 90 90 + } 91 91 + 92 92 + static void scan_for_insn(struct section *sec, unsigned long offset, 93 93 + unsigned long *insn_off, unsigned int *insn_len) 94 94 + { 95 95 + unsigned long o = 0; 96 96 + struct insn insn; 97 97 + 98 98 + while (1) { 99 99 + 100 100 + insn_decode(&insn, sec->data->d_buf + o, sec_size(sec) - o, 101 101 + INSN_MODE_64); 102 102 + 103 103 + if (o + insn.length > offset) { 104 104 + *insn_off = o; 105 105 + *insn_len = insn.length; 106 106 + return; 107 107 + } 108 108 + 109 109 + o += insn.length; 110 110 + } 111 111 + } 112 112 + 113 113 + u64 arch_adjusted_addend(struct reloc *reloc) 114 114 + { 115 115 + unsigned int type = reloc_type(reloc); 116 116 + s64 addend = reloc_addend(reloc); 117 117 + unsigned long insn_off; 118 118 + unsigned int insn_len; 119 119 + 120 120 + if (type == R_X86_64_PLT32) 121 121 + return addend + 4; 122 122 + 123 123 + if (type != R_X86_64_PC32 || !is_text_sec(reloc->sec->base)) 124 124 + return addend; 125 125 + 126 126 + scan_for_insn(reloc->sec->base, reloc_offset(reloc), 127 127 + &insn_off, &insn_len); 128 128 + 129 129 + return addend + insn_off + insn_len - reloc_offset(reloc); 82 130 } 83 131 84 132 unsigned long arch_jump_destination(struct instruction *insn) ··· 252 188 op1 = ins.opcode.bytes[0]; 253 189 op2 = ins.opcode.bytes[1]; 254 190 op3 = ins.opcode.bytes[2]; 255 255 - 256 256 - /* 257 257 - * XXX hack, decoder is buggered and thinks 0xea is 7 bytes long. 258 258 - */ 259 259 - if (op1 == 0xea) { 260 260 - insn->len = 1; 261 261 - insn->type = INSN_BUG; 262 262 - return 0; 263 263 - } 264 191 265 192 if (ins.rex_prefix.nbytes) { 266 193 rex = ins.rex_prefix.bytes[0]; ··· 558 503 break; 559 504 560 505 case 0x90: 506 506 + if (rex_b) /* XCHG %r8, %rax */ 507 507 + break; 508 508 + 509 509 + if (prefix == 0xf3) /* REP NOP := PAUSE */ 510 510 + break; 511 511 + 561 512 insn->type = INSN_NOP; 562 513 break; 563 514 ··· 617 556 618 557 } else if (op2 == 0x0b || op2 == 0xb9) { 619 558 620 620 - /* ud2 */ 559 559 + /* ud2, ud1 */ 621 560 insn->type = INSN_BUG; 622 561 623 623 - } else if (op2 == 0x0d || op2 == 0x1f) { 562 562 + } else if (op2 == 0x1f) { 624 563 625 625 - /* nopl/nopw */ 626 626 - insn->type = INSN_NOP; 564 564 + /* 0f 1f /0 := NOPL */ 565 565 + if (modrm_reg == 0) 566 566 + insn->type = INSN_NOP; 627 567 628 568 } else if (op2 == 0x1e) { 629 569 ··· 752 690 case 0xca: /* retf */ 753 691 case 0xcb: /* retf */ 754 692 insn->type = INSN_SYSRET; 693 693 + break; 694 694 + 695 695 + case 0xd6: /* udb */ 696 696 + insn->type = INSN_BUG; 755 697 break; 756 698 757 699 case 0xe0: /* loopne */ ··· 958 892 return false; 959 893 } 960 894 } 895 895 + 896 896 + #ifdef DISAS 897 897 + 898 898 + int arch_disas_info_init(struct disassemble_info *dinfo) 899 899 + { 900 900 + return disas_info_init(dinfo, bfd_arch_i386, 901 901 + bfd_mach_i386_i386, bfd_mach_x86_64, 902 902 + "att"); 903 903 + } 904 904 + 905 905 + #endif /* DISAS */

-1

tools/objtool/arch/x86/orc.c

reviewed

··· 5 5 #include <objtool/check.h> 6 6 #include <objtool/orc.h> 7 7 #include <objtool/warn.h> 8 8 - #include <objtool/endianness.h> 9 8 10 9 int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, struct instruction *insn) 11 10 {

+11 -1

tools/objtool/arch/x86/special.c

reviewed

··· 4 4 #include <objtool/special.h> 5 5 #include <objtool/builtin.h> 6 6 #include <objtool/warn.h> 7 7 + #include <asm/cpufeatures.h> 8 8 + 9 9 + /* cpu feature name array generated from cpufeatures.h */ 10 10 + #include "cpu-feature-names.c" 7 11 8 12 void arch_handle_alternative(struct special_alt *alt) 9 13 { ··· 93 89 /* look for a relocation which references .rodata */ 94 90 text_reloc = find_reloc_by_dest_range(file->elf, insn->sec, 95 91 insn->offset, insn->len); 96 96 - if (!text_reloc || text_reloc->sym->type != STT_SECTION || 92 92 + if (!text_reloc || !is_sec_sym(text_reloc->sym) || 97 93 !text_reloc->sym->sec->rodata) 98 94 return NULL; 99 95 ··· 137 133 138 134 *table_size = 0; 139 135 return rodata_reloc; 136 136 + } 137 137 + 138 138 + const char *arch_cpu_feature_name(int feature_number) 139 139 + { 140 140 + return (feature_number < ARRAY_SIZE(cpu_feature_names)) ? 141 141 + cpu_feature_names[feature_number] : NULL; 140 142 }

+59 -43

tools/objtool/builtin-check.c

reviewed

··· 73 73 74 74 static const struct option check_options[] = { 75 75 OPT_GROUP("Actions:"), 76 76 + OPT_BOOLEAN(0, "checksum", &opts.checksum, "generate per-function checksums"), 77 77 + OPT_BOOLEAN(0, "cfi", &opts.cfi, "annotate kernel control flow integrity (kCFI) function preambles"), 78 78 + OPT_STRING_OPTARG('d', "disas", &opts.disas, "function-pattern", "disassemble functions", "*"), 76 79 OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr,skylake", "patch toolchain bugs/limitations", parse_hacks), 77 77 - OPT_BOOLEAN('i', "ibt", &opts.ibt, "validate and annotate IBT"), 78 78 - OPT_BOOLEAN('m', "mcount", &opts.mcount, "annotate mcount/fentry calls for ftrace"), 79 79 - OPT_BOOLEAN('n', "noinstr", &opts.noinstr, "validate noinstr rules"), 80 80 - OPT_BOOLEAN(0, "orc", &opts.orc, "generate ORC metadata"), 81 81 - OPT_BOOLEAN('r', "retpoline", &opts.retpoline, "validate and annotate retpoline usage"), 82 82 - OPT_BOOLEAN(0, "rethunk", &opts.rethunk, "validate and annotate rethunk usage"), 83 83 - OPT_BOOLEAN(0, "unret", &opts.unret, "validate entry unret placement"), 84 84 - OPT_INTEGER(0, "prefix", &opts.prefix, "generate prefix symbols"), 85 85 - OPT_BOOLEAN('l', "sls", &opts.sls, "validate straight-line-speculation mitigations"), 86 86 - OPT_BOOLEAN('s', "stackval", &opts.stackval, "validate frame pointer rules"), 87 87 - OPT_BOOLEAN('t', "static-call", &opts.static_call, "annotate static calls"), 88 88 - OPT_BOOLEAN('u', "uaccess", &opts.uaccess, "validate uaccess rules for SMAP"), 89 89 - OPT_BOOLEAN(0 , "cfi", &opts.cfi, "annotate kernel control flow integrity (kCFI) function preambles"), 90 90 - OPT_BOOLEAN(0 , "noabs", &opts.noabs, "reject absolute references in allocatable sections"), 91 91 - OPT_CALLBACK_OPTARG(0, "dump", NULL, NULL, "orc", "dump metadata", parse_dump), 80 80 + OPT_BOOLEAN('i', "ibt", &opts.ibt, "validate and annotate IBT"), 81 81 + OPT_BOOLEAN('m', "mcount", &opts.mcount, "annotate mcount/fentry calls for ftrace"), 82 82 + OPT_BOOLEAN(0, "noabs", &opts.noabs, "reject absolute references in allocatable sections"), 83 83 + OPT_BOOLEAN('n', "noinstr", &opts.noinstr, "validate noinstr rules"), 84 84 + OPT_BOOLEAN(0, "orc", &opts.orc, "generate ORC metadata"), 85 85 + OPT_BOOLEAN('r', "retpoline", &opts.retpoline, "validate and annotate retpoline usage"), 86 86 + OPT_BOOLEAN(0, "rethunk", &opts.rethunk, "validate and annotate rethunk usage"), 87 87 + OPT_BOOLEAN(0, "unret", &opts.unret, "validate entry unret placement"), 88 88 + OPT_INTEGER(0, "prefix", &opts.prefix, "generate prefix symbols"), 89 89 + OPT_BOOLEAN('l', "sls", &opts.sls, "validate straight-line-speculation mitigations"), 90 90 + OPT_BOOLEAN('s', "stackval", &opts.stackval, "validate frame pointer rules"), 91 91 + OPT_BOOLEAN('t', "static-call", &opts.static_call, "annotate static calls"), 92 92 + OPT_BOOLEAN('u', "uaccess", &opts.uaccess, "validate uaccess rules for SMAP"), 93 93 + OPT_CALLBACK_OPTARG(0, "dump", NULL, NULL, "orc", "dump metadata", parse_dump), 92 94 93 95 OPT_GROUP("Options:"), 94 94 - OPT_BOOLEAN(0, "backtrace", &opts.backtrace, "unwind on error"), 95 95 - OPT_BOOLEAN(0, "dry-run", &opts.dryrun, "don't write modifications"), 96 96 - OPT_BOOLEAN(0, "link", &opts.link, "object is a linked object"), 97 97 - OPT_BOOLEAN(0, "module", &opts.module, "object is part of a kernel module"), 98 98 - OPT_BOOLEAN(0, "mnop", &opts.mnop, "nop out mcount call sites"), 99 99 - OPT_BOOLEAN(0, "no-unreachable", &opts.no_unreachable, "skip 'unreachable instruction' warnings"), 100 100 - OPT_STRING('o', "output", &opts.output, "file", "output file name"), 101 101 - OPT_BOOLEAN(0, "sec-address", &opts.sec_address, "print section addresses in warnings"), 102 102 - OPT_BOOLEAN(0, "stats", &opts.stats, "print statistics"), 103 103 - OPT_BOOLEAN('v', "verbose", &opts.verbose, "verbose warnings"), 104 104 - OPT_BOOLEAN(0, "Werror", &opts.werror, "return error on warnings"), 96 96 + OPT_BOOLEAN(0, "backtrace", &opts.backtrace, "unwind on error"), 97 97 + OPT_BOOLEAN(0, "backup", &opts.backup, "create backup (.orig) file on warning/error"), 98 98 + OPT_STRING(0, "debug-checksum", &opts.debug_checksum, "funcs", "enable checksum debug output"), 99 99 + OPT_BOOLEAN(0, "dry-run", &opts.dryrun, "don't write modifications"), 100 100 + OPT_BOOLEAN(0, "link", &opts.link, "object is a linked object"), 101 101 + OPT_BOOLEAN(0, "module", &opts.module, "object is part of a kernel module"), 102 102 + OPT_BOOLEAN(0, "mnop", &opts.mnop, "nop out mcount call sites"), 103 103 + OPT_BOOLEAN(0, "no-unreachable", &opts.no_unreachable, "skip 'unreachable instruction' warnings"), 104 104 + OPT_STRING('o', "output", &opts.output, "file", "output file name"), 105 105 + OPT_BOOLEAN(0, "sec-address", &opts.sec_address, "print section addresses in warnings"), 106 106 + OPT_BOOLEAN(0, "stats", &opts.stats, "print statistics"), 107 107 + OPT_STRING(0, "trace", &opts.trace, "func", "trace function validation"), 108 108 + OPT_BOOLEAN('v', "verbose", &opts.verbose, "verbose warnings"), 109 109 + OPT_BOOLEAN(0, "werror", &opts.werror, "return error on warnings"), 110 110 + OPT_BOOLEAN(0, "wide", &opts.wide, "wide output"), 105 111 106 112 OPT_END(), 107 113 }; ··· 165 159 return false; 166 160 } 167 161 168 168 - if (opts.hack_jump_label || 162 162 + #ifndef BUILD_KLP 163 163 + if (opts.checksum) { 164 164 + ERROR("--checksum not supported; install xxhash-devel/libxxhash-dev (version >= 0.8) and recompile"); 165 165 + return false; 166 166 + } 167 167 + #endif 168 168 + 169 169 + if (opts.debug_checksum && !opts.checksum) { 170 170 + ERROR("--debug-checksum requires --checksum"); 171 171 + return false; 172 172 + } 173 173 + 174 174 + if (opts.checksum || 175 175 + opts.disas || 176 176 + opts.hack_jump_label || 169 177 opts.hack_noinstr || 170 178 opts.ibt || 171 179 opts.mcount || ··· 263 243 ERROR_GLIBC("strdup(%s)", argv[i]); 264 244 exit(1); 265 245 } 266 266 - }; 246 246 + } 267 247 } 268 248 269 269 - void print_args(void) 249 249 + int make_backup(void) 270 250 { 271 271 - char *backup = NULL; 272 272 - 273 273 - if (opts.output || opts.dryrun) 274 274 - goto print; 251 251 + char *backup; 275 252 276 253 /* 277 254 * Make a backup before kbuild deletes the file so the error ··· 277 260 backup = malloc(strlen(objname) + strlen(ORIG_SUFFIX) + 1); 278 261 if (!backup) { 279 262 ERROR_GLIBC("malloc"); 280 280 - goto print; 263 263 + return 1; 281 264 } 282 265 283 266 strcpy(backup, objname); 284 267 strcat(backup, ORIG_SUFFIX); 285 285 - if (copy_file(objname, backup)) { 286 286 - backup = NULL; 287 287 - goto print; 288 288 - } 268 268 + if (copy_file(objname, backup)) 269 269 + return 1; 289 270 290 290 - print: 291 271 /* 292 292 - * Print the cmdline args to make it easier to recreate. If '--output' 293 293 - * wasn't used, add it to the printed args with the backup as input. 272 272 + * Print the cmdline args to make it easier to recreate. 294 273 */ 274 274 + 295 275 fprintf(stderr, "%s", orig_argv[0]); 296 276 297 277 for (int i = 1; i < orig_argc; i++) { 298 278 char *arg = orig_argv[i]; 299 279 300 300 - if (backup && !strcmp(arg, objname)) 280 280 + /* Modify the printed args to use the backup */ 281 281 + if (!opts.output && !strcmp(arg, objname)) 301 282 fprintf(stderr, " %s -o %s", backup, objname); 302 283 else 303 284 fprintf(stderr, " %s", arg); 304 285 } 305 286 306 287 fprintf(stderr, "\n"); 288 288 + return 0; 307 289 } 308 290 309 291 int objtool_run(int argc, const char **argv) ··· 348 332 if (!opts.dryrun && file->elf->changed && elf_write(file->elf)) 349 333 return 1; 350 334 351 351 - return 0; 335 335 + return elf_close(file->elf); 352 336 }

+53

tools/objtool/builtin-klp.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + #include <subcmd/parse-options.h> 3 3 + #include <string.h> 4 4 + #include <stdlib.h> 5 5 + #include <objtool/builtin.h> 6 6 + #include <objtool/objtool.h> 7 7 + #include <objtool/klp.h> 8 8 + 9 9 + struct subcmd { 10 10 + const char *name; 11 11 + const char *description; 12 12 + int (*fn)(int, const char **); 13 13 + }; 14 14 + 15 15 + static struct subcmd subcmds[] = { 16 16 + { "diff", "Generate binary diff of two object files", cmd_klp_diff, }, 17 17 + { "post-link", "Finalize klp symbols/relocs after module linking", cmd_klp_post_link, }, 18 18 + }; 19 19 + 20 20 + static void cmd_klp_usage(void) 21 21 + { 22 22 + fprintf(stderr, "usage: objtool klp <subcommand> [<options>]\n\n"); 23 23 + fprintf(stderr, "Subcommands:\n"); 24 24 + 25 25 + for (int i = 0; i < ARRAY_SIZE(subcmds); i++) { 26 26 + struct subcmd *cmd = &subcmds[i]; 27 27 + 28 28 + fprintf(stderr, " %s\t%s\n", cmd->name, cmd->description); 29 29 + } 30 30 + 31 31 + exit(1); 32 32 + } 33 33 + 34 34 + int cmd_klp(int argc, const char **argv) 35 35 + { 36 36 + argc--; 37 37 + argv++; 38 38 + 39 39 + if (!argc) 40 40 + cmd_klp_usage(); 41 41 + 42 42 + if (argc) { 43 43 + for (int i = 0; i < ARRAY_SIZE(subcmds); i++) { 44 44 + struct subcmd *cmd = &subcmds[i]; 45 45 + 46 46 + if (!strcmp(cmd->name, argv[0])) 47 47 + return cmd->fn(argc, argv); 48 48 + } 49 49 + } 50 50 + 51 51 + cmd_klp_usage(); 52 52 + return 0; 53 53 + }

+885 -677

tools/objtool/check.c

reviewed

··· 3 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 4 */ 5 5 6 6 + #define _GNU_SOURCE /* memmem() */ 7 7 + #include <fnmatch.h> 6 8 #include <string.h> 7 9 #include <stdlib.h> 8 10 #include <inttypes.h> ··· 13 11 #include <objtool/builtin.h> 14 12 #include <objtool/cfi.h> 15 13 #include <objtool/arch.h> 14 14 + #include <objtool/disas.h> 16 15 #include <objtool/check.h> 17 16 #include <objtool/special.h> 17 17 + #include <objtool/trace.h> 18 18 #include <objtool/warn.h> 19 19 - #include <objtool/endianness.h> 19 19 + #include <objtool/checksum.h> 20 20 + #include <objtool/util.h> 20 21 21 22 #include <linux/objtool_types.h> 22 23 #include <linux/hashtable.h> ··· 27 22 #include <linux/static_call_types.h> 28 23 #include <linux/string.h> 29 24 30 30 - struct alternative { 31 31 - struct alternative *next; 32 32 - struct instruction *insn; 33 33 - }; 34 34 - 35 25 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 36 26 37 27 static struct cfi_init_state initial_func_cfi; 38 28 static struct cfi_state init_cfi; 39 29 static struct cfi_state func_cfi; 40 30 static struct cfi_state force_undefined_cfi; 31 31 + 32 32 + struct disas_context *objtool_disas_ctx; 33 33 + 34 34 + size_t sym_name_max_len; 41 35 42 36 struct instruction *find_insn(struct objtool_file *file, 43 37 struct section *sec, unsigned long offset) ··· 110 106 #define for_each_insn(file, insn) \ 111 107 for (struct section *__sec, *__fake = (struct section *)1; \ 112 108 __fake; __fake = NULL) \ 113 113 - for_each_sec(file, __sec) \ 109 109 + for_each_sec(file->elf, __sec) \ 114 110 sec_for_each_insn(file, __sec, insn) 115 111 116 112 #define func_for_each_insn(file, func, insn) \ ··· 134 130 #define sec_for_each_insn_continue(file, insn) \ 135 131 for (insn = next_insn_same_sec(file, insn); insn; \ 136 132 insn = next_insn_same_sec(file, insn)) 137 137 - 138 138 - static inline struct symbol *insn_call_dest(struct instruction *insn) 139 139 - { 140 140 - if (insn->type == INSN_JUMP_DYNAMIC || 141 141 - insn->type == INSN_CALL_DYNAMIC) 142 142 - return NULL; 143 143 - 144 144 - return insn->_call_dest; 145 145 - } 146 133 147 134 static inline struct reloc *insn_jump_table(struct instruction *insn) 148 135 { ··· 178 183 179 184 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */ 180 185 return (is_static_jump(insn) && insn_call_dest(insn)); 181 181 - } 182 182 - 183 183 - /* 184 184 - * Checks if a string ends with another. 185 185 - */ 186 186 - static bool str_ends_with(const char *s, const char *sub) 187 187 - { 188 188 - const int slen = strlen(s); 189 189 - const int sublen = strlen(sub); 190 190 - 191 191 - if (sublen > slen) 192 192 - return 0; 193 193 - 194 194 - return !memcmp(s + slen - sublen, sub, sublen); 195 186 } 196 187 197 188 /* ··· 243 262 if (!func) 244 263 return false; 245 264 246 246 - if (func->bind == STB_GLOBAL || func->bind == STB_WEAK) { 265 265 + if (!is_local_sym(func)) { 247 266 if (is_rust_noreturn(func)) 248 267 return true; 249 268 ··· 252 271 return true; 253 272 } 254 273 255 255 - if (func->bind == STB_WEAK) 274 274 + if (is_weak_sym(func)) 256 275 return false; 257 276 258 277 if (!func->len) ··· 412 431 struct symbol *func; 413 432 unsigned long offset; 414 433 struct instruction *insn; 415 415 - int ret; 416 434 417 417 - for_each_sec(file, sec) { 435 435 + for_each_sec(file->elf, sec) { 418 436 struct instruction *insns = NULL; 419 437 u8 prev_len = 0; 420 438 u8 idx = 0; 421 439 422 422 - if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 440 440 + if (!is_text_sec(sec)) 423 441 continue; 424 442 425 443 if (strcmp(sec->name, ".altinstr_replacement") && ··· 441 461 if (!strcmp(sec->name, ".init.text") && !opts.module) 442 462 sec->init = true; 443 463 444 444 - for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) { 464 464 + for (offset = 0; offset < sec_size(sec); offset += insn->len) { 445 465 if (!insns || idx == INSN_CHUNK_MAX) { 446 446 - insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE); 466 466 + insns = calloc(INSN_CHUNK_SIZE, sizeof(*insn)); 447 467 if (!insns) { 448 468 ERROR_GLIBC("calloc"); 449 469 return -1; ··· 460 480 insn->offset = offset; 461 481 insn->prev_len = prev_len; 462 482 463 463 - ret = arch_decode_instruction(file, sec, offset, 464 464 - sec->sh.sh_size - offset, 465 465 - insn); 466 466 - if (ret) 467 467 - return ret; 483 483 + if (arch_decode_instruction(file, sec, offset, sec_size(sec) - offset, insn)) 484 484 + return -1; 468 485 469 486 prev_len = insn->len; 470 487 ··· 478 501 } 479 502 480 503 sec_for_each_sym(sec, func) { 481 481 - if (func->type != STT_NOTYPE && func->type != STT_FUNC) 504 504 + if (!is_notype_sym(func) && !is_func_sym(func)) 482 505 continue; 483 506 484 484 - if (func->offset == sec->sh.sh_size) { 507 507 + if (func->offset == sec_size(sec)) { 485 508 /* Heuristic: likely an "end" symbol */ 486 486 - if (func->type == STT_NOTYPE) 509 509 + if (is_notype_sym(func)) 487 510 continue; 488 511 ERROR("%s(): STT_FUNC at end of section", func->name); 489 512 return -1; ··· 499 522 500 523 sym_for_each_insn(file, func, insn) { 501 524 insn->sym = func; 502 502 - if (func->type == STT_FUNC && 525 525 + if (is_func_sym(func) && 503 526 insn->type == INSN_ENDBR && 504 527 list_empty(&insn->call_node)) { 505 528 if (insn->offset == func->offset) { ··· 543 566 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long); 544 567 545 568 func = reloc->sym; 546 546 - if (func->type == STT_SECTION) 569 569 + if (is_sec_sym(func)) 547 570 func = find_symbol_by_offset(reloc->sym->sec, 548 571 reloc_addend(reloc)); 549 572 if (!func) { ··· 577 600 }; 578 601 const char *pv_ops; 579 602 struct symbol *sym; 580 580 - int idx, nr, ret; 603 603 + int idx, nr; 581 604 582 605 if (!opts.noinstr) 583 606 return 0; ··· 589 612 return 0; 590 613 591 614 nr = sym->len / sizeof(unsigned long); 592 592 - file->pv_ops = calloc(sizeof(struct pv_state), nr); 615 615 + file->pv_ops = calloc(nr, sizeof(struct pv_state)); 593 616 if (!file->pv_ops) { 594 617 ERROR_GLIBC("calloc"); 595 618 return -1; ··· 599 622 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 600 623 601 624 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) { 602 602 - ret = add_pv_ops(file, pv_ops); 603 603 - if (ret) 604 604 - return ret; 625 625 + if (add_pv_ops(file, pv_ops)) 626 626 + return -1; 605 627 } 606 628 607 629 return 0; 630 630 + } 631 631 + 632 632 + static bool is_livepatch_module(struct objtool_file *file) 633 633 + { 634 634 + struct section *sec; 635 635 + 636 636 + if (!opts.module) 637 637 + return false; 638 638 + 639 639 + sec = find_section_by_name(file->elf, ".modinfo"); 640 640 + if (!sec) 641 641 + return false; 642 642 + 643 643 + return memmem(sec->data->d_buf, sec_size(sec), "\0livepatch=Y", 12); 608 644 } 609 645 610 646 static int create_static_call_sections(struct objtool_file *file) ··· 631 641 632 642 sec = find_section_by_name(file->elf, ".static_call_sites"); 633 643 if (sec) { 634 634 - INIT_LIST_HEAD(&file->static_call_list); 635 635 - WARN("file already has .static_call_sites section, skipping"); 644 644 + /* 645 645 + * Livepatch modules may have already extracted the static call 646 646 + * site entries to take advantage of vmlinux static call 647 647 + * privileges. 648 648 + */ 649 649 + if (!file->klp) 650 650 + WARN("file already has .static_call_sites section, skipping"); 651 651 + 636 652 return 0; 637 653 } 638 654 ··· 682 686 683 687 key_sym = find_symbol_by_name(file->elf, tmp); 684 688 if (!key_sym) { 685 685 - if (!opts.module) { 689 689 + if (!opts.module || file->klp) { 686 690 ERROR("static_call: can't find static_call_key symbol: %s", tmp); 687 691 return -1; 688 692 } ··· 825 829 struct symbol *sym = insn->sym; 826 830 *site = 0; 827 831 828 828 - if (opts.module && sym && sym->type == STT_FUNC && 832 832 + if (opts.module && sym && is_func_sym(sym) && 829 833 insn->offset == sym->offset && 830 834 (!strcmp(sym->name, "init_module") || 831 835 !strcmp(sym->name, "cleanup_module"))) { ··· 853 857 854 858 sec = find_section_by_name(file->elf, ".cfi_sites"); 855 859 if (sec) { 856 856 - INIT_LIST_HEAD(&file->call_list); 857 860 WARN("file already has .cfi_sites section, skipping"); 858 861 return 0; 859 862 } 860 863 861 864 idx = 0; 862 862 - for_each_sym(file, sym) { 863 863 - if (sym->type != STT_FUNC) 865 865 + for_each_sym(file->elf, sym) { 866 866 + if (!is_func_sym(sym)) 864 867 continue; 865 868 866 869 if (strncmp(sym->name, "__cfi_", 6)) ··· 874 879 return -1; 875 880 876 881 idx = 0; 877 877 - for_each_sym(file, sym) { 878 878 - if (sym->type != STT_FUNC) 882 882 + for_each_sym(file->elf, sym) { 883 883 + if (!is_func_sym(sym)) 879 884 continue; 880 885 881 886 if (strncmp(sym->name, "__cfi_", 6)) ··· 901 906 902 907 sec = find_section_by_name(file->elf, "__mcount_loc"); 903 908 if (sec) { 904 904 - INIT_LIST_HEAD(&file->mcount_loc_list); 905 905 - WARN("file already has __mcount_loc section, skipping"); 909 909 + /* 910 910 + * Livepatch modules have already extracted their __mcount_loc 911 911 + * entries to cover the !CONFIG_FTRACE_MCOUNT_USE_OBJTOOL case. 912 912 + */ 913 913 + if (!file->klp) 914 914 + WARN("file already has __mcount_loc section, skipping"); 915 915 + 906 916 return 0; 907 917 } 908 918 ··· 951 951 952 952 sec = find_section_by_name(file->elf, ".call_sites"); 953 953 if (sec) { 954 954 - INIT_LIST_HEAD(&file->call_list); 955 954 WARN("file already has .call_sites section, skipping"); 956 955 return 0; 957 956 } ··· 980 981 981 982 return 0; 982 983 } 984 984 + 985 985 + #ifdef BUILD_KLP 986 986 + static int create_sym_checksum_section(struct objtool_file *file) 987 987 + { 988 988 + struct section *sec; 989 989 + struct symbol *sym; 990 990 + unsigned int idx = 0; 991 991 + struct sym_checksum *checksum; 992 992 + size_t entsize = sizeof(struct sym_checksum); 993 993 + 994 994 + sec = find_section_by_name(file->elf, ".discard.sym_checksum"); 995 995 + if (sec) { 996 996 + if (!opts.dryrun) 997 997 + WARN("file already has .discard.sym_checksum section, skipping"); 998 998 + 999 999 + return 0; 1000 1000 + } 1001 1001 + 1002 1002 + for_each_sym(file->elf, sym) 1003 1003 + if (sym->csum.checksum) 1004 1004 + idx++; 1005 1005 + 1006 1006 + if (!idx) 1007 1007 + return 0; 1008 1008 + 1009 1009 + sec = elf_create_section_pair(file->elf, ".discard.sym_checksum", entsize, 1010 1010 + idx, idx); 1011 1011 + if (!sec) 1012 1012 + return -1; 1013 1013 + 1014 1014 + idx = 0; 1015 1015 + for_each_sym(file->elf, sym) { 1016 1016 + if (!sym->csum.checksum) 1017 1017 + continue; 1018 1018 + 1019 1019 + if (!elf_init_reloc(file->elf, sec->rsec, idx, idx * entsize, 1020 1020 + sym, 0, R_TEXT64)) 1021 1021 + return -1; 1022 1022 + 1023 1023 + checksum = (struct sym_checksum *)sec->data->d_buf + idx; 1024 1024 + checksum->addr = 0; /* reloc */ 1025 1025 + checksum->checksum = sym->csum.checksum; 1026 1026 + 1027 1027 + mark_sec_changed(file->elf, sec, true); 1028 1028 + 1029 1029 + idx++; 1030 1030 + } 1031 1031 + 1032 1032 + return 0; 1033 1033 + } 1034 1034 + #else 1035 1035 + static int create_sym_checksum_section(struct objtool_file *file) { return -EINVAL; } 1036 1036 + #endif 983 1037 984 1038 /* 985 1039 * Warnings shouldn't be reported for ignored functions. ··· 1485 1433 } 1486 1434 1487 1435 static bool is_first_func_insn(struct objtool_file *file, 1488 1488 - struct instruction *insn, struct symbol *sym) 1436 1436 + struct instruction *insn) 1489 1437 { 1490 1490 - if (insn->offset == sym->offset) 1438 1438 + struct symbol *func = insn_func(insn); 1439 1439 + 1440 1440 + if (!func) 1441 1441 + return false; 1442 1442 + 1443 1443 + if (insn->offset == func->offset) 1491 1444 return true; 1492 1445 1493 1446 /* Allow direct CALL/JMP past ENDBR */ ··· 1500 1443 struct instruction *prev = prev_insn_same_sym(file, insn); 1501 1444 1502 1445 if (prev && prev->type == INSN_ENDBR && 1503 1503 - insn->offset == sym->offset + prev->len) 1446 1446 + insn->offset == func->offset + prev->len) 1504 1447 return true; 1505 1448 } 1506 1449 ··· 1508 1451 } 1509 1452 1510 1453 /* 1511 1511 - * A sibling call is a tail-call to another symbol -- to differentiate from a 1512 1512 - * recursive tail-call which is to the same symbol. 1513 1513 - */ 1514 1514 - static bool jump_is_sibling_call(struct objtool_file *file, 1515 1515 - struct instruction *from, struct instruction *to) 1516 1516 - { 1517 1517 - struct symbol *fs = from->sym; 1518 1518 - struct symbol *ts = to->sym; 1519 1519 - 1520 1520 - /* Not a sibling call if from/to a symbol hole */ 1521 1521 - if (!fs || !ts) 1522 1522 - return false; 1523 1523 - 1524 1524 - /* Not a sibling call if not targeting the start of a symbol. */ 1525 1525 - if (!is_first_func_insn(file, to, ts)) 1526 1526 - return false; 1527 1527 - 1528 1528 - /* Disallow sibling calls into STT_NOTYPE */ 1529 1529 - if (ts->type == STT_NOTYPE) 1530 1530 - return false; 1531 1531 - 1532 1532 - /* Must not be self to be a sibling */ 1533 1533 - return fs->pfunc != ts->pfunc; 1534 1534 - } 1535 1535 - 1536 1536 - /* 1537 1454 * Find the destination instructions for all jumps. 1538 1455 */ 1539 1456 static int add_jump_destinations(struct objtool_file *file) 1540 1457 { 1541 1541 - struct instruction *insn, *jump_dest; 1458 1458 + struct instruction *insn; 1542 1459 struct reloc *reloc; 1543 1543 - struct section *dest_sec; 1544 1544 - unsigned long dest_off; 1545 1545 - int ret; 1546 1460 1547 1461 for_each_insn(file, insn) { 1548 1462 struct symbol *func = insn_func(insn); 1463 1463 + struct instruction *dest_insn; 1464 1464 + struct section *dest_sec; 1465 1465 + struct symbol *dest_sym; 1466 1466 + unsigned long dest_off; 1467 1467 + 1468 1468 + if (!is_static_jump(insn)) 1469 1469 + continue; 1549 1470 1550 1471 if (insn->jump_dest) { 1551 1472 /* ··· 1532 1497 */ 1533 1498 continue; 1534 1499 } 1535 1535 - if (!is_static_jump(insn)) 1536 1536 - continue; 1537 1500 1538 1501 reloc = insn_reloc(file, insn); 1539 1502 if (!reloc) { 1540 1503 dest_sec = insn->sec; 1541 1504 dest_off = arch_jump_destination(insn); 1542 1542 - } else if (reloc->sym->type == STT_SECTION) { 1543 1543 - dest_sec = reloc->sym->sec; 1544 1544 - dest_off = arch_dest_reloc_offset(reloc_addend(reloc)); 1545 1545 - } else if (reloc->sym->retpoline_thunk) { 1546 1546 - ret = add_retpoline_call(file, insn); 1547 1547 - if (ret) 1548 1548 - return ret; 1549 1549 - continue; 1550 1550 - } else if (reloc->sym->return_thunk) { 1551 1551 - add_return_call(file, insn, true); 1552 1552 - continue; 1553 1553 - } else if (func) { 1554 1554 - /* 1555 1555 - * External sibling call or internal sibling call with 1556 1556 - * STT_FUNC reloc. 1557 1557 - */ 1558 1558 - ret = add_call_dest(file, insn, reloc->sym, true); 1559 1559 - if (ret) 1560 1560 - return ret; 1561 1561 - continue; 1562 1562 - } else if (reloc->sym->sec->idx) { 1563 1563 - dest_sec = reloc->sym->sec; 1564 1564 - dest_off = reloc->sym->sym.st_value + 1565 1565 - arch_dest_reloc_offset(reloc_addend(reloc)); 1505 1505 + dest_sym = dest_sec->sym; 1566 1506 } else { 1567 1567 - /* non-func asm code jumping to another file */ 1568 1568 - continue; 1507 1507 + dest_sym = reloc->sym; 1508 1508 + if (is_undef_sym(dest_sym)) { 1509 1509 + if (dest_sym->retpoline_thunk) { 1510 1510 + if (add_retpoline_call(file, insn)) 1511 1511 + return -1; 1512 1512 + continue; 1513 1513 + } 1514 1514 + 1515 1515 + if (dest_sym->return_thunk) { 1516 1516 + add_return_call(file, insn, true); 1517 1517 + continue; 1518 1518 + } 1519 1519 + 1520 1520 + /* External symbol */ 1521 1521 + if (func) { 1522 1522 + /* External sibling call */ 1523 1523 + if (add_call_dest(file, insn, dest_sym, true)) 1524 1524 + return -1; 1525 1525 + continue; 1526 1526 + } 1527 1527 + 1528 1528 + /* Non-func asm code jumping to external symbol */ 1529 1529 + continue; 1530 1530 + } 1531 1531 + 1532 1532 + dest_sec = dest_sym->sec; 1533 1533 + dest_off = dest_sym->offset + arch_insn_adjusted_addend(insn, reloc); 1569 1534 } 1570 1535 1571 1571 - jump_dest = find_insn(file, dest_sec, dest_off); 1572 1572 - if (!jump_dest) { 1536 1536 + dest_insn = find_insn(file, dest_sec, dest_off); 1537 1537 + if (!dest_insn) { 1573 1538 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); 1574 1539 1575 1540 /* 1576 1576 - * This is a special case for retbleed_untrain_ret(). 1577 1577 - * It jumps to __x86_return_thunk(), but objtool 1578 1578 - * can't find the thunk's starting RET 1579 1579 - * instruction, because the RET is also in the 1580 1580 - * middle of another instruction. Objtool only 1581 1581 - * knows about the outer instruction. 1541 1541 + * retbleed_untrain_ret() jumps to 1542 1542 + * __x86_return_thunk(), but objtool can't find 1543 1543 + * the thunk's starting RET instruction, 1544 1544 + * because the RET is also in the middle of 1545 1545 + * another instruction. Objtool only knows 1546 1546 + * about the outer instruction. 1582 1547 */ 1583 1548 if (sym && sym->embedded_insn) { 1584 1549 add_return_call(file, insn, false); ··· 1586 1551 } 1587 1552 1588 1553 /* 1589 1589 - * GCOV/KCOV dead code can jump to the end of the 1590 1590 - * function/section. 1554 1554 + * GCOV/KCOV dead code can jump to the end of 1555 1555 + * the function/section. 1591 1556 */ 1592 1557 if (file->ignore_unreachables && func && 1593 1558 dest_sec == insn->sec && 1594 1559 dest_off == func->offset + func->len) 1595 1560 continue; 1596 1561 1597 1597 - ERROR_INSN(insn, "can't find jump dest instruction at %s+0x%lx", 1598 1598 - dest_sec->name, dest_off); 1562 1562 + ERROR_INSN(insn, "can't find jump dest instruction at %s", 1563 1563 + offstr(dest_sec, dest_off)); 1599 1564 return -1; 1600 1565 } 1601 1566 1602 1602 - /* 1603 1603 - * An intra-TU jump in retpoline.o might not have a relocation 1604 1604 - * for its jump dest, in which case the above 1605 1605 - * add_{retpoline,return}_call() didn't happen. 1606 1606 - */ 1607 1607 - if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) { 1608 1608 - if (jump_dest->sym->retpoline_thunk) { 1609 1609 - ret = add_retpoline_call(file, insn); 1610 1610 - if (ret) 1611 1611 - return ret; 1612 1612 - continue; 1613 1613 - } 1614 1614 - if (jump_dest->sym->return_thunk) { 1615 1615 - add_return_call(file, insn, true); 1616 1616 - continue; 1617 1617 - } 1567 1567 + if (!dest_sym || is_sec_sym(dest_sym)) { 1568 1568 + dest_sym = dest_insn->sym; 1569 1569 + if (!dest_sym) 1570 1570 + goto set_jump_dest; 1618 1571 } 1619 1572 1620 1620 - /* 1621 1621 - * Cross-function jump. 1622 1622 - */ 1623 1623 - if (func && insn_func(jump_dest) && func != insn_func(jump_dest)) { 1624 1624 - 1625 1625 - /* 1626 1626 - * For GCC 8+, create parent/child links for any cold 1627 1627 - * subfunctions. This is _mostly_ redundant with a 1628 1628 - * similar initialization in read_symbols(). 1629 1629 - * 1630 1630 - * If a function has aliases, we want the *first* such 1631 1631 - * function in the symbol table to be the subfunction's 1632 1632 - * parent. In that case we overwrite the 1633 1633 - * initialization done in read_symbols(). 1634 1634 - * 1635 1635 - * However this code can't completely replace the 1636 1636 - * read_symbols() code because this doesn't detect the 1637 1637 - * case where the parent function's only reference to a 1638 1638 - * subfunction is through a jump table. 1639 1639 - */ 1640 1640 - if (!strstr(func->name, ".cold") && 1641 1641 - strstr(insn_func(jump_dest)->name, ".cold")) { 1642 1642 - func->cfunc = insn_func(jump_dest); 1643 1643 - insn_func(jump_dest)->pfunc = func; 1644 1644 - } 1645 1645 - } 1646 1646 - 1647 1647 - if (jump_is_sibling_call(file, insn, jump_dest)) { 1648 1648 - /* 1649 1649 - * Internal sibling call without reloc or with 1650 1650 - * STT_SECTION reloc. 1651 1651 - */ 1652 1652 - ret = add_call_dest(file, insn, insn_func(jump_dest), true); 1653 1653 - if (ret) 1654 1654 - return ret; 1573 1573 + if (dest_sym->retpoline_thunk && dest_insn->offset == dest_sym->offset) { 1574 1574 + if (add_retpoline_call(file, insn)) 1575 1575 + return -1; 1655 1576 continue; 1656 1577 } 1657 1578 1658 1658 - insn->jump_dest = jump_dest; 1579 1579 + if (dest_sym->return_thunk && dest_insn->offset == dest_sym->offset) { 1580 1580 + add_return_call(file, insn, true); 1581 1581 + continue; 1582 1582 + } 1583 1583 + 1584 1584 + if (!insn->sym || insn->sym->pfunc == dest_sym->pfunc) 1585 1585 + goto set_jump_dest; 1586 1586 + 1587 1587 + /* 1588 1588 + * Internal cross-function jump. 1589 1589 + */ 1590 1590 + 1591 1591 + if (is_first_func_insn(file, dest_insn)) { 1592 1592 + /* Internal sibling call */ 1593 1593 + if (add_call_dest(file, insn, dest_sym, true)) 1594 1594 + return -1; 1595 1595 + continue; 1596 1596 + } 1597 1597 + 1598 1598 + set_jump_dest: 1599 1599 + insn->jump_dest = dest_insn; 1659 1600 } 1660 1601 1661 1602 return 0; ··· 1657 1646 unsigned long dest_off; 1658 1647 struct symbol *dest; 1659 1648 struct reloc *reloc; 1660 1660 - int ret; 1661 1649 1662 1650 for_each_insn(file, insn) { 1663 1651 struct symbol *func = insn_func(insn); ··· 1668 1658 dest_off = arch_jump_destination(insn); 1669 1659 dest = find_call_destination(insn->sec, dest_off); 1670 1660 1671 1671 - ret = add_call_dest(file, insn, dest, false); 1672 1672 - if (ret) 1673 1673 - return ret; 1661 1661 + if (add_call_dest(file, insn, dest, false)) 1662 1662 + return -1; 1674 1663 1675 1664 if (func && func->ignore) 1676 1665 continue; ··· 1679 1670 return -1; 1680 1671 } 1681 1672 1682 1682 - if (func && insn_call_dest(insn)->type != STT_FUNC) { 1673 1673 + if (func && !is_func_sym(insn_call_dest(insn))) { 1683 1674 ERROR_INSN(insn, "unsupported call to non-function"); 1684 1675 return -1; 1685 1676 } 1686 1677 1687 1687 - } else if (reloc->sym->type == STT_SECTION) { 1688 1688 - dest_off = arch_dest_reloc_offset(reloc_addend(reloc)); 1678 1678 + } else if (is_sec_sym(reloc->sym)) { 1679 1679 + dest_off = arch_insn_adjusted_addend(insn, reloc); 1689 1680 dest = find_call_destination(reloc->sym->sec, dest_off); 1690 1681 if (!dest) { 1691 1682 ERROR_INSN(insn, "can't find call dest symbol at %s+0x%lx", ··· 1693 1684 return -1; 1694 1685 } 1695 1686 1696 1696 - ret = add_call_dest(file, insn, dest, false); 1697 1697 - if (ret) 1698 1698 - return ret; 1687 1687 + if (add_call_dest(file, insn, dest, false)) 1688 1688 + return -1; 1699 1689 1700 1690 } else if (reloc->sym->retpoline_thunk) { 1701 1701 - ret = add_retpoline_call(file, insn); 1702 1702 - if (ret) 1703 1703 - return ret; 1691 1691 + if (add_retpoline_call(file, insn)) 1692 1692 + return -1; 1704 1693 1705 1694 } else { 1706 1706 - ret = add_call_dest(file, insn, reloc->sym, false); 1707 1707 - if (ret) 1708 1708 - return ret; 1695 1695 + if (add_call_dest(file, insn, reloc->sym, false)) 1696 1696 + return -1; 1709 1697 } 1710 1698 } 1711 1699 ··· 1751 1745 orig_alt_group->last_insn = last_orig_insn; 1752 1746 orig_alt_group->nop = NULL; 1753 1747 orig_alt_group->ignore = orig_insn->ignore_alts; 1748 1748 + orig_alt_group->feature = 0; 1754 1749 } else { 1755 1750 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - 1756 1751 orig_alt_group->first_insn->offset != special_alt->orig_len) { ··· 1791 1784 nop->type = INSN_NOP; 1792 1785 nop->sym = orig_insn->sym; 1793 1786 nop->alt_group = new_alt_group; 1787 1787 + nop->fake = 1; 1794 1788 } 1795 1789 1796 1790 if (!special_alt->new_len) { ··· 1856 1848 new_alt_group->nop = nop; 1857 1849 new_alt_group->ignore = (*new_insn)->ignore_alts; 1858 1850 new_alt_group->cfi = orig_alt_group->cfi; 1851 1851 + new_alt_group->feature = special_alt->feature; 1859 1852 return 0; 1860 1853 } 1861 1854 ··· 1921 1912 struct list_head special_alts; 1922 1913 struct instruction *orig_insn, *new_insn; 1923 1914 struct special_alt *special_alt, *tmp; 1915 1915 + enum alternative_type alt_type; 1924 1916 struct alternative *alt; 1925 1925 - int ret; 1917 1917 + struct alternative *a; 1926 1918 1927 1919 if (special_get_alts(file->elf, &special_alts)) 1928 1920 return -1; ··· 1955 1945 continue; 1956 1946 } 1957 1947 1958 1958 - ret = handle_group_alt(file, special_alt, orig_insn, 1959 1959 - &new_insn); 1960 1960 - if (ret) 1961 1961 - return ret; 1948 1948 + if (handle_group_alt(file, special_alt, orig_insn, &new_insn)) 1949 1949 + return -1; 1950 1950 + 1951 1951 + alt_type = ALT_TYPE_INSTRUCTIONS; 1962 1952 1963 1953 } else if (special_alt->jump_or_nop) { 1964 1964 - ret = handle_jump_alt(file, special_alt, orig_insn, 1965 1965 - &new_insn); 1966 1966 - if (ret) 1967 1967 - return ret; 1954 1954 + if (handle_jump_alt(file, special_alt, orig_insn, &new_insn)) 1955 1955 + return -1; 1956 1956 + 1957 1957 + alt_type = ALT_TYPE_JUMP_TABLE; 1958 1958 + } else { 1959 1959 + alt_type = ALT_TYPE_EX_TABLE; 1968 1960 } 1969 1961 1970 1962 alt = calloc(1, sizeof(*alt)); ··· 1976 1964 } 1977 1965 1978 1966 alt->insn = new_insn; 1979 1979 - alt->next = orig_insn->alts; 1980 1980 - orig_insn->alts = alt; 1967 1967 + alt->type = alt_type; 1968 1968 + alt->next = NULL; 1969 1969 + 1970 1970 + /* 1971 1971 + * Store alternatives in the same order they have been 1972 1972 + * defined. 1973 1973 + */ 1974 1974 + if (!orig_insn->alts) { 1975 1975 + orig_insn->alts = alt; 1976 1976 + } else { 1977 1977 + for (a = orig_insn->alts; a->next; a = a->next) 1978 1978 + ; 1979 1979 + a->next = alt; 1980 1980 + } 1981 1981 1982 1982 list_del(&special_alt->list); 1983 1983 free(special_alt); ··· 2166 2142 struct symbol *func) 2167 2143 { 2168 2144 struct instruction *insn; 2169 2169 - int ret; 2170 2145 2171 2146 func_for_each_insn(file, func, insn) { 2172 2147 if (!insn_jump_table(insn)) 2173 2148 continue; 2174 2149 2175 2175 - ret = add_jump_table(file, insn); 2176 2176 - if (ret) 2177 2177 - return ret; 2150 2150 + if (add_jump_table(file, insn)) 2151 2151 + return -1; 2178 2152 } 2179 2153 2180 2154 return 0; ··· 2186 2164 static int add_jump_table_alts(struct objtool_file *file) 2187 2165 { 2188 2166 struct symbol *func; 2189 2189 - int ret; 2190 2167 2191 2168 if (!file->rodata) 2192 2169 return 0; 2193 2170 2194 2194 - for_each_sym(file, func) { 2195 2195 - if (func->type != STT_FUNC) 2171 2171 + for_each_sym(file->elf, func) { 2172 2172 + if (!is_func_sym(func) || func->alias != func) 2196 2173 continue; 2197 2174 2198 2175 mark_func_jump_tables(file, func); 2199 2199 - ret = add_func_jump_tables(file, func); 2200 2200 - if (ret) 2201 2201 - return ret; 2176 2176 + if (add_func_jump_tables(file, func)) 2177 2177 + return -1; 2202 2178 } 2203 2179 2204 2180 return 0; ··· 2230 2210 return -1; 2231 2211 } 2232 2212 2233 2233 - if (sec->sh.sh_size % sizeof(struct unwind_hint)) { 2213 2213 + if (sec_size(sec) % sizeof(struct unwind_hint)) { 2234 2214 ERROR("struct unwind_hint size mismatch"); 2235 2215 return -1; 2236 2216 } 2237 2217 2238 2218 file->hints = true; 2239 2219 2240 2240 - for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) { 2220 2220 + for (i = 0; i < sec_size(sec) / sizeof(struct unwind_hint); i++) { 2241 2221 hint = (struct unwind_hint *)sec->data->d_buf + i; 2242 2222 2243 2223 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); ··· 2246 2226 return -1; 2247 2227 } 2248 2228 2249 2249 - if (reloc->sym->type == STT_SECTION) { 2250 2250 - offset = reloc_addend(reloc); 2251 2251 - } else if (reloc->sym->local_label) { 2252 2252 - offset = reloc->sym->offset; 2253 2253 - } else { 2254 2254 - ERROR("unexpected relocation symbol type in %s", sec->rsec->name); 2255 2255 - return -1; 2256 2256 - } 2229 2229 + offset = reloc->sym->offset + reloc_addend(reloc); 2257 2230 2258 2231 insn = find_insn(file, reloc->sym->sec, offset); 2259 2232 if (!insn) { ··· 2275 2262 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 2276 2263 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 2277 2264 2278 2278 - if (sym && sym->bind == STB_GLOBAL) { 2265 2265 + if (sym && is_global_sym(sym)) { 2279 2266 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { 2280 2267 ERROR_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR"); 2281 2268 return -1; ··· 2313 2300 struct instruction *insn; 2314 2301 struct reloc *reloc; 2315 2302 uint64_t offset; 2316 2316 - int type, ret; 2303 2303 + int type; 2317 2304 2318 2305 sec = find_section_by_name(file->elf, ".discard.annotate_insn"); 2319 2306 if (!sec) ··· 2331 2318 sec->sh.sh_entsize = 8; 2332 2319 } 2333 2320 2334 2334 - for_each_reloc(sec->rsec, reloc) { 2335 2335 - type = *(u32 *)(sec->data->d_buf + (reloc_idx(reloc) * sec->sh.sh_entsize) + 4); 2336 2336 - type = bswap_if_needed(file->elf, type); 2321 2321 + if (sec_num_entries(sec) != sec_num_entries(sec->rsec)) { 2322 2322 + ERROR("bad .discard.annotate_insn section: missing relocs"); 2323 2323 + return -1; 2324 2324 + } 2337 2325 2326 2326 + for_each_reloc(sec->rsec, reloc) { 2327 2327 + type = annotype(file->elf, sec, reloc); 2338 2328 offset = reloc->sym->offset + reloc_addend(reloc); 2339 2329 insn = find_insn(file, reloc->sym->sec, offset); 2340 2330 ··· 2346 2330 return -1; 2347 2331 } 2348 2332 2349 2349 - ret = func(file, type, insn); 2350 2350 - if (ret < 0) 2351 2351 - return ret; 2333 2333 + if (func(file, type, insn)) 2334 2334 + return -1; 2352 2335 } 2353 2336 2354 2337 return 0; ··· 2486 2471 static int classify_symbols(struct objtool_file *file) 2487 2472 { 2488 2473 struct symbol *func; 2474 2474 + size_t len; 2489 2475 2490 2490 - for_each_sym(file, func) { 2491 2491 - if (func->type == STT_NOTYPE && strstarts(func->name, ".L")) 2476 2476 + for_each_sym(file->elf, func) { 2477 2477 + if (is_notype_sym(func) && strstarts(func->name, ".L")) 2492 2478 func->local_label = true; 2493 2479 2494 2494 - if (func->bind != STB_GLOBAL) 2480 2480 + if (!is_global_sym(func)) 2495 2481 continue; 2496 2482 2497 2483 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, ··· 2513 2497 2514 2498 if (is_profiling_func(func->name)) 2515 2499 func->profiling_func = true; 2500 2500 + 2501 2501 + len = strlen(func->name); 2502 2502 + if (len > sym_name_max_len) 2503 2503 + sym_name_max_len = len; 2516 2504 } 2517 2505 2518 2506 return 0; ··· 2537 2517 * 2538 2518 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2539 2519 */ 2540 2540 - for_each_sec(file, sec) { 2520 2520 + for_each_sec(file->elf, sec) { 2541 2521 if ((!strncmp(sec->name, ".rodata", 7) && 2542 2522 !strstr(sec->name, ".str1.")) || 2543 2523 !strncmp(sec->name, ".data.rel.ro", 12)) { ··· 2549 2529 file->rodata = found; 2550 2530 } 2551 2531 2532 2532 + static void mark_holes(struct objtool_file *file) 2533 2533 + { 2534 2534 + struct instruction *insn; 2535 2535 + bool in_hole = false; 2536 2536 + 2537 2537 + if (!opts.link) 2538 2538 + return; 2539 2539 + 2540 2540 + /* 2541 2541 + * Whole archive runs might encounter dead code from weak symbols. 2542 2542 + * This is where the linker will have dropped the weak symbol in 2543 2543 + * favour of a regular symbol, but leaves the code in place. 2544 2544 + */ 2545 2545 + for_each_insn(file, insn) { 2546 2546 + if (insn->sym || !find_symbol_hole_containing(insn->sec, insn->offset)) { 2547 2547 + in_hole = false; 2548 2548 + continue; 2549 2549 + } 2550 2550 + 2551 2551 + /* Skip function padding and pfx code */ 2552 2552 + if (!in_hole && insn->type == INSN_NOP) 2553 2553 + continue; 2554 2554 + 2555 2555 + in_hole = true; 2556 2556 + insn->hole = 1; 2557 2557 + 2558 2558 + /* 2559 2559 + * If this hole jumps to a .cold function, mark it ignore. 2560 2560 + */ 2561 2561 + if (insn->jump_dest) { 2562 2562 + struct symbol *dest_func = insn_func(insn->jump_dest); 2563 2563 + 2564 2564 + if (dest_func && dest_func->cold) 2565 2565 + dest_func->ignore = true; 2566 2566 + } 2567 2567 + } 2568 2568 + } 2569 2569 + 2570 2570 + static bool validate_branch_enabled(void) 2571 2571 + { 2572 2572 + return opts.stackval || 2573 2573 + opts.orc || 2574 2574 + opts.uaccess || 2575 2575 + opts.checksum; 2576 2576 + } 2577 2577 + 2552 2578 static int decode_sections(struct objtool_file *file) 2553 2579 { 2554 2554 - int ret; 2580 2580 + file->klp = is_livepatch_module(file); 2555 2581 2556 2582 mark_rodata(file); 2557 2583 2558 2558 - ret = init_pv_ops(file); 2559 2559 - if (ret) 2560 2560 - return ret; 2584 2584 + if (init_pv_ops(file)) 2585 2585 + return -1; 2561 2586 2562 2587 /* 2563 2588 * Must be before add_{jump_call}_destination. 2564 2589 */ 2565 2565 - ret = classify_symbols(file); 2566 2566 - if (ret) 2567 2567 - return ret; 2590 2590 + if (classify_symbols(file)) 2591 2591 + return -1; 2568 2592 2569 2569 - ret = decode_instructions(file); 2570 2570 - if (ret) 2571 2571 - return ret; 2593 2593 + if (decode_instructions(file)) 2594 2594 + return -1; 2572 2595 2573 2573 - ret = add_ignores(file); 2574 2574 - if (ret) 2575 2575 - return ret; 2596 2596 + if (add_ignores(file)) 2597 2597 + return -1; 2576 2598 2577 2599 add_uaccess_safe(file); 2578 2600 2579 2579 - ret = read_annotate(file, __annotate_early); 2580 2580 - if (ret) 2581 2581 - return ret; 2601 2601 + if (read_annotate(file, __annotate_early)) 2602 2602 + return -1; 2582 2603 2583 2604 /* 2584 2605 * Must be before add_jump_destinations(), which depends on 'func' 2585 2606 * being set for alternatives, to enable proper sibling call detection. 2586 2607 */ 2587 2587 - if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) { 2588 2588 - ret = add_special_section_alts(file); 2589 2589 - if (ret) 2590 2590 - return ret; 2608 2608 + if (validate_branch_enabled() || opts.noinstr || opts.hack_jump_label || opts.disas) { 2609 2609 + if (add_special_section_alts(file)) 2610 2610 + return -1; 2591 2611 } 2592 2612 2593 2593 - ret = add_jump_destinations(file); 2594 2594 - if (ret) 2595 2595 - return ret; 2613 2613 + if (add_jump_destinations(file)) 2614 2614 + return -1; 2596 2615 2597 2616 /* 2598 2617 * Must be before add_call_destination(); it changes INSN_CALL to 2599 2618 * INSN_JUMP. 2600 2619 */ 2601 2601 - ret = read_annotate(file, __annotate_ifc); 2602 2602 - if (ret) 2603 2603 - return ret; 2620 2620 + if (read_annotate(file, __annotate_ifc)) 2621 2621 + return -1; 2604 2622 2605 2605 - ret = add_call_destinations(file); 2606 2606 - if (ret) 2607 2607 - return ret; 2623 2623 + if (add_call_destinations(file)) 2624 2624 + return -1; 2608 2625 2609 2609 - ret = add_jump_table_alts(file); 2610 2610 - if (ret) 2611 2611 - return ret; 2626 2626 + if (add_jump_table_alts(file)) 2627 2627 + return -1; 2612 2628 2613 2613 - ret = read_unwind_hints(file); 2614 2614 - if (ret) 2615 2615 - return ret; 2629 2629 + if (read_unwind_hints(file)) 2630 2630 + return -1; 2631 2631 + 2632 2632 + /* Must be after add_jump_destinations() */ 2633 2633 + mark_holes(file); 2616 2634 2617 2635 /* 2618 2636 * Must be after add_call_destinations() such that it can override 2619 2637 * dead_end_function() marks. 2620 2638 */ 2621 2621 - ret = read_annotate(file, __annotate_late); 2622 2622 - if (ret) 2623 2623 - return ret; 2639 2639 + if (read_annotate(file, __annotate_late)) 2640 2640 + return -1; 2624 2641 2625 2642 return 0; 2626 2643 } ··· 3411 3354 if (!reloc || strcmp(reloc->sym->name, "pv_ops")) 3412 3355 return false; 3413 3356 3414 3414 - idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *)); 3357 3357 + idx = arch_insn_adjusted_addend(insn, reloc) / sizeof(void *); 3415 3358 3416 3359 if (file->pv_ops[idx].clean) 3417 3360 return true; ··· 3577 3520 return false; 3578 3521 3579 3522 /* ANNOTATE_IGNORE_ALTERNATIVE */ 3580 3580 - if (insn->alt_group->ignore) 3523 3523 + if (insn->alt_group->ignore) { 3524 3524 + TRACE_ALT(insn, "alt group ignored"); 3581 3525 return true; 3526 3526 + } 3582 3527 3583 3528 /* 3584 3529 * For NOP patched with CLAC/STAC, only follow the latter to avoid ··· 3602 3543 return alt_insn->type == INSN_CLAC || alt_insn->type == INSN_STAC; 3603 3544 } 3604 3545 3546 3546 + static int checksum_debug_init(struct objtool_file *file) 3547 3547 + { 3548 3548 + char *dup, *s; 3549 3549 + 3550 3550 + if (!opts.debug_checksum) 3551 3551 + return 0; 3552 3552 + 3553 3553 + dup = strdup(opts.debug_checksum); 3554 3554 + if (!dup) { 3555 3555 + ERROR_GLIBC("strdup"); 3556 3556 + return -1; 3557 3557 + } 3558 3558 + 3559 3559 + s = dup; 3560 3560 + while (*s) { 3561 3561 + struct symbol *func; 3562 3562 + char *comma; 3563 3563 + 3564 3564 + comma = strchr(s, ','); 3565 3565 + if (comma) 3566 3566 + *comma = '\0'; 3567 3567 + 3568 3568 + func = find_symbol_by_name(file->elf, s); 3569 3569 + if (!func || !is_func_sym(func)) 3570 3570 + WARN("--debug-checksum: can't find '%s'", s); 3571 3571 + else 3572 3572 + func->debug_checksum = 1; 3573 3573 + 3574 3574 + if (!comma) 3575 3575 + break; 3576 3576 + 3577 3577 + s = comma + 1; 3578 3578 + } 3579 3579 + 3580 3580 + free(dup); 3581 3581 + return 0; 3582 3582 + } 3583 3583 + 3584 3584 + static void checksum_update_insn(struct objtool_file *file, struct symbol *func, 3585 3585 + struct instruction *insn) 3586 3586 + { 3587 3587 + struct reloc *reloc = insn_reloc(file, insn); 3588 3588 + unsigned long offset; 3589 3589 + struct symbol *sym; 3590 3590 + 3591 3591 + if (insn->fake) 3592 3592 + return; 3593 3593 + 3594 3594 + checksum_update(func, insn, insn->sec->data->d_buf + insn->offset, insn->len); 3595 3595 + 3596 3596 + if (!reloc) { 3597 3597 + struct symbol *call_dest = insn_call_dest(insn); 3598 3598 + 3599 3599 + if (call_dest) 3600 3600 + checksum_update(func, insn, call_dest->demangled_name, 3601 3601 + strlen(call_dest->demangled_name)); 3602 3602 + return; 3603 3603 + } 3604 3604 + 3605 3605 + sym = reloc->sym; 3606 3606 + offset = arch_insn_adjusted_addend(insn, reloc); 3607 3607 + 3608 3608 + if (is_string_sec(sym->sec)) { 3609 3609 + char *str; 3610 3610 + 3611 3611 + str = sym->sec->data->d_buf + sym->offset + offset; 3612 3612 + checksum_update(func, insn, str, strlen(str)); 3613 3613 + return; 3614 3614 + } 3615 3615 + 3616 3616 + if (is_sec_sym(sym)) { 3617 3617 + sym = find_symbol_containing(reloc->sym->sec, offset); 3618 3618 + if (!sym) 3619 3619 + return; 3620 3620 + 3621 3621 + offset -= sym->offset; 3622 3622 + } 3623 3623 + 3624 3624 + checksum_update(func, insn, sym->demangled_name, strlen(sym->demangled_name)); 3625 3625 + checksum_update(func, insn, &offset, sizeof(offset)); 3626 3626 + } 3627 3627 + 3628 3628 + static int validate_branch(struct objtool_file *file, struct symbol *func, 3629 3629 + struct instruction *insn, struct insn_state state); 3630 3630 + static int do_validate_branch(struct objtool_file *file, struct symbol *func, 3631 3631 + struct instruction *insn, struct insn_state state); 3632 3632 + 3633 3633 + static int validate_insn(struct objtool_file *file, struct symbol *func, 3634 3634 + struct instruction *insn, struct insn_state *statep, 3635 3635 + struct instruction *prev_insn, struct instruction *next_insn, 3636 3636 + bool *dead_end) 3637 3637 + { 3638 3638 + /* prev_state and alt_name are not used if there is no disassembly support */ 3639 3639 + struct insn_state prev_state __maybe_unused; 3640 3640 + char *alt_name __maybe_unused = NULL; 3641 3641 + struct alternative *alt; 3642 3642 + u8 visited; 3643 3643 + int ret; 3644 3644 + 3645 3645 + /* 3646 3646 + * Any returns before the end of this function are effectively dead 3647 3647 + * ends, i.e. validate_branch() has reached the end of the branch. 3648 3648 + */ 3649 3649 + *dead_end = true; 3650 3650 + 3651 3651 + visited = VISITED_BRANCH << statep->uaccess; 3652 3652 + if (insn->visited & VISITED_BRANCH_MASK) { 3653 3653 + if (!insn->hint && !insn_cfi_match(insn, &statep->cfi)) 3654 3654 + return 1; 3655 3655 + 3656 3656 + if (insn->visited & visited) { 3657 3657 + TRACE_INSN(insn, "already visited"); 3658 3658 + return 0; 3659 3659 + } 3660 3660 + } else { 3661 3661 + nr_insns_visited++; 3662 3662 + } 3663 3663 + 3664 3664 + if (statep->noinstr) 3665 3665 + statep->instr += insn->instr; 3666 3666 + 3667 3667 + if (insn->hint) { 3668 3668 + if (insn->restore) { 3669 3669 + struct instruction *save_insn, *i; 3670 3670 + 3671 3671 + i = insn; 3672 3672 + save_insn = NULL; 3673 3673 + 3674 3674 + sym_for_each_insn_continue_reverse(file, func, i) { 3675 3675 + if (i->save) { 3676 3676 + save_insn = i; 3677 3677 + break; 3678 3678 + } 3679 3679 + } 3680 3680 + 3681 3681 + if (!save_insn) { 3682 3682 + WARN_INSN(insn, "no corresponding CFI save for CFI restore"); 3683 3683 + return 1; 3684 3684 + } 3685 3685 + 3686 3686 + if (!save_insn->visited) { 3687 3687 + /* 3688 3688 + * If the restore hint insn is at the 3689 3689 + * beginning of a basic block and was 3690 3690 + * branched to from elsewhere, and the 3691 3691 + * save insn hasn't been visited yet, 3692 3692 + * defer following this branch for now. 3693 3693 + * It will be seen later via the 3694 3694 + * straight-line path. 3695 3695 + */ 3696 3696 + if (!prev_insn) { 3697 3697 + TRACE_INSN(insn, "defer restore"); 3698 3698 + return 0; 3699 3699 + } 3700 3700 + 3701 3701 + WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo"); 3702 3702 + return 1; 3703 3703 + } 3704 3704 + 3705 3705 + insn->cfi = save_insn->cfi; 3706 3706 + nr_cfi_reused++; 3707 3707 + } 3708 3708 + 3709 3709 + statep->cfi = *insn->cfi; 3710 3710 + } else { 3711 3711 + /* XXX track if we actually changed statep->cfi */ 3712 3712 + 3713 3713 + if (prev_insn && !cficmp(prev_insn->cfi, &statep->cfi)) { 3714 3714 + insn->cfi = prev_insn->cfi; 3715 3715 + nr_cfi_reused++; 3716 3716 + } else { 3717 3717 + insn->cfi = cfi_hash_find_or_add(&statep->cfi); 3718 3718 + } 3719 3719 + } 3720 3720 + 3721 3721 + insn->visited |= visited; 3722 3722 + 3723 3723 + if (propagate_alt_cfi(file, insn)) 3724 3724 + return 1; 3725 3725 + 3726 3726 + if (insn->alts) { 3727 3727 + for (alt = insn->alts; alt; alt = alt->next) { 3728 3728 + TRACE_ALT_BEGIN(insn, alt, alt_name); 3729 3729 + ret = validate_branch(file, func, alt->insn, *statep); 3730 3730 + TRACE_ALT_END(insn, alt, alt_name); 3731 3731 + if (ret) { 3732 3732 + BT_INSN(insn, "(alt)"); 3733 3733 + return ret; 3734 3734 + } 3735 3735 + } 3736 3736 + TRACE_ALT_INFO_NOADDR(insn, "/ ", "DEFAULT"); 3737 3737 + } 3738 3738 + 3739 3739 + if (skip_alt_group(insn)) 3740 3740 + return 0; 3741 3741 + 3742 3742 + prev_state = *statep; 3743 3743 + ret = handle_insn_ops(insn, next_insn, statep); 3744 3744 + TRACE_INSN_STATE(insn, &prev_state, statep); 3745 3745 + 3746 3746 + if (ret) 3747 3747 + return 1; 3748 3748 + 3749 3749 + switch (insn->type) { 3750 3750 + 3751 3751 + case INSN_RETURN: 3752 3752 + TRACE_INSN(insn, "return"); 3753 3753 + return validate_return(func, insn, statep); 3754 3754 + 3755 3755 + case INSN_CALL: 3756 3756 + case INSN_CALL_DYNAMIC: 3757 3757 + if (insn->type == INSN_CALL) 3758 3758 + TRACE_INSN(insn, "call"); 3759 3759 + else 3760 3760 + TRACE_INSN(insn, "indirect call"); 3761 3761 + 3762 3762 + ret = validate_call(file, insn, statep); 3763 3763 + if (ret) 3764 3764 + return ret; 3765 3765 + 3766 3766 + if (opts.stackval && func && !is_special_call(insn) && 3767 3767 + !has_valid_stack_frame(statep)) { 3768 3768 + WARN_INSN(insn, "call without frame pointer save/setup"); 3769 3769 + return 1; 3770 3770 + } 3771 3771 + 3772 3772 + break; 3773 3773 + 3774 3774 + case INSN_JUMP_CONDITIONAL: 3775 3775 + case INSN_JUMP_UNCONDITIONAL: 3776 3776 + if (is_sibling_call(insn)) { 3777 3777 + TRACE_INSN(insn, "sibling call"); 3778 3778 + ret = validate_sibling_call(file, insn, statep); 3779 3779 + if (ret) 3780 3780 + return ret; 3781 3781 + 3782 3782 + } else if (insn->jump_dest) { 3783 3783 + if (insn->type == INSN_JUMP_UNCONDITIONAL) 3784 3784 + TRACE_INSN(insn, "unconditional jump"); 3785 3785 + else 3786 3786 + TRACE_INSN(insn, "jump taken"); 3787 3787 + 3788 3788 + ret = validate_branch(file, func, insn->jump_dest, *statep); 3789 3789 + if (ret) { 3790 3790 + BT_INSN(insn, "(branch)"); 3791 3791 + return ret; 3792 3792 + } 3793 3793 + } 3794 3794 + 3795 3795 + if (insn->type == INSN_JUMP_UNCONDITIONAL) 3796 3796 + return 0; 3797 3797 + 3798 3798 + TRACE_INSN(insn, "jump not taken"); 3799 3799 + break; 3800 3800 + 3801 3801 + case INSN_JUMP_DYNAMIC: 3802 3802 + case INSN_JUMP_DYNAMIC_CONDITIONAL: 3803 3803 + TRACE_INSN(insn, "indirect jump"); 3804 3804 + if (is_sibling_call(insn)) { 3805 3805 + ret = validate_sibling_call(file, insn, statep); 3806 3806 + if (ret) 3807 3807 + return ret; 3808 3808 + } 3809 3809 + 3810 3810 + if (insn->type == INSN_JUMP_DYNAMIC) 3811 3811 + return 0; 3812 3812 + 3813 3813 + break; 3814 3814 + 3815 3815 + case INSN_SYSCALL: 3816 3816 + TRACE_INSN(insn, "syscall"); 3817 3817 + if (func && (!next_insn || !next_insn->hint)) { 3818 3818 + WARN_INSN(insn, "unsupported instruction in callable function"); 3819 3819 + return 1; 3820 3820 + } 3821 3821 + 3822 3822 + break; 3823 3823 + 3824 3824 + case INSN_SYSRET: 3825 3825 + TRACE_INSN(insn, "sysret"); 3826 3826 + if (func && (!next_insn || !next_insn->hint)) { 3827 3827 + WARN_INSN(insn, "unsupported instruction in callable function"); 3828 3828 + return 1; 3829 3829 + } 3830 3830 + 3831 3831 + return 0; 3832 3832 + 3833 3833 + case INSN_STAC: 3834 3834 + TRACE_INSN(insn, "stac"); 3835 3835 + if (!opts.uaccess) 3836 3836 + break; 3837 3837 + 3838 3838 + if (statep->uaccess) { 3839 3839 + WARN_INSN(insn, "recursive UACCESS enable"); 3840 3840 + return 1; 3841 3841 + } 3842 3842 + 3843 3843 + statep->uaccess = true; 3844 3844 + break; 3845 3845 + 3846 3846 + case INSN_CLAC: 3847 3847 + TRACE_INSN(insn, "clac"); 3848 3848 + if (!opts.uaccess) 3849 3849 + break; 3850 3850 + 3851 3851 + if (!statep->uaccess && func) { 3852 3852 + WARN_INSN(insn, "redundant UACCESS disable"); 3853 3853 + return 1; 3854 3854 + } 3855 3855 + 3856 3856 + if (func_uaccess_safe(func) && !statep->uaccess_stack) { 3857 3857 + WARN_INSN(insn, "UACCESS-safe disables UACCESS"); 3858 3858 + return 1; 3859 3859 + } 3860 3860 + 3861 3861 + statep->uaccess = false; 3862 3862 + break; 3863 3863 + 3864 3864 + case INSN_STD: 3865 3865 + TRACE_INSN(insn, "std"); 3866 3866 + if (statep->df) { 3867 3867 + WARN_INSN(insn, "recursive STD"); 3868 3868 + return 1; 3869 3869 + } 3870 3870 + 3871 3871 + statep->df = true; 3872 3872 + break; 3873 3873 + 3874 3874 + case INSN_CLD: 3875 3875 + TRACE_INSN(insn, "cld"); 3876 3876 + if (!statep->df && func) { 3877 3877 + WARN_INSN(insn, "redundant CLD"); 3878 3878 + return 1; 3879 3879 + } 3880 3880 + 3881 3881 + statep->df = false; 3882 3882 + break; 3883 3883 + 3884 3884 + default: 3885 3885 + break; 3886 3886 + } 3887 3887 + 3888 3888 + if (insn->dead_end) 3889 3889 + TRACE_INSN(insn, "dead end"); 3890 3890 + 3891 3891 + *dead_end = insn->dead_end; 3892 3892 + return 0; 3893 3893 + } 3894 3894 + 3605 3895 /* 3606 3896 * Follow the branch starting at the given instruction, and recursively follow 3607 3897 * any other branches (jumps). Meanwhile, track the frame pointer state at 3608 3898 * each instruction and validate all the rules described in 3609 3899 * tools/objtool/Documentation/objtool.txt. 3610 3900 */ 3611 3611 - static int validate_branch(struct objtool_file *file, struct symbol *func, 3612 3612 - struct instruction *insn, struct insn_state state) 3901 3901 + static int do_validate_branch(struct objtool_file *file, struct symbol *func, 3902 3902 + struct instruction *insn, struct insn_state state) 3613 3903 { 3614 3614 - struct alternative *alt; 3615 3904 struct instruction *next_insn, *prev_insn = NULL; 3616 3616 - struct section *sec; 3617 3617 - u8 visited; 3905 3905 + bool dead_end; 3618 3906 int ret; 3619 3907 3620 3908 if (func && func->ignore) 3621 3909 return 0; 3622 3910 3623 3623 - sec = insn->sec; 3624 3624 - 3625 3625 - while (1) { 3911 3911 + do { 3912 3912 + insn->trace = 0; 3626 3913 next_insn = next_insn_to_validate(file, insn); 3914 3914 + 3915 3915 + if (opts.checksum && func && insn->sec) 3916 3916 + checksum_update_insn(file, func, insn); 3627 3917 3628 3918 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { 3629 3919 /* Ignore KCFI type preambles, which always fall through */ 3630 3630 - if (!strncmp(func->name, "__cfi_", 6) || 3631 3631 - !strncmp(func->name, "__pfx_", 6) || 3632 3632 - !strncmp(func->name, "__pi___cfi_", 11) || 3633 3633 - !strncmp(func->name, "__pi___pfx_", 11)) 3920 3920 + if (is_prefix_func(func)) 3634 3921 return 0; 3635 3922 3636 3923 if (file->ignore_unreachables) ··· 3989 3584 return 1; 3990 3585 } 3991 3586 3992 3992 - visited = VISITED_BRANCH << state.uaccess; 3993 3993 - if (insn->visited & VISITED_BRANCH_MASK) { 3994 3994 - if (!insn->hint && !insn_cfi_match(insn, &state.cfi)) 3995 3995 - return 1; 3587 3587 + ret = validate_insn(file, func, insn, &state, prev_insn, next_insn, 3588 3588 + &dead_end); 3996 3589 3997 3997 - if (insn->visited & visited) 3998 3998 - return 0; 3999 3999 - } else { 4000 4000 - nr_insns_visited++; 4001 4001 - } 4002 4002 - 4003 4003 - if (state.noinstr) 4004 4004 - state.instr += insn->instr; 4005 4005 - 4006 4006 - if (insn->hint) { 4007 4007 - if (insn->restore) { 4008 4008 - struct instruction *save_insn, *i; 4009 4009 - 4010 4010 - i = insn; 4011 4011 - save_insn = NULL; 4012 4012 - 4013 4013 - sym_for_each_insn_continue_reverse(file, func, i) { 4014 4014 - if (i->save) { 4015 4015 - save_insn = i; 4016 4016 - break; 4017 4017 - } 4018 4018 - } 4019 4019 - 4020 4020 - if (!save_insn) { 4021 4021 - WARN_INSN(insn, "no corresponding CFI save for CFI restore"); 4022 4022 - return 1; 4023 4023 - } 4024 4024 - 4025 4025 - if (!save_insn->visited) { 4026 4026 - /* 4027 4027 - * If the restore hint insn is at the 4028 4028 - * beginning of a basic block and was 4029 4029 - * branched to from elsewhere, and the 4030 4030 - * save insn hasn't been visited yet, 4031 4031 - * defer following this branch for now. 4032 4032 - * It will be seen later via the 4033 4033 - * straight-line path. 4034 4034 - */ 4035 4035 - if (!prev_insn) 4036 4036 - return 0; 4037 4037 - 4038 4038 - WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo"); 4039 4039 - return 1; 4040 4040 - } 4041 4041 - 4042 4042 - insn->cfi = save_insn->cfi; 4043 4043 - nr_cfi_reused++; 4044 4044 - } 4045 4045 - 4046 4046 - state.cfi = *insn->cfi; 4047 4047 - } else { 4048 4048 - /* XXX track if we actually changed state.cfi */ 4049 4049 - 4050 4050 - if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) { 4051 4051 - insn->cfi = prev_insn->cfi; 4052 4052 - nr_cfi_reused++; 4053 4053 - } else { 4054 4054 - insn->cfi = cfi_hash_find_or_add(&state.cfi); 4055 4055 - } 4056 4056 - } 4057 4057 - 4058 4058 - insn->visited |= visited; 4059 4059 - 4060 4060 - if (propagate_alt_cfi(file, insn)) 4061 4061 - return 1; 4062 4062 - 4063 4063 - if (insn->alts) { 4064 4064 - for (alt = insn->alts; alt; alt = alt->next) { 4065 4065 - ret = validate_branch(file, func, alt->insn, state); 4066 4066 - if (ret) { 4067 4067 - BT_INSN(insn, "(alt)"); 4068 4068 - return ret; 4069 4069 - } 4070 4070 - } 4071 4071 - } 4072 4072 - 4073 4073 - if (skip_alt_group(insn)) 4074 4074 - return 0; 4075 4075 - 4076 4076 - if (handle_insn_ops(insn, next_insn, &state)) 4077 4077 - return 1; 4078 4078 - 4079 4079 - switch (insn->type) { 4080 4080 - 4081 4081 - case INSN_RETURN: 4082 4082 - return validate_return(func, insn, &state); 4083 4083 - 4084 4084 - case INSN_CALL: 4085 4085 - case INSN_CALL_DYNAMIC: 4086 4086 - ret = validate_call(file, insn, &state); 3590 3590 + if (!insn->trace) { 4087 3591 if (ret) 4088 4088 - return ret; 4089 4089 - 4090 4090 - if (opts.stackval && func && !is_special_call(insn) && 4091 4091 - !has_valid_stack_frame(&state)) { 4092 4092 - WARN_INSN(insn, "call without frame pointer save/setup"); 4093 4093 - return 1; 4094 4094 - } 4095 4095 - 4096 4096 - break; 4097 4097 - 4098 4098 - case INSN_JUMP_CONDITIONAL: 4099 4099 - case INSN_JUMP_UNCONDITIONAL: 4100 4100 - if (is_sibling_call(insn)) { 4101 4101 - ret = validate_sibling_call(file, insn, &state); 4102 4102 - if (ret) 4103 4103 - return ret; 4104 4104 - 4105 4105 - } else if (insn->jump_dest) { 4106 4106 - ret = validate_branch(file, func, 4107 4107 - insn->jump_dest, state); 4108 4108 - if (ret) { 4109 4109 - BT_INSN(insn, "(branch)"); 4110 4110 - return ret; 4111 4111 - } 4112 4112 - } 4113 4113 - 4114 4114 - if (insn->type == INSN_JUMP_UNCONDITIONAL) 4115 4115 - return 0; 4116 4116 - 4117 4117 - break; 4118 4118 - 4119 4119 - case INSN_JUMP_DYNAMIC: 4120 4120 - case INSN_JUMP_DYNAMIC_CONDITIONAL: 4121 4121 - if (is_sibling_call(insn)) { 4122 4122 - ret = validate_sibling_call(file, insn, &state); 4123 4123 - if (ret) 4124 4124 - return ret; 4125 4125 - } 4126 4126 - 4127 4127 - if (insn->type == INSN_JUMP_DYNAMIC) 4128 4128 - return 0; 4129 4129 - 4130 4130 - break; 4131 4131 - 4132 4132 - case INSN_SYSCALL: 4133 4133 - if (func && (!next_insn || !next_insn->hint)) { 4134 4134 - WARN_INSN(insn, "unsupported instruction in callable function"); 4135 4135 - return 1; 4136 4136 - } 4137 4137 - 4138 4138 - break; 4139 4139 - 4140 4140 - case INSN_SYSRET: 4141 4141 - if (func && (!next_insn || !next_insn->hint)) { 4142 4142 - WARN_INSN(insn, "unsupported instruction in callable function"); 4143 4143 - return 1; 4144 4144 - } 4145 4145 - 4146 4146 - return 0; 4147 4147 - 4148 4148 - case INSN_STAC: 4149 4149 - if (!opts.uaccess) 4150 4150 - break; 4151 4151 - 4152 4152 - if (state.uaccess) { 4153 4153 - WARN_INSN(insn, "recursive UACCESS enable"); 4154 4154 - return 1; 4155 4155 - } 4156 4156 - 4157 4157 - state.uaccess = true; 4158 4158 - break; 4159 4159 - 4160 4160 - case INSN_CLAC: 4161 4161 - if (!opts.uaccess) 4162 4162 - break; 4163 4163 - 4164 4164 - if (!state.uaccess && func) { 4165 4165 - WARN_INSN(insn, "redundant UACCESS disable"); 4166 4166 - return 1; 4167 4167 - } 4168 4168 - 4169 4169 - if (func_uaccess_safe(func) && !state.uaccess_stack) { 4170 4170 - WARN_INSN(insn, "UACCESS-safe disables UACCESS"); 4171 4171 - return 1; 4172 4172 - } 4173 4173 - 4174 4174 - state.uaccess = false; 4175 4175 - break; 4176 4176 - 4177 4177 - case INSN_STD: 4178 4178 - if (state.df) { 4179 4179 - WARN_INSN(insn, "recursive STD"); 4180 4180 - return 1; 4181 4181 - } 4182 4182 - 4183 4183 - state.df = true; 4184 4184 - break; 4185 4185 - 4186 4186 - case INSN_CLD: 4187 4187 - if (!state.df && func) { 4188 4188 - WARN_INSN(insn, "redundant CLD"); 4189 4189 - return 1; 4190 4190 - } 4191 4191 - 4192 4192 - state.df = false; 4193 4193 - break; 4194 4194 - 4195 4195 - default: 4196 4196 - break; 3592 3592 + TRACE_INSN(insn, "warning (%d)", ret); 3593 3593 + else 3594 3594 + TRACE_INSN(insn, NULL); 4197 3595 } 4198 3596 4199 4199 - if (insn->dead_end) 4200 4200 - return 0; 4201 4201 - 4202 4202 - if (!next_insn) { 3597 3597 + if (!dead_end && !next_insn) { 4203 3598 if (state.cfi.cfa.base == CFI_UNDEFINED) 4204 3599 return 0; 4205 3600 if (file->ignore_unreachables) ··· 4007 3802 4008 3803 WARN("%s%sunexpected end of section %s", 4009 3804 func ? func->name : "", func ? "(): " : "", 4010 4010 - sec->name); 3805 3805 + insn->sec->name); 4011 3806 return 1; 4012 3807 } 4013 3808 4014 3809 prev_insn = insn; 4015 3810 insn = next_insn; 4016 4016 - } 4017 3811 4018 4018 - return 0; 3812 3812 + } while (!dead_end); 3813 3813 + 3814 3814 + return ret; 3815 3815 + } 3816 3816 + 3817 3817 + static int validate_branch(struct objtool_file *file, struct symbol *func, 3818 3818 + struct instruction *insn, struct insn_state state) 3819 3819 + { 3820 3820 + int ret; 3821 3821 + 3822 3822 + trace_depth_inc(); 3823 3823 + ret = do_validate_branch(file, func, insn, state); 3824 3824 + trace_depth_dec(); 3825 3825 + 3826 3826 + return ret; 4019 3827 } 4020 3828 4021 3829 static int validate_unwind_hint(struct objtool_file *file, ··· 4036 3818 struct insn_state *state) 4037 3819 { 4038 3820 if (insn->hint && !insn->visited) { 4039 4039 - int ret = validate_branch(file, insn_func(insn), insn, *state); 3821 3821 + struct symbol *func = insn_func(insn); 3822 3822 + int ret; 3823 3823 + 3824 3824 + if (opts.checksum) 3825 3825 + checksum_init(func); 3826 3826 + 3827 3827 + ret = validate_branch(file, func, insn, *state); 4040 3828 if (ret) 4041 3829 BT_INSN(insn, "<=== (hint)"); 4042 3830 return ret; ··· 4286 4062 struct instruction *prev_insn; 4287 4063 int i; 4288 4064 4289 4289 - if (insn->type == INSN_NOP || insn->type == INSN_TRAP || (func && func->ignore)) 4065 4065 + if (insn->type == INSN_NOP || insn->type == INSN_TRAP || 4066 4066 + insn->hole || (func && func->ignore)) 4290 4067 return true; 4291 4068 4292 4069 /* ··· 4297 4072 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 4298 4073 !strcmp(insn->sec->name, ".altinstr_aux")) 4299 4074 return true; 4300 4300 - 4301 4301 - /* 4302 4302 - * Whole archive runs might encounter dead code from weak symbols. 4303 4303 - * This is where the linker will have dropped the weak symbol in 4304 4304 - * favour of a regular symbol, but leaves the code in place. 4305 4305 - * 4306 4306 - * In this case we'll find a piece of code (whole function) that is not 4307 4307 - * covered by a !section symbol. Ignore them. 4308 4308 - */ 4309 4309 - if (opts.link && !func) { 4310 4310 - int size = find_symbol_hole_containing(insn->sec, insn->offset); 4311 4311 - unsigned long end = insn->offset + size; 4312 4312 - 4313 4313 - if (!size) /* not a hole */ 4314 4314 - return false; 4315 4315 - 4316 4316 - if (size < 0) /* hole until the end */ 4317 4317 - return true; 4318 4318 - 4319 4319 - sec_for_each_insn_continue(file, insn) { 4320 4320 - /* 4321 4321 - * If we reach a visited instruction at or before the 4322 4322 - * end of the hole, ignore the unreachable. 4323 4323 - */ 4324 4324 - if (insn->visited) 4325 4325 - return true; 4326 4326 - 4327 4327 - if (insn->offset >= end) 4328 4328 - break; 4329 4329 - 4330 4330 - /* 4331 4331 - * If this hole jumps to a .cold function, mark it ignore too. 4332 4332 - */ 4333 4333 - if (insn->jump_dest && insn_func(insn->jump_dest) && 4334 4334 - strstr(insn_func(insn->jump_dest)->name, ".cold")) { 4335 4335 - insn_func(insn->jump_dest)->ignore = true; 4336 4336 - } 4337 4337 - } 4338 4338 - 4339 4339 - return false; 4340 4340 - } 4341 4075 4342 4076 if (!func) 4343 4077 return false; ··· 4349 4165 return false; 4350 4166 } 4351 4167 4352 4352 - static int add_prefix_symbol(struct objtool_file *file, struct symbol *func) 4168 4168 + /* 4169 4169 + * For FineIBT or kCFI, a certain number of bytes preceding the function may be 4170 4170 + * NOPs. Those NOPs may be rewritten at runtime and executed, so give them a 4171 4171 + * proper function name: __pfx_<func>. 4172 4172 + * 4173 4173 + * The NOPs may not exist for the following cases: 4174 4174 + * 4175 4175 + * - compiler cloned functions (*.cold, *.part0, etc) 4176 4176 + * - asm functions created with inline asm or without SYM_FUNC_START() 4177 4177 + * 4178 4178 + * Also, the function may already have a prefix from a previous objtool run 4179 4179 + * (livepatch extracted functions, or manually running objtool multiple times). 4180 4180 + * 4181 4181 + * So return 0 if the NOPs are missing or the function already has a prefix 4182 4182 + * symbol. 4183 4183 + */ 4184 4184 + static int create_prefix_symbol(struct objtool_file *file, struct symbol *func) 4353 4185 { 4354 4186 struct instruction *insn, *prev; 4187 4187 + char name[SYM_NAME_LEN]; 4355 4188 struct cfi_state *cfi; 4356 4189 4357 4357 - insn = find_insn(file, func->sec, func->offset); 4358 4358 - if (!insn) 4190 4190 + if (!is_func_sym(func) || is_prefix_func(func) || 4191 4191 + func->cold || func->static_call_tramp) 4192 4192 + return 0; 4193 4193 + 4194 4194 + if ((strlen(func->name) + sizeof("__pfx_") > SYM_NAME_LEN)) { 4195 4195 + WARN("%s: symbol name too long, can't create __pfx_ symbol", 4196 4196 + func->name); 4197 4197 + return 0; 4198 4198 + } 4199 4199 + 4200 4200 + if (snprintf_check(name, SYM_NAME_LEN, "__pfx_%s", func->name)) 4359 4201 return -1; 4202 4202 + 4203 4203 + if (file->klp) { 4204 4204 + struct symbol *pfx; 4205 4205 + 4206 4206 + pfx = find_symbol_by_offset(func->sec, func->offset - opts.prefix); 4207 4207 + if (pfx && is_prefix_func(pfx) && !strcmp(pfx->name, name)) 4208 4208 + return 0; 4209 4209 + } 4210 4210 + 4211 4211 + insn = find_insn(file, func->sec, func->offset); 4212 4212 + if (!insn) { 4213 4213 + WARN("%s: can't find starting instruction", func->name); 4214 4214 + return -1; 4215 4215 + } 4360 4216 4361 4217 for (prev = prev_insn_same_sec(file, insn); 4362 4218 prev; ··· 4404 4180 u64 offset; 4405 4181 4406 4182 if (prev->type != INSN_NOP) 4407 4407 - return -1; 4183 4183 + return 0; 4408 4184 4409 4185 offset = func->offset - prev->offset; 4410 4186 4411 4187 if (offset > opts.prefix) 4412 4412 - return -1; 4188 4188 + return 0; 4413 4189 4414 4190 if (offset < opts.prefix) 4415 4191 continue; 4416 4192 4417 4417 - elf_create_prefix_symbol(file->elf, func, opts.prefix); 4193 4193 + if (!elf_create_symbol(file->elf, name, func->sec, 4194 4194 + GELF_ST_BIND(func->sym.st_info), 4195 4195 + GELF_ST_TYPE(func->sym.st_info), 4196 4196 + prev->offset, opts.prefix)) 4197 4197 + return -1; 4198 4198 + 4418 4199 break; 4419 4200 } 4420 4201 4421 4202 if (!prev) 4422 4422 - return -1; 4203 4203 + return 0; 4423 4204 4424 4205 if (!insn->cfi) { 4425 4206 /* ··· 4442 4213 return 0; 4443 4214 } 4444 4215 4445 4445 - static int add_prefix_symbols(struct objtool_file *file) 4216 4216 + static int create_prefix_symbols(struct objtool_file *file) 4446 4217 { 4447 4218 struct section *sec; 4448 4219 struct symbol *func; 4449 4220 4450 4450 - for_each_sec(file, sec) { 4451 4451 - if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 4221 4221 + for_each_sec(file->elf, sec) { 4222 4222 + if (!is_text_sec(sec)) 4452 4223 continue; 4453 4224 4454 4225 sec_for_each_sym(sec, func) { 4455 4455 - if (func->type != STT_FUNC) 4456 4456 - continue; 4457 4457 - 4458 4458 - add_prefix_symbol(file, func); 4226 4226 + if (create_prefix_symbol(file, func)) 4227 4227 + return -1; 4459 4228 } 4460 4229 } 4461 4230 ··· 4464 4237 struct symbol *sym, struct insn_state *state) 4465 4238 { 4466 4239 struct instruction *insn; 4240 4240 + struct symbol *func; 4467 4241 int ret; 4468 4242 4469 4243 if (!sym->len) { ··· 4482 4254 if (opts.uaccess) 4483 4255 state->uaccess = sym->uaccess_safe; 4484 4256 4485 4485 - ret = validate_branch(file, insn_func(insn), insn, *state); 4257 4257 + func = insn_func(insn); 4258 4258 + 4259 4259 + if (opts.checksum) 4260 4260 + checksum_init(func); 4261 4261 + 4262 4262 + if (opts.trace && !fnmatch(opts.trace, sym->name, 0)) { 4263 4263 + trace_enable(); 4264 4264 + TRACE("%s: validation begin\n", sym->name); 4265 4265 + } 4266 4266 + 4267 4267 + ret = validate_branch(file, func, insn, *state); 4486 4268 if (ret) 4487 4269 BT_INSN(insn, "<=== (sym)"); 4270 4270 + 4271 4271 + TRACE("%s: validation %s\n\n", sym->name, ret ? "failed" : "end"); 4272 4272 + trace_disable(); 4273 4273 + 4274 4274 + if (opts.checksum) 4275 4275 + checksum_finish(func); 4276 4276 + 4488 4277 return ret; 4489 4278 } 4490 4279 ··· 4512 4267 int warnings = 0; 4513 4268 4514 4269 sec_for_each_sym(sec, func) { 4515 4515 - if (func->type != STT_FUNC) 4270 4270 + if (!is_func_sym(func)) 4516 4271 continue; 4517 4272 4518 4273 init_insn_state(file, &state, sec); ··· 4555 4310 struct section *sec; 4556 4311 int warnings = 0; 4557 4312 4558 4558 - for_each_sec(file, sec) { 4559 4559 - if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 4313 4313 + for_each_sec(file->elf, sec) { 4314 4314 + if (!is_text_sec(sec)) 4560 4315 continue; 4561 4316 4562 4317 warnings += validate_section(file, sec); ··· 4683 4438 reloc_offset(reloc) + 1, 4684 4439 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) { 4685 4440 4686 4686 - off = reloc->sym->offset; 4687 4687 - if (reloc_type(reloc) == R_X86_64_PC32 || 4688 4688 - reloc_type(reloc) == R_X86_64_PLT32) 4689 4689 - off += arch_dest_reloc_offset(reloc_addend(reloc)); 4690 4690 - else 4691 4691 - off += reloc_addend(reloc); 4441 4441 + off = reloc->sym->offset + arch_insn_adjusted_addend(insn, reloc); 4692 4442 4693 4443 dest = find_insn(file, reloc->sym->sec, off); 4694 4444 if (!dest) ··· 4734 4494 for_each_insn(file, insn) 4735 4495 warnings += validate_ibt_insn(file, insn); 4736 4496 4737 4737 - for_each_sec(file, sec) { 4497 4497 + for_each_sec(file->elf, sec) { 4738 4498 4739 4499 /* Already done by validate_ibt_insn() */ 4740 4740 - if (sec->sh.sh_flags & SHF_EXECINSTR) 4500 4500 + if (is_text_sec(sec)) 4741 4501 continue; 4742 4502 4743 4503 if (!sec->rsec) ··· 4752 4512 !strncmp(sec->name, ".debug", 6) || 4753 4513 !strcmp(sec->name, ".altinstructions") || 4754 4514 !strcmp(sec->name, ".ibt_endbr_seal") || 4515 4515 + !strcmp(sec->name, ".kcfi_traps") || 4755 4516 !strcmp(sec->name, ".orc_unwind_ip") || 4756 4756 - !strcmp(sec->name, ".parainstructions") || 4757 4517 !strcmp(sec->name, ".retpoline_sites") || 4758 4518 !strcmp(sec->name, ".smp_locks") || 4759 4519 !strcmp(sec->name, ".static_call_sites") || ··· 4762 4522 !strcmp(sec->name, "__bug_table") || 4763 4523 !strcmp(sec->name, "__ex_table") || 4764 4524 !strcmp(sec->name, "__jump_table") || 4525 4525 + !strcmp(sec->name, "__klp_funcs") || 4765 4526 !strcmp(sec->name, "__mcount_loc") || 4766 4766 - !strcmp(sec->name, ".kcfi_traps") || 4767 4527 !strcmp(sec->name, ".llvm.call-graph-profile") || 4768 4528 !strcmp(sec->name, ".llvm_bb_addr_map") || 4769 4529 !strcmp(sec->name, "__tracepoints") || 4770 4770 - strstr(sec->name, "__patchable_function_entries")) 4530 4530 + !strcmp(sec->name, ".return_sites") || 4531 4531 + !strcmp(sec->name, ".call_sites") || 4532 4532 + !strcmp(sec->name, "__patchable_function_entries")) 4771 4533 continue; 4772 4534 4773 4535 for_each_reloc(sec->rsec, reloc) ··· 4843 4601 return warnings; 4844 4602 } 4845 4603 4846 4846 - /* 'funcs' is a space-separated list of function names */ 4847 4847 - static void disas_funcs(const char *funcs) 4848 4848 - { 4849 4849 - const char *objdump_str, *cross_compile; 4850 4850 - int size, ret; 4851 4851 - char *cmd; 4852 4852 - 4853 4853 - cross_compile = getenv("CROSS_COMPILE"); 4854 4854 - if (!cross_compile) 4855 4855 - cross_compile = ""; 4856 4856 - 4857 4857 - objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '" 4858 4858 - "BEGIN { split(_funcs, funcs); }" 4859 4859 - "/^$/ { func_match = 0; }" 4860 4860 - "/<.*>:/ { " 4861 4861 - "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);" 4862 4862 - "for (i in funcs) {" 4863 4863 - "if (funcs[i] == f) {" 4864 4864 - "func_match = 1;" 4865 4865 - "base = strtonum(\"0x\" $1);" 4866 4866 - "break;" 4867 4867 - "}" 4868 4868 - "}" 4869 4869 - "}" 4870 4870 - "{" 4871 4871 - "if (func_match) {" 4872 4872 - "addr = strtonum(\"0x\" $1);" 4873 4873 - "printf(\"%%04x \", addr - base);" 4874 4874 - "print;" 4875 4875 - "}" 4876 4876 - "}' 1>&2"; 4877 4877 - 4878 4878 - /* fake snprintf() to calculate the size */ 4879 4879 - size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1; 4880 4880 - if (size <= 0) { 4881 4881 - WARN("objdump string size calculation failed"); 4882 4882 - return; 4883 4883 - } 4884 4884 - 4885 4885 - cmd = malloc(size); 4886 4886 - 4887 4887 - /* real snprintf() */ 4888 4888 - snprintf(cmd, size, objdump_str, cross_compile, objname, funcs); 4889 4889 - ret = system(cmd); 4890 4890 - if (ret) { 4891 4891 - WARN("disassembly failed: %d", ret); 4892 4892 - return; 4893 4893 - } 4894 4894 - } 4895 4895 - 4896 4896 - static void disas_warned_funcs(struct objtool_file *file) 4897 4897 - { 4898 4898 - struct symbol *sym; 4899 4899 - char *funcs = NULL, *tmp; 4900 4900 - 4901 4901 - for_each_sym(file, sym) { 4902 4902 - if (sym->warned) { 4903 4903 - if (!funcs) { 4904 4904 - funcs = malloc(strlen(sym->name) + 1); 4905 4905 - if (!funcs) { 4906 4906 - ERROR_GLIBC("malloc"); 4907 4907 - return; 4908 4908 - } 4909 4909 - strcpy(funcs, sym->name); 4910 4910 - } else { 4911 4911 - tmp = malloc(strlen(funcs) + strlen(sym->name) + 2); 4912 4912 - if (!tmp) { 4913 4913 - ERROR_GLIBC("malloc"); 4914 4914 - return; 4915 4915 - } 4916 4916 - sprintf(tmp, "%s %s", funcs, sym->name); 4917 4917 - free(funcs); 4918 4918 - funcs = tmp; 4919 4919 - } 4920 4920 - } 4921 4921 - } 4922 4922 - 4923 4923 - if (funcs) 4924 4924 - disas_funcs(funcs); 4925 4925 - } 4926 4926 - 4927 4604 __weak bool arch_absolute_reloc(struct elf *elf, struct reloc *reloc) 4928 4605 { 4929 4606 unsigned int type = reloc_type(reloc); ··· 4857 4696 struct reloc *reloc; 4858 4697 int ret = 0; 4859 4698 4860 4860 - for_each_sec(file, sec) { 4699 4699 + for_each_sec(file->elf, sec) { 4861 4700 /* absolute references in non-loadable sections are fine */ 4862 4701 if (!(sec->sh.sh_flags & SHF_ALLOC)) 4863 4702 continue; ··· 4912 4751 free(chunk->addr); 4913 4752 } 4914 4753 4754 4754 + const char *objtool_disas_insn(struct instruction *insn) 4755 4755 + { 4756 4756 + struct disas_context *dctx = objtool_disas_ctx; 4757 4757 + 4758 4758 + if (!dctx) 4759 4759 + return ""; 4760 4760 + 4761 4761 + disas_insn(dctx, insn); 4762 4762 + return disas_result(dctx); 4763 4763 + } 4764 4764 + 4915 4765 int check(struct objtool_file *file) 4916 4766 { 4767 4767 + struct disas_context *disas_ctx = NULL; 4917 4768 int ret = 0, warnings = 0; 4769 4769 + 4770 4770 + /* 4771 4771 + * Create a disassembly context if we might disassemble any 4772 4772 + * instruction or function. 4773 4773 + */ 4774 4774 + if (opts.verbose || opts.backtrace || opts.trace || opts.disas) { 4775 4775 + disas_ctx = disas_context_create(file); 4776 4776 + if (!disas_ctx) { 4777 4777 + opts.disas = false; 4778 4778 + opts.trace = false; 4779 4779 + } 4780 4780 + objtool_disas_ctx = disas_ctx; 4781 4781 + } 4918 4782 4919 4783 arch_initial_func_cfi_state(&initial_func_cfi); 4920 4784 init_cfi_state(&init_cfi); ··· 4956 4770 cfi_hash_add(&init_cfi); 4957 4771 cfi_hash_add(&func_cfi); 4958 4772 4773 4773 + ret = checksum_debug_init(file); 4774 4774 + if (ret) 4775 4775 + goto out; 4776 4776 + 4959 4777 ret = decode_sections(file); 4960 4778 if (ret) 4961 4779 goto out; ··· 4970 4780 if (opts.retpoline) 4971 4781 warnings += validate_retpoline(file); 4972 4782 4973 4973 - if (opts.stackval || opts.orc || opts.uaccess) { 4783 4783 + if (validate_branch_enabled()) { 4974 4784 int w = 0; 4975 4785 4976 4786 w += validate_functions(file); ··· 5035 4845 } 5036 4846 5037 4847 if (opts.prefix) { 5038 5038 - ret = add_prefix_symbols(file); 4848 4848 + ret = create_prefix_symbols(file); 5039 4849 if (ret) 5040 4850 goto out; 5041 4851 } ··· 5049 4859 if (opts.noabs) 5050 4860 warnings += check_abs_references(file); 5051 4861 4862 4862 + if (opts.checksum) { 4863 4863 + ret = create_sym_checksum_section(file); 4864 4864 + if (ret) 4865 4865 + goto out; 4866 4866 + } 4867 4867 + 5052 4868 if (opts.orc && nr_insns) { 5053 4869 ret = orc_create(file); 5054 4870 if (ret) 5055 4871 goto out; 5056 4872 } 5057 5057 - 5058 5058 - free_insns(file); 5059 4873 5060 4874 if (opts.stats) { 5061 4875 printf("nr_insns_visited: %ld\n", nr_insns_visited); ··· 5069 4875 } 5070 4876 5071 4877 out: 4878 4878 + if (ret || warnings) { 4879 4879 + if (opts.werror && warnings) 4880 4880 + ret = 1; 4881 4881 + 4882 4882 + if (opts.verbose) { 4883 4883 + if (opts.werror && warnings) 4884 4884 + WARN("%d warning(s) upgraded to errors", warnings); 4885 4885 + disas_warned_funcs(disas_ctx); 4886 4886 + } 4887 4887 + } 4888 4888 + 4889 4889 + if (opts.disas) 4890 4890 + disas_funcs(disas_ctx); 4891 4891 + 4892 4892 + if (disas_ctx) { 4893 4893 + disas_context_destroy(disas_ctx); 4894 4894 + objtool_disas_ctx = NULL; 4895 4895 + } 4896 4896 + 4897 4897 + free_insns(file); 4898 4898 + 5072 4899 if (!ret && !warnings) 5073 4900 return 0; 5074 4901 5075 5075 - if (opts.werror && warnings) 5076 5076 - ret = 1; 5077 5077 - 5078 5078 - if (opts.verbose) { 5079 5079 - if (opts.werror && warnings) 5080 5080 - WARN("%d warning(s) upgraded to errors", warnings); 5081 5081 - print_args(); 5082 5082 - disas_warned_funcs(file); 5083 5083 - } 4902 4902 + if (opts.backup && make_backup()) 4903 4903 + return 1; 5084 4904 5085 4905 return ret; 5086 4906 }

+1248

tools/objtool/disas.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + /* 3 3 + * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 4 + */ 5 5 + 6 6 + #define _GNU_SOURCE 7 7 + #include <fnmatch.h> 8 8 + 9 9 + #include <objtool/arch.h> 10 10 + #include <objtool/check.h> 11 11 + #include <objtool/disas.h> 12 12 + #include <objtool/special.h> 13 13 + #include <objtool/warn.h> 14 14 + 15 15 + #include <bfd.h> 16 16 + #include <linux/string.h> 17 17 + #include <tools/dis-asm-compat.h> 18 18 + 19 19 + /* 20 20 + * Size of the buffer for storing the result of disassembling 21 21 + * a single instruction. 22 22 + */ 23 23 + #define DISAS_RESULT_SIZE 1024 24 24 + 25 25 + struct disas_context { 26 26 + struct objtool_file *file; 27 27 + struct instruction *insn; 28 28 + bool alt_applied; 29 29 + char result[DISAS_RESULT_SIZE]; 30 30 + disassembler_ftype disassembler; 31 31 + struct disassemble_info info; 32 32 + }; 33 33 + 34 34 + /* 35 35 + * Maximum number of alternatives 36 36 + */ 37 37 + #define DISAS_ALT_MAX 5 38 38 + 39 39 + /* 40 40 + * Maximum number of instructions per alternative 41 41 + */ 42 42 + #define DISAS_ALT_INSN_MAX 50 43 43 + 44 44 + /* 45 45 + * Information to disassemble an alternative 46 46 + */ 47 47 + struct disas_alt { 48 48 + struct instruction *orig_insn; /* original instruction */ 49 49 + struct alternative *alt; /* alternative or NULL if default code */ 50 50 + char *name; /* name for this alternative */ 51 51 + int width; /* formatting width */ 52 52 + struct { 53 53 + char *str; /* instruction string */ 54 54 + int offset; /* instruction offset */ 55 55 + int nops; /* number of nops */ 56 56 + } insn[DISAS_ALT_INSN_MAX]; /* alternative instructions */ 57 57 + int insn_idx; /* index of the next instruction to print */ 58 58 + }; 59 59 + 60 60 + #define DALT_DEFAULT(dalt) (!(dalt)->alt) 61 61 + #define DALT_INSN(dalt) (DALT_DEFAULT(dalt) ? (dalt)->orig_insn : (dalt)->alt->insn) 62 62 + #define DALT_GROUP(dalt) (DALT_INSN(dalt)->alt_group) 63 63 + #define DALT_ALTID(dalt) ((dalt)->orig_insn->offset) 64 64 + 65 65 + #define ALT_FLAGS_SHIFT 16 66 66 + #define ALT_FLAG_NOT (1 << 0) 67 67 + #define ALT_FLAG_DIRECT_CALL (1 << 1) 68 68 + #define ALT_FEATURE_MASK ((1 << ALT_FLAGS_SHIFT) - 1) 69 69 + 70 70 + static int alt_feature(unsigned int ft_flags) 71 71 + { 72 72 + return (ft_flags & ALT_FEATURE_MASK); 73 73 + } 74 74 + 75 75 + static int alt_flags(unsigned int ft_flags) 76 76 + { 77 77 + return (ft_flags >> ALT_FLAGS_SHIFT); 78 78 + } 79 79 + 80 80 + /* 81 81 + * Wrapper around asprintf() to allocate and format a string. 82 82 + * Return the allocated string or NULL on error. 83 83 + */ 84 84 + static char *strfmt(const char *fmt, ...) 85 85 + { 86 86 + va_list ap; 87 87 + char *str; 88 88 + int rv; 89 89 + 90 90 + va_start(ap, fmt); 91 91 + rv = vasprintf(&str, fmt, ap); 92 92 + va_end(ap); 93 93 + 94 94 + return rv == -1 ? NULL : str; 95 95 + } 96 96 + 97 97 + static int sprint_name(char *str, const char *name, unsigned long offset) 98 98 + { 99 99 + int len; 100 100 + 101 101 + if (offset) 102 102 + len = sprintf(str, "%s+0x%lx", name, offset); 103 103 + else 104 104 + len = sprintf(str, "%s", name); 105 105 + 106 106 + return len; 107 107 + } 108 108 + 109 109 + #define DINFO_FPRINTF(dinfo, ...) \ 110 110 + ((*(dinfo)->fprintf_func)((dinfo)->stream, __VA_ARGS__)) 111 111 + 112 112 + static int disas_result_fprintf(struct disas_context *dctx, 113 113 + const char *fmt, va_list ap) 114 114 + { 115 115 + char *buf = dctx->result; 116 116 + int avail, len; 117 117 + 118 118 + len = strlen(buf); 119 119 + if (len >= DISAS_RESULT_SIZE - 1) { 120 120 + WARN_FUNC(dctx->insn->sec, dctx->insn->offset, 121 121 + "disassembly buffer is full"); 122 122 + return -1; 123 123 + } 124 124 + avail = DISAS_RESULT_SIZE - len; 125 125 + 126 126 + len = vsnprintf(buf + len, avail, fmt, ap); 127 127 + if (len < 0 || len >= avail) { 128 128 + WARN_FUNC(dctx->insn->sec, dctx->insn->offset, 129 129 + "disassembly buffer is truncated"); 130 130 + return -1; 131 131 + } 132 132 + 133 133 + return 0; 134 134 + } 135 135 + 136 136 + static int disas_fprintf(void *stream, const char *fmt, ...) 137 137 + { 138 138 + va_list arg; 139 139 + int rv; 140 140 + 141 141 + va_start(arg, fmt); 142 142 + rv = disas_result_fprintf(stream, fmt, arg); 143 143 + va_end(arg); 144 144 + 145 145 + return rv; 146 146 + } 147 147 + 148 148 + /* 149 149 + * For init_disassemble_info_compat(). 150 150 + */ 151 151 + static int disas_fprintf_styled(void *stream, 152 152 + enum disassembler_style style, 153 153 + const char *fmt, ...) 154 154 + { 155 155 + va_list arg; 156 156 + int rv; 157 157 + 158 158 + va_start(arg, fmt); 159 159 + rv = disas_result_fprintf(stream, fmt, arg); 160 160 + va_end(arg); 161 161 + 162 162 + return rv; 163 163 + } 164 164 + 165 165 + static void disas_print_addr_sym(struct section *sec, struct symbol *sym, 166 166 + bfd_vma addr, struct disassemble_info *dinfo) 167 167 + { 168 168 + char symstr[1024]; 169 169 + char *str; 170 170 + 171 171 + if (sym) { 172 172 + sprint_name(symstr, sym->name, addr - sym->offset); 173 173 + DINFO_FPRINTF(dinfo, "0x%lx <%s>", addr, symstr); 174 174 + } else { 175 175 + str = offstr(sec, addr); 176 176 + DINFO_FPRINTF(dinfo, "0x%lx <%s>", addr, str); 177 177 + free(str); 178 178 + } 179 179 + } 180 180 + 181 181 + static bool disas_print_addr_alt(bfd_vma addr, struct disassemble_info *dinfo) 182 182 + { 183 183 + struct disas_context *dctx = dinfo->application_data; 184 184 + struct instruction *orig_first_insn; 185 185 + struct alt_group *alt_group; 186 186 + unsigned long offset; 187 187 + struct symbol *sym; 188 188 + 189 189 + /* 190 190 + * Check if we are processing an alternative at the original 191 191 + * instruction address (i.e. if alt_applied is true) and if 192 192 + * we are referencing an address inside the alternative. 193 193 + * 194 194 + * For example, this happens if there is a branch inside an 195 195 + * alternative. In that case, the address should be updated 196 196 + * to a reference inside the original instruction flow. 197 197 + */ 198 198 + if (!dctx->alt_applied) 199 199 + return false; 200 200 + 201 201 + alt_group = dctx->insn->alt_group; 202 202 + if (!alt_group || !alt_group->orig_group || 203 203 + addr < alt_group->first_insn->offset || 204 204 + addr > alt_group->last_insn->offset) 205 205 + return false; 206 206 + 207 207 + orig_first_insn = alt_group->orig_group->first_insn; 208 208 + offset = addr - alt_group->first_insn->offset; 209 209 + 210 210 + addr = orig_first_insn->offset + offset; 211 211 + sym = orig_first_insn->sym; 212 212 + 213 213 + disas_print_addr_sym(orig_first_insn->sec, sym, addr, dinfo); 214 214 + 215 215 + return true; 216 216 + } 217 217 + 218 218 + static void disas_print_addr_noreloc(bfd_vma addr, 219 219 + struct disassemble_info *dinfo) 220 220 + { 221 221 + struct disas_context *dctx = dinfo->application_data; 222 222 + struct instruction *insn = dctx->insn; 223 223 + struct symbol *sym = NULL; 224 224 + 225 225 + if (disas_print_addr_alt(addr, dinfo)) 226 226 + return; 227 227 + 228 228 + if (insn->sym && addr >= insn->sym->offset && 229 229 + addr < insn->sym->offset + insn->sym->len) { 230 230 + sym = insn->sym; 231 231 + } 232 232 + 233 233 + disas_print_addr_sym(insn->sec, sym, addr, dinfo); 234 234 + } 235 235 + 236 236 + static void disas_print_addr_reloc(bfd_vma addr, struct disassemble_info *dinfo) 237 237 + { 238 238 + struct disas_context *dctx = dinfo->application_data; 239 239 + struct instruction *insn = dctx->insn; 240 240 + unsigned long offset; 241 241 + struct reloc *reloc; 242 242 + char symstr[1024]; 243 243 + char *str; 244 244 + 245 245 + reloc = find_reloc_by_dest_range(dctx->file->elf, insn->sec, 246 246 + insn->offset, insn->len); 247 247 + if (!reloc) { 248 248 + /* 249 249 + * There is no relocation for this instruction although 250 250 + * the address to resolve points to the next instruction. 251 251 + * So this is an effective reference to the next IP, for 252 252 + * example: "lea 0x0(%rip),%rdi". The kernel can reference 253 253 + * the next IP with _THIS_IP_ macro. 254 254 + */ 255 255 + DINFO_FPRINTF(dinfo, "0x%lx <_THIS_IP_>", addr); 256 256 + return; 257 257 + } 258 258 + 259 259 + offset = arch_insn_adjusted_addend(insn, reloc); 260 260 + 261 261 + /* 262 262 + * If the relocation symbol is a section name (for example ".bss") 263 263 + * then we try to further resolve the name. 264 264 + */ 265 265 + if (reloc->sym->type == STT_SECTION) { 266 266 + str = offstr(reloc->sym->sec, reloc->sym->offset + offset); 267 267 + DINFO_FPRINTF(dinfo, "0x%lx <%s>", addr, str); 268 268 + free(str); 269 269 + } else { 270 270 + sprint_name(symstr, reloc->sym->name, offset); 271 271 + DINFO_FPRINTF(dinfo, "0x%lx <%s>", addr, symstr); 272 272 + } 273 273 + } 274 274 + 275 275 + /* 276 276 + * Resolve an address into a "<symbol>+<offset>" string. 277 277 + */ 278 278 + static void disas_print_address(bfd_vma addr, struct disassemble_info *dinfo) 279 279 + { 280 280 + struct disas_context *dctx = dinfo->application_data; 281 281 + struct instruction *insn = dctx->insn; 282 282 + struct instruction *jump_dest; 283 283 + struct symbol *sym; 284 284 + bool is_reloc; 285 285 + 286 286 + /* 287 287 + * If the instruction is a call/jump and it references a 288 288 + * destination then this is likely the address we are looking 289 289 + * up. So check it first. 290 290 + */ 291 291 + jump_dest = insn->jump_dest; 292 292 + if (jump_dest && jump_dest->sym && jump_dest->offset == addr) { 293 293 + if (!disas_print_addr_alt(addr, dinfo)) 294 294 + disas_print_addr_sym(jump_dest->sec, jump_dest->sym, 295 295 + addr, dinfo); 296 296 + return; 297 297 + } 298 298 + 299 299 + /* 300 300 + * If the address points to the next instruction then there is 301 301 + * probably a relocation. It can be a false positive when the 302 302 + * current instruction is referencing the address of the next 303 303 + * instruction. This particular case will be handled in 304 304 + * disas_print_addr_reloc(). 305 305 + */ 306 306 + is_reloc = (addr == insn->offset + insn->len); 307 307 + 308 308 + /* 309 309 + * The call destination offset can be the address we are looking 310 310 + * up, or 0 if there is a relocation. 311 311 + */ 312 312 + sym = insn_call_dest(insn); 313 313 + if (sym && (sym->offset == addr || (sym->offset == 0 && is_reloc))) { 314 314 + DINFO_FPRINTF(dinfo, "0x%lx <%s>", addr, sym->name); 315 315 + return; 316 316 + } 317 317 + 318 318 + if (!is_reloc) 319 319 + disas_print_addr_noreloc(addr, dinfo); 320 320 + else 321 321 + disas_print_addr_reloc(addr, dinfo); 322 322 + } 323 323 + 324 324 + /* 325 325 + * Initialize disassemble info arch, mach (32 or 64-bit) and options. 326 326 + */ 327 327 + int disas_info_init(struct disassemble_info *dinfo, 328 328 + int arch, int mach32, int mach64, 329 329 + const char *options) 330 330 + { 331 331 + struct disas_context *dctx = dinfo->application_data; 332 332 + struct objtool_file *file = dctx->file; 333 333 + 334 334 + dinfo->arch = arch; 335 335 + 336 336 + switch (file->elf->ehdr.e_ident[EI_CLASS]) { 337 337 + case ELFCLASS32: 338 338 + dinfo->mach = mach32; 339 339 + break; 340 340 + case ELFCLASS64: 341 341 + dinfo->mach = mach64; 342 342 + break; 343 343 + default: 344 344 + return -1; 345 345 + } 346 346 + 347 347 + dinfo->disassembler_options = options; 348 348 + 349 349 + return 0; 350 350 + } 351 351 + 352 352 + struct disas_context *disas_context_create(struct objtool_file *file) 353 353 + { 354 354 + struct disas_context *dctx; 355 355 + struct disassemble_info *dinfo; 356 356 + int err; 357 357 + 358 358 + dctx = malloc(sizeof(*dctx)); 359 359 + if (!dctx) { 360 360 + WARN("failed to allocate disassembly context"); 361 361 + return NULL; 362 362 + } 363 363 + 364 364 + dctx->file = file; 365 365 + dinfo = &dctx->info; 366 366 + 367 367 + init_disassemble_info_compat(dinfo, dctx, 368 368 + disas_fprintf, disas_fprintf_styled); 369 369 + 370 370 + dinfo->read_memory_func = buffer_read_memory; 371 371 + dinfo->print_address_func = disas_print_address; 372 372 + dinfo->application_data = dctx; 373 373 + 374 374 + /* 375 375 + * bfd_openr() is not used to avoid doing ELF data processing 376 376 + * and caching that has already being done. Here, we just need 377 377 + * to identify the target file so we call an arch specific 378 378 + * function to fill some disassemble info (arch, mach). 379 379 + */ 380 380 + 381 381 + dinfo->arch = bfd_arch_unknown; 382 382 + dinfo->mach = 0; 383 383 + 384 384 + err = arch_disas_info_init(dinfo); 385 385 + if (err || dinfo->arch == bfd_arch_unknown || dinfo->mach == 0) { 386 386 + WARN("failed to init disassembly arch"); 387 387 + goto error; 388 388 + } 389 389 + 390 390 + dinfo->endian = (file->elf->ehdr.e_ident[EI_DATA] == ELFDATA2MSB) ? 391 391 + BFD_ENDIAN_BIG : BFD_ENDIAN_LITTLE; 392 392 + 393 393 + disassemble_init_for_target(dinfo); 394 394 + 395 395 + dctx->disassembler = disassembler(dinfo->arch, 396 396 + dinfo->endian == BFD_ENDIAN_BIG, 397 397 + dinfo->mach, NULL); 398 398 + if (!dctx->disassembler) { 399 399 + WARN("failed to create disassembler function"); 400 400 + goto error; 401 401 + } 402 402 + 403 403 + return dctx; 404 404 + 405 405 + error: 406 406 + free(dctx); 407 407 + return NULL; 408 408 + } 409 409 + 410 410 + void disas_context_destroy(struct disas_context *dctx) 411 411 + { 412 412 + free(dctx); 413 413 + } 414 414 + 415 415 + char *disas_result(struct disas_context *dctx) 416 416 + { 417 417 + return dctx->result; 418 418 + } 419 419 + 420 420 + #define DISAS_INSN_OFFSET_SPACE 10 421 421 + #define DISAS_INSN_SPACE 60 422 422 + 423 423 + #define DISAS_PRINSN(dctx, insn, depth) \ 424 424 + disas_print_insn(stdout, dctx, insn, depth, "\n") 425 425 + 426 426 + /* 427 427 + * Print a message in the instruction flow. If sec is not NULL then the 428 428 + * address at the section offset is printed in addition of the message, 429 429 + * otherwise only the message is printed. 430 430 + */ 431 431 + static int disas_vprint(FILE *stream, struct section *sec, unsigned long offset, 432 432 + int depth, const char *format, va_list ap) 433 433 + { 434 434 + const char *addr_str; 435 435 + int i, n; 436 436 + int len; 437 437 + 438 438 + len = sym_name_max_len + DISAS_INSN_OFFSET_SPACE; 439 439 + if (depth < 0) { 440 440 + len += depth; 441 441 + depth = 0; 442 442 + } 443 443 + 444 444 + n = 0; 445 445 + 446 446 + if (sec) { 447 447 + addr_str = offstr(sec, offset); 448 448 + n += fprintf(stream, "%6lx: %-*s ", offset, len, addr_str); 449 449 + free((char *)addr_str); 450 450 + } else { 451 451 + len += DISAS_INSN_OFFSET_SPACE + 1; 452 452 + n += fprintf(stream, "%-*s", len, ""); 453 453 + } 454 454 + 455 455 + /* print vertical bars to show the code flow */ 456 456 + for (i = 0; i < depth; i++) 457 457 + n += fprintf(stream, "| "); 458 458 + 459 459 + if (format) 460 460 + n += vfprintf(stream, format, ap); 461 461 + 462 462 + return n; 463 463 + } 464 464 + 465 465 + static int disas_print(FILE *stream, struct section *sec, unsigned long offset, 466 466 + int depth, const char *format, ...) 467 467 + { 468 468 + va_list args; 469 469 + int len; 470 470 + 471 471 + va_start(args, format); 472 472 + len = disas_vprint(stream, sec, offset, depth, format, args); 473 473 + va_end(args); 474 474 + 475 475 + return len; 476 476 + } 477 477 + 478 478 + /* 479 479 + * Print a message in the instruction flow. If insn is not NULL then 480 480 + * the instruction address is printed in addition of the message, 481 481 + * otherwise only the message is printed. In all cases, the instruction 482 482 + * itself is not printed. 483 483 + */ 484 484 + void disas_print_info(FILE *stream, struct instruction *insn, int depth, 485 485 + const char *format, ...) 486 486 + { 487 487 + struct section *sec; 488 488 + unsigned long off; 489 489 + va_list args; 490 490 + 491 491 + if (insn) { 492 492 + sec = insn->sec; 493 493 + off = insn->offset; 494 494 + } else { 495 495 + sec = NULL; 496 496 + off = 0; 497 497 + } 498 498 + 499 499 + va_start(args, format); 500 500 + disas_vprint(stream, sec, off, depth, format, args); 501 501 + va_end(args); 502 502 + } 503 503 + 504 504 + /* 505 505 + * Print an instruction address (offset and function), the instruction itself 506 506 + * and an optional message. 507 507 + */ 508 508 + void disas_print_insn(FILE *stream, struct disas_context *dctx, 509 509 + struct instruction *insn, int depth, 510 510 + const char *format, ...) 511 511 + { 512 512 + char fake_nop_insn[32]; 513 513 + const char *insn_str; 514 514 + bool fake_nop; 515 515 + va_list args; 516 516 + int len; 517 517 + 518 518 + /* 519 519 + * Alternative can insert a fake nop, sometimes with no 520 520 + * associated section so nothing to disassemble. 521 521 + */ 522 522 + fake_nop = (!insn->sec && insn->type == INSN_NOP); 523 523 + if (fake_nop) { 524 524 + snprintf(fake_nop_insn, 32, "<fake nop> (%d bytes)", insn->len); 525 525 + insn_str = fake_nop_insn; 526 526 + } else { 527 527 + disas_insn(dctx, insn); 528 528 + insn_str = disas_result(dctx); 529 529 + } 530 530 + 531 531 + /* print the instruction */ 532 532 + len = (depth + 1) * 2 < DISAS_INSN_SPACE ? DISAS_INSN_SPACE - (depth+1) * 2 : 1; 533 533 + disas_print_info(stream, insn, depth, "%-*s", len, insn_str); 534 534 + 535 535 + /* print message if any */ 536 536 + if (!format) 537 537 + return; 538 538 + 539 539 + if (strcmp(format, "\n") == 0) { 540 540 + fprintf(stream, "\n"); 541 541 + return; 542 542 + } 543 543 + 544 544 + fprintf(stream, " - "); 545 545 + va_start(args, format); 546 546 + vfprintf(stream, format, args); 547 547 + va_end(args); 548 548 + } 549 549 + 550 550 + /* 551 551 + * Disassemble a single instruction. Return the size of the instruction. 552 552 + * 553 553 + * If alt_applied is true then insn should be an instruction from of an 554 554 + * alternative (i.e. insn->alt_group != NULL), and it is disassembled 555 555 + * at the location of the original code it is replacing. When the 556 556 + * instruction references any address inside the alternative then 557 557 + * these references will be re-adjusted to replace the original code. 558 558 + */ 559 559 + static size_t disas_insn_common(struct disas_context *dctx, 560 560 + struct instruction *insn, 561 561 + bool alt_applied) 562 562 + { 563 563 + disassembler_ftype disasm = dctx->disassembler; 564 564 + struct disassemble_info *dinfo = &dctx->info; 565 565 + 566 566 + dctx->insn = insn; 567 567 + dctx->alt_applied = alt_applied; 568 568 + dctx->result[0] = '\0'; 569 569 + 570 570 + if (insn->type == INSN_NOP) { 571 571 + DINFO_FPRINTF(dinfo, "nop%d", insn->len); 572 572 + return insn->len; 573 573 + } 574 574 + 575 575 + /* 576 576 + * Set the disassembler buffer to read data from the section 577 577 + * containing the instruction to disassemble. 578 578 + */ 579 579 + dinfo->buffer = insn->sec->data->d_buf; 580 580 + dinfo->buffer_vma = 0; 581 581 + dinfo->buffer_length = insn->sec->sh.sh_size; 582 582 + 583 583 + return disasm(insn->offset, &dctx->info); 584 584 + } 585 585 + 586 586 + size_t disas_insn(struct disas_context *dctx, struct instruction *insn) 587 587 + { 588 588 + return disas_insn_common(dctx, insn, false); 589 589 + } 590 590 + 591 591 + static size_t disas_insn_alt(struct disas_context *dctx, 592 592 + struct instruction *insn) 593 593 + { 594 594 + return disas_insn_common(dctx, insn, true); 595 595 + } 596 596 + 597 597 + static struct instruction *next_insn_same_alt(struct objtool_file *file, 598 598 + struct alt_group *alt_grp, 599 599 + struct instruction *insn) 600 600 + { 601 601 + if (alt_grp->last_insn == insn || alt_grp->nop == insn) 602 602 + return NULL; 603 603 + 604 604 + return next_insn_same_sec(file, insn); 605 605 + } 606 606 + 607 607 + #define alt_for_each_insn(file, alt_grp, insn) \ 608 608 + for (insn = alt_grp->first_insn; \ 609 609 + insn; \ 610 610 + insn = next_insn_same_alt(file, alt_grp, insn)) 611 611 + 612 612 + /* 613 613 + * Provide a name for the type of alternatives present at the 614 614 + * specified instruction. 615 615 + * 616 616 + * An instruction can have alternatives with different types, for 617 617 + * example alternative instructions and an exception table. In that 618 618 + * case the name for the alternative instructions type is used. 619 619 + * 620 620 + * Return NULL if the instruction as no alternative. 621 621 + */ 622 622 + const char *disas_alt_type_name(struct instruction *insn) 623 623 + { 624 624 + struct alternative *alt; 625 625 + const char *name; 626 626 + 627 627 + name = NULL; 628 628 + for (alt = insn->alts; alt; alt = alt->next) { 629 629 + if (alt->type == ALT_TYPE_INSTRUCTIONS) { 630 630 + name = "alternative"; 631 631 + break; 632 632 + } 633 633 + 634 634 + switch (alt->type) { 635 635 + case ALT_TYPE_EX_TABLE: 636 636 + name = "ex_table"; 637 637 + break; 638 638 + case ALT_TYPE_JUMP_TABLE: 639 639 + name = "jump_table"; 640 640 + break; 641 641 + default: 642 642 + name = "unknown"; 643 643 + break; 644 644 + } 645 645 + } 646 646 + 647 647 + return name; 648 648 + } 649 649 + 650 650 + /* 651 651 + * Provide a name for an alternative. 652 652 + */ 653 653 + char *disas_alt_name(struct alternative *alt) 654 654 + { 655 655 + char pfx[4] = { 0 }; 656 656 + char *str = NULL; 657 657 + const char *name; 658 658 + int feature; 659 659 + int flags; 660 660 + int num; 661 661 + 662 662 + switch (alt->type) { 663 663 + 664 664 + case ALT_TYPE_EX_TABLE: 665 665 + str = strdup("EXCEPTION"); 666 666 + break; 667 667 + 668 668 + case ALT_TYPE_JUMP_TABLE: 669 669 + str = strdup("JUMP"); 670 670 + break; 671 671 + 672 672 + case ALT_TYPE_INSTRUCTIONS: 673 673 + /* 674 674 + * This is a non-default group alternative. Create a name 675 675 + * based on the feature and flags associated with this 676 676 + * alternative. Use either the feature name (it is available) 677 677 + * or the feature number. And add a prefix to show the flags 678 678 + * used. 679 679 + * 680 680 + * Prefix flags characters: 681 681 + * 682 682 + * '!' alternative used when feature not enabled 683 683 + * '+' direct call alternative 684 684 + * '?' unknown flag 685 685 + */ 686 686 + 687 687 + if (!alt->insn->alt_group) 688 688 + return NULL; 689 689 + 690 690 + feature = alt->insn->alt_group->feature; 691 691 + num = alt_feature(feature); 692 692 + flags = alt_flags(feature); 693 693 + str = pfx; 694 694 + 695 695 + if (flags & ~(ALT_FLAG_NOT | ALT_FLAG_DIRECT_CALL)) 696 696 + *str++ = '?'; 697 697 + if (flags & ALT_FLAG_DIRECT_CALL) 698 698 + *str++ = '+'; 699 699 + if (flags & ALT_FLAG_NOT) 700 700 + *str++ = '!'; 701 701 + 702 702 + name = arch_cpu_feature_name(num); 703 703 + if (!name) 704 704 + str = strfmt("%sFEATURE 0x%X", pfx, num); 705 705 + else 706 706 + str = strfmt("%s%s", pfx, name); 707 707 + 708 708 + break; 709 709 + } 710 710 + 711 711 + return str; 712 712 + } 713 713 + 714 714 + /* 715 715 + * Initialize an alternative. The default alternative should be initialized 716 716 + * with alt=NULL. 717 717 + */ 718 718 + static int disas_alt_init(struct disas_alt *dalt, 719 719 + struct instruction *orig_insn, 720 720 + struct alternative *alt) 721 721 + { 722 722 + dalt->orig_insn = orig_insn; 723 723 + dalt->alt = alt; 724 724 + dalt->insn_idx = 0; 725 725 + dalt->name = alt ? disas_alt_name(alt) : strdup("DEFAULT"); 726 726 + if (!dalt->name) 727 727 + return -1; 728 728 + dalt->width = strlen(dalt->name); 729 729 + 730 730 + return 0; 731 731 + } 732 732 + 733 733 + static int disas_alt_add_insn(struct disas_alt *dalt, int index, char *insn_str, 734 734 + int offset, int nops) 735 735 + { 736 736 + int len; 737 737 + 738 738 + if (index >= DISAS_ALT_INSN_MAX) { 739 739 + WARN("Alternative %lx.%s has more instructions than supported", 740 740 + DALT_ALTID(dalt), dalt->name); 741 741 + return -1; 742 742 + } 743 743 + 744 744 + len = strlen(insn_str); 745 745 + dalt->insn[index].str = insn_str; 746 746 + dalt->insn[index].offset = offset; 747 747 + dalt->insn[index].nops = nops; 748 748 + if (len > dalt->width) 749 749 + dalt->width = len; 750 750 + 751 751 + return 0; 752 752 + } 753 753 + 754 754 + static int disas_alt_jump(struct disas_alt *dalt) 755 755 + { 756 756 + struct instruction *orig_insn; 757 757 + struct instruction *dest_insn; 758 758 + char suffix[2] = { 0 }; 759 759 + char *str; 760 760 + int nops; 761 761 + 762 762 + orig_insn = dalt->orig_insn; 763 763 + dest_insn = dalt->alt->insn; 764 764 + 765 765 + if (orig_insn->type == INSN_NOP) { 766 766 + if (orig_insn->len == 5) 767 767 + suffix[0] = 'q'; 768 768 + str = strfmt("jmp%-3s %lx <%s+0x%lx>", suffix, 769 769 + dest_insn->offset, dest_insn->sym->name, 770 770 + dest_insn->offset - dest_insn->sym->offset); 771 771 + nops = 0; 772 772 + } else { 773 773 + str = strfmt("nop%d", orig_insn->len); 774 774 + nops = orig_insn->len; 775 775 + } 776 776 + 777 777 + if (!str) 778 778 + return -1; 779 779 + 780 780 + disas_alt_add_insn(dalt, 0, str, 0, nops); 781 781 + 782 782 + return 1; 783 783 + } 784 784 + 785 785 + /* 786 786 + * Disassemble an exception table alternative. 787 787 + */ 788 788 + static int disas_alt_extable(struct disas_alt *dalt) 789 789 + { 790 790 + struct instruction *alt_insn; 791 791 + char *str; 792 792 + 793 793 + alt_insn = dalt->alt->insn; 794 794 + str = strfmt("resume at 0x%lx <%s+0x%lx>", 795 795 + alt_insn->offset, alt_insn->sym->name, 796 796 + alt_insn->offset - alt_insn->sym->offset); 797 797 + if (!str) 798 798 + return -1; 799 799 + 800 800 + disas_alt_add_insn(dalt, 0, str, 0, 0); 801 801 + 802 802 + return 1; 803 803 + } 804 804 + 805 805 + /* 806 806 + * Disassemble an alternative and store instructions in the disas_alt 807 807 + * structure. Return the number of instructions in the alternative. 808 808 + */ 809 809 + static int disas_alt_group(struct disas_context *dctx, struct disas_alt *dalt) 810 810 + { 811 811 + struct objtool_file *file; 812 812 + struct instruction *insn; 813 813 + int offset; 814 814 + char *str; 815 815 + int count; 816 816 + int nops; 817 817 + int err; 818 818 + 819 819 + file = dctx->file; 820 820 + count = 0; 821 821 + offset = 0; 822 822 + nops = 0; 823 823 + 824 824 + alt_for_each_insn(file, DALT_GROUP(dalt), insn) { 825 825 + 826 826 + disas_insn_alt(dctx, insn); 827 827 + str = strdup(disas_result(dctx)); 828 828 + if (!str) 829 829 + return -1; 830 830 + 831 831 + nops = insn->type == INSN_NOP ? insn->len : 0; 832 832 + err = disas_alt_add_insn(dalt, count, str, offset, nops); 833 833 + if (err) 834 834 + break; 835 835 + offset += insn->len; 836 836 + count++; 837 837 + } 838 838 + 839 839 + return count; 840 840 + } 841 841 + 842 842 + /* 843 843 + * Disassemble the default alternative. 844 844 + */ 845 845 + static int disas_alt_default(struct disas_context *dctx, struct disas_alt *dalt) 846 846 + { 847 847 + char *str; 848 848 + int nops; 849 849 + int err; 850 850 + 851 851 + if (DALT_GROUP(dalt)) 852 852 + return disas_alt_group(dctx, dalt); 853 853 + 854 854 + /* 855 855 + * Default alternative with no alt_group: this is the default 856 856 + * code associated with either a jump table or an exception 857 857 + * table and no other instruction alternatives. In that case 858 858 + * the default alternative is made of a single instruction. 859 859 + */ 860 860 + disas_insn(dctx, dalt->orig_insn); 861 861 + str = strdup(disas_result(dctx)); 862 862 + if (!str) 863 863 + return -1; 864 864 + nops = dalt->orig_insn->type == INSN_NOP ? dalt->orig_insn->len : 0; 865 865 + err = disas_alt_add_insn(dalt, 0, str, 0, nops); 866 866 + if (err) 867 867 + return -1; 868 868 + 869 869 + return 1; 870 870 + } 871 871 + 872 872 + /* 873 873 + * For each alternative, if there is an instruction at the specified 874 874 + * offset then print this instruction, otherwise print a blank entry. 875 875 + * The offset is an offset from the start of the alternative. 876 876 + * 877 877 + * Return the offset for the next instructions to print, or -1 if all 878 878 + * instructions have been printed. 879 879 + */ 880 880 + static int disas_alt_print_insn(struct disas_alt *dalts, int alt_count, 881 881 + int insn_count, int offset) 882 882 + { 883 883 + struct disas_alt *dalt; 884 884 + int offset_next; 885 885 + char *str; 886 886 + int i, j; 887 887 + 888 888 + offset_next = -1; 889 889 + 890 890 + for (i = 0; i < alt_count; i++) { 891 891 + dalt = &dalts[i]; 892 892 + j = dalt->insn_idx; 893 893 + if (j == -1) { 894 894 + printf("| %-*s ", dalt->width, ""); 895 895 + continue; 896 896 + } 897 897 + 898 898 + if (dalt->insn[j].offset == offset) { 899 899 + str = dalt->insn[j].str; 900 900 + printf("| %-*s ", dalt->width, str ?: ""); 901 901 + if (++j < insn_count) { 902 902 + dalt->insn_idx = j; 903 903 + } else { 904 904 + dalt->insn_idx = -1; 905 905 + continue; 906 906 + } 907 907 + } else { 908 908 + printf("| %-*s ", dalt->width, ""); 909 909 + } 910 910 + 911 911 + if (dalt->insn[j].offset > 0 && 912 912 + (offset_next == -1 || 913 913 + (dalt->insn[j].offset < offset_next))) 914 914 + offset_next = dalt->insn[j].offset; 915 915 + } 916 916 + printf("\n"); 917 917 + 918 918 + return offset_next; 919 919 + } 920 920 + 921 921 + /* 922 922 + * Print all alternatives side-by-side. 923 923 + */ 924 924 + static void disas_alt_print_wide(char *alt_name, struct disas_alt *dalts, int alt_count, 925 925 + int insn_count) 926 926 + { 927 927 + struct instruction *orig_insn; 928 928 + int offset_next; 929 929 + int offset; 930 930 + int i; 931 931 + 932 932 + orig_insn = dalts[0].orig_insn; 933 933 + 934 934 + /* 935 935 + * Print an header with the name of each alternative. 936 936 + */ 937 937 + disas_print_info(stdout, orig_insn, -2, NULL); 938 938 + 939 939 + if (strlen(alt_name) > dalts[0].width) 940 940 + dalts[0].width = strlen(alt_name); 941 941 + printf("| %-*s ", dalts[0].width, alt_name); 942 942 + 943 943 + for (i = 1; i < alt_count; i++) 944 944 + printf("| %-*s ", dalts[i].width, dalts[i].name); 945 945 + 946 946 + printf("\n"); 947 947 + 948 948 + /* 949 949 + * Print instructions for each alternative. 950 950 + */ 951 951 + offset_next = 0; 952 952 + do { 953 953 + offset = offset_next; 954 954 + disas_print(stdout, orig_insn->sec, orig_insn->offset + offset, 955 955 + -2, NULL); 956 956 + offset_next = disas_alt_print_insn(dalts, alt_count, insn_count, 957 957 + offset); 958 958 + } while (offset_next > offset); 959 959 + } 960 960 + 961 961 + /* 962 962 + * Print all alternatives one above the other. 963 963 + */ 964 964 + static void disas_alt_print_compact(char *alt_name, struct disas_alt *dalts, 965 965 + int alt_count, int insn_count) 966 966 + { 967 967 + struct instruction *orig_insn; 968 968 + int width; 969 969 + int i, j; 970 970 + int len; 971 971 + 972 972 + orig_insn = dalts[0].orig_insn; 973 973 + 974 974 + len = disas_print(stdout, orig_insn->sec, orig_insn->offset, 0, NULL); 975 975 + printf("%s\n", alt_name); 976 976 + 977 977 + /* 978 978 + * If all alternatives have a single instruction then print each 979 979 + * alternative on a single line. Otherwise, print alternatives 980 980 + * one above the other with a clear separation. 981 981 + */ 982 982 + 983 983 + if (insn_count == 1) { 984 984 + width = 0; 985 985 + for (i = 0; i < alt_count; i++) { 986 986 + if (dalts[i].width > width) 987 987 + width = dalts[i].width; 988 988 + } 989 989 + 990 990 + for (i = 0; i < alt_count; i++) { 991 991 + printf("%*s= %-*s (if %s)\n", len, "", width, 992 992 + dalts[i].insn[0].str, dalts[i].name); 993 993 + } 994 994 + 995 995 + return; 996 996 + } 997 997 + 998 998 + for (i = 0; i < alt_count; i++) { 999 999 + printf("%*s= %s\n", len, "", dalts[i].name); 1000 1000 + for (j = 0; j < insn_count; j++) { 1001 1001 + if (!dalts[i].insn[j].str) 1002 1002 + break; 1003 1003 + disas_print(stdout, orig_insn->sec, 1004 1004 + orig_insn->offset + dalts[i].insn[j].offset, 0, 1005 1005 + "| %s\n", dalts[i].insn[j].str); 1006 1006 + } 1007 1007 + printf("%*s|\n", len, ""); 1008 1008 + } 1009 1009 + } 1010 1010 + 1011 1011 + /* 1012 1012 + * Trim NOPs in alternatives. This replaces trailing NOPs in alternatives 1013 1013 + * with a single indication of the number of bytes covered with NOPs. 1014 1014 + * 1015 1015 + * Return the maximum numbers of instructions in all alternatives after 1016 1016 + * trailing NOPs have been trimmed. 1017 1017 + */ 1018 1018 + static int disas_alt_trim_nops(struct disas_alt *dalts, int alt_count, 1019 1019 + int insn_count) 1020 1020 + { 1021 1021 + struct disas_alt *dalt; 1022 1022 + int nops_count; 1023 1023 + const char *s; 1024 1024 + int offset; 1025 1025 + int count; 1026 1026 + int nops; 1027 1027 + int i, j; 1028 1028 + 1029 1029 + count = 0; 1030 1030 + for (i = 0; i < alt_count; i++) { 1031 1031 + offset = 0; 1032 1032 + nops = 0; 1033 1033 + nops_count = 0; 1034 1034 + dalt = &dalts[i]; 1035 1035 + for (j = insn_count - 1; j >= 0; j--) { 1036 1036 + if (!dalt->insn[j].str || !dalt->insn[j].nops) 1037 1037 + break; 1038 1038 + offset = dalt->insn[j].offset; 1039 1039 + free(dalt->insn[j].str); 1040 1040 + dalt->insn[j].offset = 0; 1041 1041 + dalt->insn[j].str = NULL; 1042 1042 + nops += dalt->insn[j].nops; 1043 1043 + nops_count++; 1044 1044 + } 1045 1045 + 1046 1046 + /* 1047 1047 + * All trailing NOPs have been removed. If there was a single 1048 1048 + * NOP instruction then re-add it. If there was a block of 1049 1049 + * NOPs then indicate the number of bytes than the block 1050 1050 + * covers (nop*<number-of-bytes>). 1051 1051 + */ 1052 1052 + if (nops_count) { 1053 1053 + s = nops_count == 1 ? "" : "*"; 1054 1054 + dalt->insn[j + 1].str = strfmt("nop%s%d", s, nops); 1055 1055 + dalt->insn[j + 1].offset = offset; 1056 1056 + dalt->insn[j + 1].nops = nops; 1057 1057 + j++; 1058 1058 + } 1059 1059 + 1060 1060 + if (j > count) 1061 1061 + count = j; 1062 1062 + } 1063 1063 + 1064 1064 + return count + 1; 1065 1065 + } 1066 1066 + 1067 1067 + /* 1068 1068 + * Disassemble an alternative. 1069 1069 + * 1070 1070 + * Return the last instruction in the default alternative so that 1071 1071 + * disassembly can continue with the next instruction. Return NULL 1072 1072 + * on error. 1073 1073 + */ 1074 1074 + static void *disas_alt(struct disas_context *dctx, 1075 1075 + struct instruction *orig_insn) 1076 1076 + { 1077 1077 + struct disas_alt dalts[DISAS_ALT_MAX] = { 0 }; 1078 1078 + struct instruction *last_insn = NULL; 1079 1079 + struct alternative *alt; 1080 1080 + struct disas_alt *dalt; 1081 1081 + int insn_count = 0; 1082 1082 + int alt_count = 0; 1083 1083 + char *alt_name; 1084 1084 + int count; 1085 1085 + int i, j; 1086 1086 + int err; 1087 1087 + 1088 1088 + alt_name = strfmt("<%s.%lx>", disas_alt_type_name(orig_insn), 1089 1089 + orig_insn->offset); 1090 1090 + if (!alt_name) { 1091 1091 + WARN("Failed to define name for alternative at instruction 0x%lx", 1092 1092 + orig_insn->offset); 1093 1093 + goto done; 1094 1094 + } 1095 1095 + 1096 1096 + /* 1097 1097 + * Initialize and disassemble the default alternative. 1098 1098 + */ 1099 1099 + err = disas_alt_init(&dalts[0], orig_insn, NULL); 1100 1100 + if (err) { 1101 1101 + WARN("%s: failed to initialize default alternative", alt_name); 1102 1102 + goto done; 1103 1103 + } 1104 1104 + 1105 1105 + insn_count = disas_alt_default(dctx, &dalts[0]); 1106 1106 + if (insn_count < 0) { 1107 1107 + WARN("%s: failed to disassemble default alternative", alt_name); 1108 1108 + goto done; 1109 1109 + } 1110 1110 + 1111 1111 + /* 1112 1112 + * Initialize and disassemble all other alternatives. 1113 1113 + */ 1114 1114 + i = 1; 1115 1115 + for (alt = orig_insn->alts; alt; alt = alt->next) { 1116 1116 + if (i >= DISAS_ALT_MAX) { 1117 1117 + WARN("%s has more alternatives than supported", alt_name); 1118 1118 + break; 1119 1119 + } 1120 1120 + 1121 1121 + dalt = &dalts[i]; 1122 1122 + err = disas_alt_init(dalt, orig_insn, alt); 1123 1123 + if (err) { 1124 1124 + WARN("%s: failed to disassemble alternative", alt_name); 1125 1125 + goto done; 1126 1126 + } 1127 1127 + 1128 1128 + count = -1; 1129 1129 + switch (dalt->alt->type) { 1130 1130 + case ALT_TYPE_INSTRUCTIONS: 1131 1131 + count = disas_alt_group(dctx, dalt); 1132 1132 + break; 1133 1133 + case ALT_TYPE_EX_TABLE: 1134 1134 + count = disas_alt_extable(dalt); 1135 1135 + break; 1136 1136 + case ALT_TYPE_JUMP_TABLE: 1137 1137 + count = disas_alt_jump(dalt); 1138 1138 + break; 1139 1139 + } 1140 1140 + if (count < 0) { 1141 1141 + WARN("%s: failed to disassemble alternative %s", 1142 1142 + alt_name, dalt->name); 1143 1143 + goto done; 1144 1144 + } 1145 1145 + 1146 1146 + insn_count = count > insn_count ? count : insn_count; 1147 1147 + i++; 1148 1148 + } 1149 1149 + alt_count = i; 1150 1150 + 1151 1151 + /* 1152 1152 + * Print default and non-default alternatives. 1153 1153 + */ 1154 1154 + 1155 1155 + insn_count = disas_alt_trim_nops(dalts, alt_count, insn_count); 1156 1156 + 1157 1157 + if (opts.wide) 1158 1158 + disas_alt_print_wide(alt_name, dalts, alt_count, insn_count); 1159 1159 + else 1160 1160 + disas_alt_print_compact(alt_name, dalts, alt_count, insn_count); 1161 1161 + 1162 1162 + last_insn = orig_insn->alt_group ? orig_insn->alt_group->last_insn : 1163 1163 + orig_insn; 1164 1164 + 1165 1165 + done: 1166 1166 + for (i = 0; i < alt_count; i++) { 1167 1167 + free(dalts[i].name); 1168 1168 + for (j = 0; j < insn_count; j++) 1169 1169 + free(dalts[i].insn[j].str); 1170 1170 + } 1171 1171 + 1172 1172 + free(alt_name); 1173 1173 + 1174 1174 + return last_insn; 1175 1175 + } 1176 1176 + 1177 1177 + /* 1178 1178 + * Disassemble a function. 1179 1179 + */ 1180 1180 + static void disas_func(struct disas_context *dctx, struct symbol *func) 1181 1181 + { 1182 1182 + struct instruction *insn_start; 1183 1183 + struct instruction *insn; 1184 1184 + 1185 1185 + printf("%s:\n", func->name); 1186 1186 + sym_for_each_insn(dctx->file, func, insn) { 1187 1187 + if (insn->alts) { 1188 1188 + insn_start = insn; 1189 1189 + insn = disas_alt(dctx, insn); 1190 1190 + if (insn) 1191 1191 + continue; 1192 1192 + /* 1193 1193 + * There was an error with disassembling 1194 1194 + * the alternative. Resume disassembling 1195 1195 + * at the current instruction, this will 1196 1196 + * disassemble the default alternative 1197 1197 + * only and continue with the code after 1198 1198 + * the alternative. 1199 1199 + */ 1200 1200 + insn = insn_start; 1201 1201 + } 1202 1202 + 1203 1203 + DISAS_PRINSN(dctx, insn, 0); 1204 1204 + } 1205 1205 + printf("\n"); 1206 1206 + } 1207 1207 + 1208 1208 + /* 1209 1209 + * Disassemble all warned functions. 1210 1210 + */ 1211 1211 + void disas_warned_funcs(struct disas_context *dctx) 1212 1212 + { 1213 1213 + struct symbol *sym; 1214 1214 + 1215 1215 + if (!dctx) 1216 1216 + return; 1217 1217 + 1218 1218 + for_each_sym(dctx->file->elf, sym) { 1219 1219 + if (sym->warned) 1220 1220 + disas_func(dctx, sym); 1221 1221 + } 1222 1222 + } 1223 1223 + 1224 1224 + void disas_funcs(struct disas_context *dctx) 1225 1225 + { 1226 1226 + bool disas_all = !strcmp(opts.disas, "*"); 1227 1227 + struct section *sec; 1228 1228 + struct symbol *sym; 1229 1229 + 1230 1230 + for_each_sec(dctx->file->elf, sec) { 1231 1231 + 1232 1232 + if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 1233 1233 + continue; 1234 1234 + 1235 1235 + sec_for_each_sym(sec, sym) { 1236 1236 + /* 1237 1237 + * If the function had a warning and the verbose 1238 1238 + * option is used then the function was already 1239 1239 + * disassemble. 1240 1240 + */ 1241 1241 + if (opts.verbose && sym->warned) 1242 1242 + continue; 1243 1243 + 1244 1244 + if (disas_all || fnmatch(opts.disas, sym->name, 0) == 0) 1245 1245 + disas_func(dctx, sym); 1246 1246 + } 1247 1247 + } 1248 1248 + }

+630 -208

tools/objtool/elf.c

reviewed

··· 16 16 #include <string.h> 17 17 #include <unistd.h> 18 18 #include <errno.h> 19 19 + #include <libgen.h> 20 20 + #include <ctype.h> 19 21 #include <linux/interval_tree_generic.h> 20 22 #include <objtool/builtin.h> 21 21 - 22 23 #include <objtool/elf.h> 23 24 #include <objtool/warn.h> 25 25 + 26 26 + #define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1)) 27 27 + #define ALIGN_UP_POW2(x) (1U << ((8 * sizeof(x)) - __builtin_clz((x) - 1U))) 28 28 + #define MAX(a, b) ((a) > (b) ? (a) : (b)) 24 29 25 30 static inline u32 str_hash(const char *str) 26 31 { ··· 97 92 98 93 static inline unsigned long __sym_last(struct symbol *s) 99 94 { 100 100 - return s->offset + s->len - 1; 95 95 + return s->offset + (s->len ? s->len - 1 : 0); 101 96 } 102 97 103 98 INTERVAL_TREE_DEFINE(struct symbol, node, unsigned long, __subtree_last, 104 104 - __sym_start, __sym_last, static, __sym) 99 99 + __sym_start, __sym_last, static inline __maybe_unused, 100 100 + __sym) 105 101 106 102 #define __sym_for_each(_iter, _tree, _start, _end) \ 107 103 for (_iter = __sym_iter_first((_tree), (_start), (_end)); \ ··· 114 108 }; 115 109 116 110 /* 117 117 - * Find !section symbol where @offset is after it. 111 111 + * Find the last symbol before @offset. 118 112 */ 119 113 static int symbol_hole_by_offset(const void *key, const struct rb_node *node) 120 114 { ··· 125 119 return -1; 126 120 127 121 if (sh->key >= s->offset + s->len) { 128 128 - if (s->type != STT_SECTION) 129 129 - sh->sym = s; 122 122 + sh->sym = s; 130 123 return 1; 131 124 } 132 125 ··· 172 167 struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset) 173 168 { 174 169 struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; 175 175 - struct symbol *iter; 170 170 + struct symbol *sym; 176 171 177 177 - __sym_for_each(iter, tree, offset, offset) { 178 178 - if (iter->offset == offset && iter->type != STT_SECTION) 179 179 - return iter; 172 172 + __sym_for_each(sym, tree, offset, offset) { 173 173 + if (sym->offset == offset && !is_sec_sym(sym)) 174 174 + return sym->alias; 180 175 } 181 176 182 177 return NULL; ··· 185 180 struct symbol *find_func_by_offset(struct section *sec, unsigned long offset) 186 181 { 187 182 struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; 188 188 - struct symbol *iter; 183 183 + struct symbol *func; 189 184 190 190 - __sym_for_each(iter, tree, offset, offset) { 191 191 - if (iter->offset == offset && iter->type == STT_FUNC) 192 192 - return iter; 185 185 + __sym_for_each(func, tree, offset, offset) { 186 186 + if (func->offset == offset && is_func_sym(func)) 187 187 + return func->alias; 193 188 } 194 189 195 190 return NULL; ··· 198 193 struct symbol *find_symbol_containing(const struct section *sec, unsigned long offset) 199 194 { 200 195 struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; 201 201 - struct symbol *iter; 196 196 + struct symbol *sym = NULL, *tmp; 202 197 203 203 - __sym_for_each(iter, tree, offset, offset) { 204 204 - if (iter->type != STT_SECTION) 205 205 - return iter; 198 198 + __sym_for_each(tmp, tree, offset, offset) { 199 199 + if (tmp->len) { 200 200 + if (!sym) { 201 201 + sym = tmp; 202 202 + continue; 203 203 + } 204 204 + 205 205 + if (sym->offset != tmp->offset || sym->len != tmp->len) { 206 206 + /* 207 207 + * In the rare case of overlapping symbols, 208 208 + * pick the smaller one. 209 209 + * 210 210 + * TODO: outlaw overlapping symbols 211 211 + */ 212 212 + if (tmp->len < sym->len) 213 213 + sym = tmp; 214 214 + } 215 215 + } 206 216 } 207 217 208 208 - return NULL; 218 218 + return sym ? sym->alias : NULL; 209 219 } 210 220 211 221 /* ··· 266 246 struct symbol *find_func_containing(struct section *sec, unsigned long offset) 267 247 { 268 248 struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; 269 269 - struct symbol *iter; 249 249 + struct symbol *func; 270 250 271 271 - __sym_for_each(iter, tree, offset, offset) { 272 272 - if (iter->type == STT_FUNC) 273 273 - return iter; 251 251 + __sym_for_each(func, tree, offset, offset) { 252 252 + if (is_func_sym(func)) 253 253 + return func->alias; 274 254 } 275 255 276 256 return NULL; ··· 282 262 283 263 elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(name)) { 284 264 if (!strcmp(sym->name, name)) 265 265 + return sym; 266 266 + } 267 267 + 268 268 + return NULL; 269 269 + } 270 270 + 271 271 + /* Find local symbol with matching STT_FILE */ 272 272 + static struct symbol *find_local_symbol_by_file_and_name(const struct elf *elf, 273 273 + struct symbol *file, 274 274 + const char *name) 275 275 + { 276 276 + struct symbol *sym; 277 277 + 278 278 + elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(name)) { 279 279 + if (sym->bind == STB_LOCAL && sym->file == file && 280 280 + !strcmp(sym->name, name)) { 281 281 + return sym; 282 282 + } 283 283 + } 284 284 + 285 285 + return NULL; 286 286 + } 287 287 + 288 288 + struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *name) 289 289 + { 290 290 + struct symbol *sym; 291 291 + 292 292 + elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(name)) { 293 293 + if (!strcmp(sym->name, name) && !is_local_sym(sym)) 285 294 return sym; 286 295 } 287 296 ··· 407 358 return -1; 408 359 } 409 360 410 410 - if (sec->sh.sh_size != 0 && !is_dwarf_section(sec)) { 361 361 + if (sec_size(sec) != 0 && !is_dwarf_section(sec)) { 411 362 sec->data = elf_getdata(s, NULL); 412 363 if (!sec->data) { 413 364 ERROR_ELF("elf_getdata"); 414 365 return -1; 415 366 } 416 367 if (sec->data->d_off != 0 || 417 417 - sec->data->d_size != sec->sh.sh_size) { 368 368 + sec->data->d_size != sec_size(sec)) { 418 369 ERROR("unexpected data attributes for %s", sec->name); 419 370 return -1; 420 371 } ··· 442 393 return 0; 443 394 } 444 395 445 445 - static void elf_add_symbol(struct elf *elf, struct symbol *sym) 396 396 + static const char *demangle_name(struct symbol *sym) 397 397 + { 398 398 + char *str; 399 399 + 400 400 + if (!is_local_sym(sym)) 401 401 + return sym->name; 402 402 + 403 403 + if (!is_func_sym(sym) && !is_object_sym(sym)) 404 404 + return sym->name; 405 405 + 406 406 + if (!strstarts(sym->name, "__UNIQUE_ID_") && !strchr(sym->name, '.')) 407 407 + return sym->name; 408 408 + 409 409 + str = strdup(sym->name); 410 410 + if (!str) { 411 411 + ERROR_GLIBC("strdup"); 412 412 + return NULL; 413 413 + } 414 414 + 415 415 + for (int i = strlen(str) - 1; i >= 0; i--) { 416 416 + char c = str[i]; 417 417 + 418 418 + if (!isdigit(c) && c != '.') { 419 419 + str[i + 1] = '\0'; 420 420 + break; 421 421 + } 422 422 + } 423 423 + 424 424 + return str; 425 425 + } 426 426 + 427 427 + static int elf_add_symbol(struct elf *elf, struct symbol *sym) 446 428 { 447 429 struct list_head *entry; 448 430 struct rb_node *pnode; ··· 485 405 sym->type = GELF_ST_TYPE(sym->sym.st_info); 486 406 sym->bind = GELF_ST_BIND(sym->sym.st_info); 487 407 488 488 - if (sym->type == STT_FILE) 408 408 + if (is_file_sym(sym)) 489 409 elf->num_files++; 490 410 491 411 sym->offset = sym->sym.st_value; 492 412 sym->len = sym->sym.st_size; 493 413 494 414 __sym_for_each(iter, &sym->sec->symbol_tree, sym->offset, sym->offset) { 495 495 - if (iter->offset == sym->offset && iter->type == sym->type) 415 415 + if (!is_undef_sym(iter) && iter->offset == sym->offset && 416 416 + iter->type == sym->type && iter->len == sym->len) 496 417 iter->alias = sym; 497 418 } 498 419 ··· 504 423 else 505 424 entry = &sym->sec->symbol_list; 506 425 list_add(&sym->list, entry); 426 426 + 427 427 + list_add_tail(&sym->global_list, &elf->symbols); 507 428 elf_hash_add(symbol, &sym->hash, sym->idx); 508 429 elf_hash_add(symbol_name, &sym->name_hash, str_hash(sym->name)); 509 430 510 510 - /* 511 511 - * Don't store empty STT_NOTYPE symbols in the rbtree. They 512 512 - * can exist within a function, confusing the sorting. 513 513 - */ 514 514 - if (!sym->len) 515 515 - __sym_remove(sym, &sym->sec->symbol_tree); 431 431 + if (is_func_sym(sym) && 432 432 + (strstarts(sym->name, "__pfx_") || 433 433 + strstarts(sym->name, "__cfi_") || 434 434 + strstarts(sym->name, "__pi___pfx_") || 435 435 + strstarts(sym->name, "__pi___cfi_"))) 436 436 + sym->prefix = 1; 437 437 + 438 438 + if (strstarts(sym->name, ".klp.sym")) 439 439 + sym->klp = 1; 440 440 + 441 441 + if (!sym->klp && !is_sec_sym(sym) && strstr(sym->name, ".cold")) { 442 442 + sym->cold = 1; 443 443 + 444 444 + /* 445 445 + * Clang doesn't mark cold subfunctions as STT_FUNC, which 446 446 + * breaks several objtool assumptions. Fake it. 447 447 + */ 448 448 + sym->type = STT_FUNC; 449 449 + } 450 450 + 451 451 + sym->pfunc = sym->cfunc = sym; 452 452 + 453 453 + sym->demangled_name = demangle_name(sym); 454 454 + if (!sym->demangled_name) 455 455 + return -1; 456 456 + 457 457 + return 0; 516 458 } 517 459 518 460 static int read_symbols(struct elf *elf) 519 461 { 520 462 struct section *symtab, *symtab_shndx, *sec; 521 521 - struct symbol *sym, *pfunc; 463 463 + struct symbol *sym, *pfunc, *file = NULL; 522 464 int symbols_nr, i; 523 465 char *coldstr; 524 466 Elf_Data *shndx_data = NULL; ··· 573 469 ERROR_GLIBC("calloc"); 574 470 return -1; 575 471 } 472 472 + 473 473 + INIT_LIST_HEAD(&elf->symbols); 474 474 + 576 475 for (i = 0; i < symbols_nr; i++) { 577 476 sym = &elf->symbol_data[i]; 578 477 ··· 584 477 if (!gelf_getsymshndx(symtab->data, shndx_data, i, &sym->sym, 585 478 &shndx)) { 586 479 ERROR_ELF("gelf_getsymshndx"); 587 587 - goto err; 480 480 + return -1; 588 481 } 589 482 590 483 sym->name = elf_strptr(elf->elf, symtab->sh.sh_link, 591 484 sym->sym.st_name); 592 485 if (!sym->name) { 593 486 ERROR_ELF("elf_strptr"); 594 594 - goto err; 487 487 + return -1; 595 488 } 596 489 597 490 if ((sym->sym.st_shndx > SHN_UNDEF && ··· 603 496 sym->sec = find_section_by_index(elf, shndx); 604 497 if (!sym->sec) { 605 498 ERROR("couldn't find section for symbol %s", sym->name); 606 606 - goto err; 499 499 + return -1; 607 500 } 608 501 if (GELF_ST_TYPE(sym->sym.st_info) == STT_SECTION) { 609 502 sym->name = sym->sec->name; ··· 612 505 } else 613 506 sym->sec = find_section_by_index(elf, 0); 614 507 615 615 - elf_add_symbol(elf, sym); 508 508 + if (elf_add_symbol(elf, sym)) 509 509 + return -1; 510 510 + 511 511 + if (sym->type == STT_FILE) 512 512 + file = sym; 513 513 + else if (sym->bind == STB_LOCAL) 514 514 + sym->file = file; 616 515 } 617 516 618 517 if (opts.stats) { ··· 631 518 sec_for_each_sym(sec, sym) { 632 519 char *pname; 633 520 size_t pnamelen; 634 634 - if (sym->type != STT_FUNC) 521 521 + 522 522 + if (!sym->cold) 635 523 continue; 636 636 - 637 637 - if (sym->pfunc == NULL) 638 638 - sym->pfunc = sym; 639 639 - 640 640 - if (sym->cfunc == NULL) 641 641 - sym->cfunc = sym; 642 524 643 525 coldstr = strstr(sym->name, ".cold"); 644 644 - if (!coldstr) 645 645 - continue; 526 526 + if (!coldstr) { 527 527 + ERROR("%s(): cold subfunction without \".cold\"?", sym->name); 528 528 + return -1; 529 529 + } 646 530 647 531 pnamelen = coldstr - sym->name; 648 532 pname = strndup(sym->name, pnamelen); ··· 648 538 return -1; 649 539 } 650 540 651 651 - pfunc = find_symbol_by_name(elf, pname); 541 541 + pfunc = find_local_symbol_by_file_and_name(elf, sym->file, pname); 542 542 + if (!pfunc) 543 543 + pfunc = find_global_symbol_by_name(elf, pname); 652 544 free(pname); 653 545 654 546 if (!pfunc) { ··· 658 546 return -1; 659 547 } 660 548 661 661 - sym->pfunc = pfunc; 549 549 + sym->pfunc = pfunc->alias; 662 550 pfunc->cfunc = sym; 551 551 + pfunc->alias->cfunc = sym; 663 552 664 553 /* 665 554 * Unfortunately, -fnoreorder-functions puts the child ··· 679 566 } 680 567 681 568 return 0; 682 682 - 683 683 - err: 684 684 - free(sym); 685 685 - return -1; 686 569 } 687 570 688 571 static int mark_group_syms(struct elf *elf) ··· 692 583 return -1; 693 584 } 694 585 695 695 - list_for_each_entry(sec, &elf->sections, list) { 586 586 + for_each_sec(elf, sec) { 696 587 if (sec->sh.sh_type == SHT_GROUP && 697 588 sec->sh.sh_link == symtab->idx) { 698 589 sym = find_symbol_by_index(elf, sec->sh.sh_info); ··· 733 624 static int elf_update_symbol(struct elf *elf, struct section *symtab, 734 625 struct section *symtab_shndx, struct symbol *sym) 735 626 { 736 736 - Elf32_Word shndx = sym->sec ? sym->sec->idx : SHN_UNDEF; 627 627 + Elf32_Word shndx; 737 628 Elf_Data *symtab_data = NULL, *shndx_data = NULL; 738 629 Elf64_Xword entsize = symtab->sh.sh_entsize; 739 630 int max_idx, idx = sym->idx; ··· 741 632 bool is_special_shndx = sym->sym.st_shndx >= SHN_LORESERVE && 742 633 sym->sym.st_shndx != SHN_XINDEX; 743 634 744 744 - if (is_special_shndx) 745 745 - shndx = sym->sym.st_shndx; 635 635 + shndx = is_special_shndx ? sym->sym.st_shndx : sym->sec->idx; 746 636 747 637 s = elf_getscn(elf->elf, symtab->idx); 748 638 if (!s) { ··· 839 731 } 840 732 841 733 /* setup extended section index magic and write the symbol */ 842 842 - if ((shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) || is_special_shndx) { 734 734 + if (shndx < SHN_LORESERVE || is_special_shndx) { 843 735 sym->sym.st_shndx = shndx; 844 736 if (!shndx_data) 845 737 shndx = 0; ··· 859 751 return 0; 860 752 } 861 753 862 862 - static struct symbol * 863 863 - __elf_create_symbol(struct elf *elf, struct symbol *sym) 754 754 + struct symbol *elf_create_symbol(struct elf *elf, const char *name, 755 755 + struct section *sec, unsigned int bind, 756 756 + unsigned int type, unsigned long offset, 757 757 + size_t size) 864 758 { 865 759 struct section *symtab, *symtab_shndx; 866 760 Elf32_Word first_non_local, new_idx; 867 867 - struct symbol *old; 761 761 + struct symbol *old, *sym; 762 762 + 763 763 + sym = calloc(1, sizeof(*sym)); 764 764 + if (!sym) { 765 765 + ERROR_GLIBC("calloc"); 766 766 + return NULL; 767 767 + } 768 768 + 769 769 + sym->name = strdup(name); 770 770 + if (!sym->name) { 771 771 + ERROR_GLIBC("strdup"); 772 772 + return NULL; 773 773 + } 774 774 + 775 775 + if (type != STT_SECTION) { 776 776 + sym->sym.st_name = elf_add_string(elf, NULL, sym->name); 777 777 + if (sym->sym.st_name == -1) 778 778 + return NULL; 779 779 + } 780 780 + 781 781 + if (sec) { 782 782 + sym->sec = sec; 783 783 + } else { 784 784 + sym->sec = find_section_by_index(elf, 0); 785 785 + if (!sym->sec) { 786 786 + ERROR("no NULL section"); 787 787 + return NULL; 788 788 + } 789 789 + } 790 790 + 791 791 + sym->sym.st_info = GELF_ST_INFO(bind, type); 792 792 + sym->sym.st_value = offset; 793 793 + sym->sym.st_size = size; 868 794 869 795 symtab = find_section_by_name(elf, ".symtab"); 870 870 - if (symtab) { 871 871 - symtab_shndx = find_section_by_name(elf, ".symtab_shndx"); 872 872 - } else { 796 796 + if (!symtab) { 873 797 ERROR("no .symtab"); 874 798 return NULL; 875 799 } 876 800 801 801 + symtab_shndx = find_section_by_name(elf, ".symtab_shndx"); 802 802 + 877 803 new_idx = sec_num_entries(symtab); 878 804 879 879 - if (GELF_ST_BIND(sym->sym.st_info) != STB_LOCAL) 805 805 + if (bind != STB_LOCAL) 880 806 goto non_local; 881 807 882 808 /* ··· 948 806 949 807 non_local: 950 808 sym->idx = new_idx; 951 951 - if (elf_update_symbol(elf, symtab, symtab_shndx, sym)) { 952 952 - ERROR("elf_update_symbol"); 809 809 + if (sym->idx && elf_update_symbol(elf, symtab, symtab_shndx, sym)) 953 810 return NULL; 954 954 - } 955 811 956 812 symtab->sh.sh_size += symtab->sh.sh_entsize; 957 813 mark_sec_changed(elf, symtab, true); ··· 959 819 mark_sec_changed(elf, symtab_shndx, true); 960 820 } 961 821 822 822 + if (elf_add_symbol(elf, sym)) 823 823 + return NULL; 824 824 + 962 825 return sym; 963 826 } 964 827 965 965 - static struct symbol * 966 966 - elf_create_section_symbol(struct elf *elf, struct section *sec) 828 828 + struct symbol *elf_create_section_symbol(struct elf *elf, struct section *sec) 967 829 { 968 830 struct symbol *sym = calloc(1, sizeof(*sym)); 969 831 970 970 - if (!sym) { 971 971 - ERROR_GLIBC("malloc"); 832 832 + sym = elf_create_symbol(elf, sec->name, sec, STB_LOCAL, STT_SECTION, 0, 0); 833 833 + if (!sym) 972 834 return NULL; 973 973 - } 974 835 975 975 - sym->name = sec->name; 976 976 - sym->sec = sec; 977 977 - 978 978 - // st_name 0 979 979 - sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION); 980 980 - // st_other 0 981 981 - // st_value 0 982 982 - // st_size 0 983 983 - 984 984 - sym = __elf_create_symbol(elf, sym); 985 985 - if (sym) 986 986 - elf_add_symbol(elf, sym); 836 836 + sec->sym = sym; 987 837 988 838 return sym; 989 839 } 990 840 991 991 - static int elf_add_string(struct elf *elf, struct section *strtab, char *str); 992 992 - 993 993 - struct symbol * 994 994 - elf_create_prefix_symbol(struct elf *elf, struct symbol *orig, long size) 995 995 - { 996 996 - struct symbol *sym = calloc(1, sizeof(*sym)); 997 997 - size_t namelen = strlen(orig->name) + sizeof("__pfx_"); 998 998 - char *name = malloc(namelen); 999 999 - 1000 1000 - if (!sym || !name) { 1001 1001 - ERROR_GLIBC("malloc"); 1002 1002 - return NULL; 1003 1003 - } 1004 1004 - 1005 1005 - snprintf(name, namelen, "__pfx_%s", orig->name); 1006 1006 - 1007 1007 - sym->name = name; 1008 1008 - sym->sec = orig->sec; 1009 1009 - 1010 1010 - sym->sym.st_name = elf_add_string(elf, NULL, name); 1011 1011 - sym->sym.st_info = orig->sym.st_info; 1012 1012 - sym->sym.st_value = orig->sym.st_value - size; 1013 1013 - sym->sym.st_size = size; 1014 1014 - 1015 1015 - sym = __elf_create_symbol(elf, sym); 1016 1016 - if (sym) 1017 1017 - elf_add_symbol(elf, sym); 1018 1018 - 1019 1019 - return sym; 1020 1020 - } 1021 1021 - 1022 1022 - static struct reloc *elf_init_reloc(struct elf *elf, struct section *rsec, 1023 1023 - unsigned int reloc_idx, 1024 1024 - unsigned long offset, struct symbol *sym, 1025 1025 - s64 addend, unsigned int type) 841 841 + struct reloc *elf_init_reloc(struct elf *elf, struct section *rsec, 842 842 + unsigned int reloc_idx, unsigned long offset, 843 843 + struct symbol *sym, s64 addend, unsigned int type) 1026 844 { 1027 845 struct reloc *reloc, empty = { 0 }; 1028 846 ··· 1020 922 unsigned long insn_off) 1021 923 { 1022 924 struct symbol *sym = insn_sec->sym; 1023 1023 - int addend = insn_off; 925 925 + s64 addend = insn_off; 1024 926 1025 1025 - if (!(insn_sec->sh.sh_flags & SHF_EXECINSTR)) { 927 927 + if (!is_text_sec(insn_sec)) { 1026 928 ERROR("bad call to %s() for data symbol %s", __func__, sym->name); 1027 929 return NULL; 1028 930 } ··· 1037 939 sym = elf_create_section_symbol(elf, insn_sec); 1038 940 if (!sym) 1039 941 return NULL; 1040 1040 - 1041 1041 - insn_sec->sym = sym; 1042 942 } 1043 943 1044 944 return elf_init_reloc(elf, sec->rsec, reloc_idx, offset, sym, addend, ··· 1049 953 struct symbol *sym, 1050 954 s64 addend) 1051 955 { 1052 1052 - if (sym->sec && (sec->sh.sh_flags & SHF_EXECINSTR)) { 956 956 + if (is_text_sec(sec)) { 1053 957 ERROR("bad call to %s() for text symbol %s", __func__, sym->name); 1054 958 return NULL; 1055 959 } ··· 1082 986 1083 987 rsec->base->rsec = rsec; 1084 988 1085 1085 - nr_reloc = 0; 989 989 + /* nr_alloc_relocs=0: libelf owns d_buf */ 990 990 + rsec->nr_alloc_relocs = 0; 991 991 + 1086 992 rsec->relocs = calloc(sec_num_entries(rsec), sizeof(*reloc)); 1087 993 if (!rsec->relocs) { 1088 994 ERROR_GLIBC("calloc"); 1089 995 return -1; 1090 996 } 997 997 + 998 998 + nr_reloc = 0; 1091 999 for (i = 0; i < sec_num_entries(rsec); i++) { 1092 1000 reloc = &rsec->relocs[i]; 1093 1001 ··· 1144 1044 goto err; 1145 1045 } 1146 1046 1047 1047 + elf->name = strdup(name); 1048 1048 + if (!elf->name) { 1049 1049 + ERROR_GLIBC("strdup"); 1050 1050 + return NULL; 1051 1051 + } 1052 1052 + 1147 1053 if ((flags & O_ACCMODE) == O_RDONLY) 1148 1054 cmd = ELF_C_READ_MMAP; 1149 1055 else if ((flags & O_ACCMODE) == O_RDWR) ··· 1187 1081 return NULL; 1188 1082 } 1189 1083 1190 1190 - static int elf_add_string(struct elf *elf, struct section *strtab, char *str) 1084 1084 + struct elf *elf_create_file(GElf_Ehdr *ehdr, const char *name) 1191 1085 { 1192 1192 - Elf_Data *data; 1193 1193 - Elf_Scn *s; 1194 1194 - int len; 1086 1086 + struct section *null, *symtab, *strtab, *shstrtab; 1087 1087 + char *dir, *base, *tmp_name; 1088 1088 + struct symbol *sym; 1089 1089 + struct elf *elf; 1090 1090 + 1091 1091 + elf_version(EV_CURRENT); 1092 1092 + 1093 1093 + elf = calloc(1, sizeof(*elf)); 1094 1094 + if (!elf) { 1095 1095 + ERROR_GLIBC("calloc"); 1096 1096 + return NULL; 1097 1097 + } 1098 1098 + 1099 1099 + INIT_LIST_HEAD(&elf->sections); 1100 1100 + 1101 1101 + dir = strdup(name); 1102 1102 + if (!dir) { 1103 1103 + ERROR_GLIBC("strdup"); 1104 1104 + return NULL; 1105 1105 + } 1106 1106 + 1107 1107 + dir = dirname(dir); 1108 1108 + 1109 1109 + base = strdup(name); 1110 1110 + if (!base) { 1111 1111 + ERROR_GLIBC("strdup"); 1112 1112 + return NULL; 1113 1113 + } 1114 1114 + 1115 1115 + base = basename(base); 1116 1116 + 1117 1117 + tmp_name = malloc(256); 1118 1118 + if (!tmp_name) { 1119 1119 + ERROR_GLIBC("malloc"); 1120 1120 + return NULL; 1121 1121 + } 1122 1122 + 1123 1123 + snprintf(tmp_name, 256, "%s/%s.XXXXXX", dir, base); 1124 1124 + 1125 1125 + elf->fd = mkstemp(tmp_name); 1126 1126 + if (elf->fd == -1) { 1127 1127 + ERROR_GLIBC("can't create tmp file"); 1128 1128 + exit(1); 1129 1129 + } 1130 1130 + 1131 1131 + elf->tmp_name = tmp_name; 1132 1132 + 1133 1133 + elf->name = strdup(name); 1134 1134 + if (!elf->name) { 1135 1135 + ERROR_GLIBC("strdup"); 1136 1136 + return NULL; 1137 1137 + } 1138 1138 + 1139 1139 + elf->elf = elf_begin(elf->fd, ELF_C_WRITE, NULL); 1140 1140 + if (!elf->elf) { 1141 1141 + ERROR_ELF("elf_begin"); 1142 1142 + return NULL; 1143 1143 + } 1144 1144 + 1145 1145 + if (!gelf_newehdr(elf->elf, ELFCLASS64)) { 1146 1146 + ERROR_ELF("gelf_newehdr"); 1147 1147 + return NULL; 1148 1148 + } 1149 1149 + 1150 1150 + memcpy(&elf->ehdr, ehdr, sizeof(elf->ehdr)); 1151 1151 + 1152 1152 + if (!gelf_update_ehdr(elf->elf, &elf->ehdr)) { 1153 1153 + ERROR_ELF("gelf_update_ehdr"); 1154 1154 + return NULL; 1155 1155 + } 1156 1156 + 1157 1157 + INIT_LIST_HEAD(&elf->symbols); 1158 1158 + 1159 1159 + if (!elf_alloc_hash(section, 1000) || 1160 1160 + !elf_alloc_hash(section_name, 1000) || 1161 1161 + !elf_alloc_hash(symbol, 10000) || 1162 1162 + !elf_alloc_hash(symbol_name, 10000) || 1163 1163 + !elf_alloc_hash(reloc, 100000)) 1164 1164 + return NULL; 1165 1165 + 1166 1166 + null = elf_create_section(elf, NULL, 0, 0, SHT_NULL, 0, 0); 1167 1167 + shstrtab = elf_create_section(elf, NULL, 0, 0, SHT_STRTAB, 1, 0); 1168 1168 + strtab = elf_create_section(elf, NULL, 0, 0, SHT_STRTAB, 1, 0); 1169 1169 + 1170 1170 + if (!null || !shstrtab || !strtab) 1171 1171 + return NULL; 1172 1172 + 1173 1173 + null->name = ""; 1174 1174 + shstrtab->name = ".shstrtab"; 1175 1175 + strtab->name = ".strtab"; 1176 1176 + 1177 1177 + null->sh.sh_name = elf_add_string(elf, shstrtab, null->name); 1178 1178 + shstrtab->sh.sh_name = elf_add_string(elf, shstrtab, shstrtab->name); 1179 1179 + strtab->sh.sh_name = elf_add_string(elf, shstrtab, strtab->name); 1180 1180 + 1181 1181 + if (null->sh.sh_name == -1 || shstrtab->sh.sh_name == -1 || strtab->sh.sh_name == -1) 1182 1182 + return NULL; 1183 1183 + 1184 1184 + elf_hash_add(section_name, &null->name_hash, str_hash(null->name)); 1185 1185 + elf_hash_add(section_name, &strtab->name_hash, str_hash(strtab->name)); 1186 1186 + elf_hash_add(section_name, &shstrtab->name_hash, str_hash(shstrtab->name)); 1187 1187 + 1188 1188 + if (elf_add_string(elf, strtab, "") == -1) 1189 1189 + return NULL; 1190 1190 + 1191 1191 + symtab = elf_create_section(elf, ".symtab", 0x18, 0x18, SHT_SYMTAB, 0x8, 0); 1192 1192 + if (!symtab) 1193 1193 + return NULL; 1194 1194 + 1195 1195 + symtab->sh.sh_link = strtab->idx; 1196 1196 + symtab->sh.sh_info = 1; 1197 1197 + 1198 1198 + elf->ehdr.e_shstrndx = shstrtab->idx; 1199 1199 + if (!gelf_update_ehdr(elf->elf, &elf->ehdr)) { 1200 1200 + ERROR_ELF("gelf_update_ehdr"); 1201 1201 + return NULL; 1202 1202 + } 1203 1203 + 1204 1204 + sym = calloc(1, sizeof(*sym)); 1205 1205 + if (!sym) { 1206 1206 + ERROR_GLIBC("calloc"); 1207 1207 + return NULL; 1208 1208 + } 1209 1209 + 1210 1210 + sym->name = ""; 1211 1211 + sym->sec = null; 1212 1212 + elf_add_symbol(elf, sym); 1213 1213 + 1214 1214 + return elf; 1215 1215 + } 1216 1216 + 1217 1217 + unsigned int elf_add_string(struct elf *elf, struct section *strtab, const char *str) 1218 1218 + { 1219 1219 + unsigned int offset; 1195 1220 1196 1221 if (!strtab) 1197 1222 strtab = find_section_by_name(elf, ".strtab"); ··· 1331 1094 return -1; 1332 1095 } 1333 1096 1334 1334 - s = elf_getscn(elf->elf, strtab->idx); 1335 1335 - if (!s) { 1336 1336 - ERROR_ELF("elf_getscn"); 1097 1097 + if (!strtab->sh.sh_addralign) { 1098 1098 + ERROR("'%s': invalid sh_addralign", strtab->name); 1337 1099 return -1; 1338 1100 } 1339 1101 1340 1340 - data = elf_newdata(s); 1341 1341 - if (!data) { 1342 1342 - ERROR_ELF("elf_newdata"); 1102 1102 + offset = ALIGN_UP(strtab->sh.sh_size, strtab->sh.sh_addralign); 1103 1103 + 1104 1104 + if (!elf_add_data(elf, strtab, str, strlen(str) + 1)) 1343 1105 return -1; 1344 1344 - } 1345 1106 1346 1346 - data->d_buf = str; 1347 1347 - data->d_size = strlen(str) + 1; 1348 1348 - data->d_align = 1; 1349 1349 - 1350 1350 - len = strtab->sh.sh_size; 1351 1351 - strtab->sh.sh_size += data->d_size; 1352 1352 - 1353 1353 - mark_sec_changed(elf, strtab, true); 1354 1354 - 1355 1355 - return len; 1107 1107 + return offset; 1356 1108 } 1357 1109 1358 1358 - struct section *elf_create_section(struct elf *elf, const char *name, 1359 1359 - size_t entsize, unsigned int nr) 1110 1110 + void *elf_add_data(struct elf *elf, struct section *sec, const void *data, size_t size) 1360 1111 { 1361 1361 - struct section *sec, *shstrtab; 1362 1362 - size_t size = entsize * nr; 1112 1112 + unsigned long offset; 1363 1113 Elf_Scn *s; 1364 1114 1365 1365 - sec = malloc(sizeof(*sec)); 1366 1366 - if (!sec) { 1367 1367 - ERROR_GLIBC("malloc"); 1115 1115 + if (!sec->sh.sh_addralign) { 1116 1116 + ERROR("'%s': invalid sh_addralign", sec->name); 1368 1117 return NULL; 1369 1118 } 1370 1370 - memset(sec, 0, sizeof(*sec)); 1371 1119 1372 1372 - INIT_LIST_HEAD(&sec->symbol_list); 1373 1373 - 1374 1374 - s = elf_newscn(elf->elf); 1120 1120 + s = elf_getscn(elf->elf, sec->idx); 1375 1121 if (!s) { 1376 1376 - ERROR_ELF("elf_newscn"); 1122 1122 + ERROR_ELF("elf_getscn"); 1377 1123 return NULL; 1378 1124 } 1379 1379 - 1380 1380 - sec->name = strdup(name); 1381 1381 - if (!sec->name) { 1382 1382 - ERROR_GLIBC("strdup"); 1383 1383 - return NULL; 1384 1384 - } 1385 1385 - 1386 1386 - sec->idx = elf_ndxscn(s); 1387 1125 1388 1126 sec->data = elf_newdata(s); 1389 1127 if (!sec->data) { ··· 1366 1154 return NULL; 1367 1155 } 1368 1156 1157 1157 + sec->data->d_buf = calloc(1, size); 1158 1158 + if (!sec->data->d_buf) { 1159 1159 + ERROR_GLIBC("calloc"); 1160 1160 + return NULL; 1161 1161 + } 1162 1162 + 1163 1163 + if (data) 1164 1164 + memcpy(sec->data->d_buf, data, size); 1165 1165 + 1369 1166 sec->data->d_size = size; 1370 1167 sec->data->d_align = 1; 1371 1168 1169 1169 + offset = ALIGN_UP(sec->sh.sh_size, sec->sh.sh_addralign); 1170 1170 + sec->sh.sh_size = offset + size; 1171 1171 + 1172 1172 + mark_sec_changed(elf, sec, true); 1173 1173 + 1174 1174 + return sec->data->d_buf; 1175 1175 + } 1176 1176 + 1177 1177 + struct section *elf_create_section(struct elf *elf, const char *name, 1178 1178 + size_t size, size_t entsize, 1179 1179 + unsigned int type, unsigned int align, 1180 1180 + unsigned int flags) 1181 1181 + { 1182 1182 + struct section *sec, *shstrtab; 1183 1183 + Elf_Scn *s; 1184 1184 + 1185 1185 + if (name && find_section_by_name(elf, name)) { 1186 1186 + ERROR("section '%s' already exists", name); 1187 1187 + return NULL; 1188 1188 + } 1189 1189 + 1190 1190 + sec = calloc(1, sizeof(*sec)); 1191 1191 + if (!sec) { 1192 1192 + ERROR_GLIBC("calloc"); 1193 1193 + return NULL; 1194 1194 + } 1195 1195 + 1196 1196 + INIT_LIST_HEAD(&sec->symbol_list); 1197 1197 + 1198 1198 + /* don't actually create the section, just the data structures */ 1199 1199 + if (type == SHT_NULL) 1200 1200 + goto add; 1201 1201 + 1202 1202 + s = elf_newscn(elf->elf); 1203 1203 + if (!s) { 1204 1204 + ERROR_ELF("elf_newscn"); 1205 1205 + return NULL; 1206 1206 + } 1207 1207 + 1208 1208 + sec->idx = elf_ndxscn(s); 1209 1209 + 1372 1210 if (size) { 1373 1373 - sec->data->d_buf = malloc(size); 1374 1374 - if (!sec->data->d_buf) { 1375 1375 - ERROR_GLIBC("malloc"); 1211 1211 + sec->data = elf_newdata(s); 1212 1212 + if (!sec->data) { 1213 1213 + ERROR_ELF("elf_newdata"); 1376 1214 return NULL; 1377 1215 } 1378 1378 - memset(sec->data->d_buf, 0, size); 1216 1216 + 1217 1217 + sec->data->d_size = size; 1218 1218 + sec->data->d_align = 1; 1219 1219 + 1220 1220 + sec->data->d_buf = calloc(1, size); 1221 1221 + if (!sec->data->d_buf) { 1222 1222 + ERROR_GLIBC("calloc"); 1223 1223 + return NULL; 1224 1224 + } 1379 1225 } 1380 1226 1381 1227 if (!gelf_getshdr(s, &sec->sh)) { ··· 1443 1173 1444 1174 sec->sh.sh_size = size; 1445 1175 sec->sh.sh_entsize = entsize; 1446 1446 - sec->sh.sh_type = SHT_PROGBITS; 1447 1447 - sec->sh.sh_addralign = 1; 1448 1448 - sec->sh.sh_flags = SHF_ALLOC; 1176 1176 + sec->sh.sh_type = type; 1177 1177 + sec->sh.sh_addralign = align; 1178 1178 + sec->sh.sh_flags = flags; 1449 1179 1450 1450 - /* Add section name to .shstrtab (or .strtab for Clang) */ 1451 1451 - shstrtab = find_section_by_name(elf, ".shstrtab"); 1452 1452 - if (!shstrtab) 1453 1453 - shstrtab = find_section_by_name(elf, ".strtab"); 1454 1454 - if (!shstrtab) { 1455 1455 - ERROR("can't find .shstrtab or .strtab section"); 1456 1456 - return NULL; 1180 1180 + if (name) { 1181 1181 + sec->name = strdup(name); 1182 1182 + if (!sec->name) { 1183 1183 + ERROR("strdup"); 1184 1184 + return NULL; 1185 1185 + } 1186 1186 + 1187 1187 + /* Add section name to .shstrtab (or .strtab for Clang) */ 1188 1188 + shstrtab = find_section_by_name(elf, ".shstrtab"); 1189 1189 + if (!shstrtab) { 1190 1190 + shstrtab = find_section_by_name(elf, ".strtab"); 1191 1191 + if (!shstrtab) { 1192 1192 + ERROR("can't find .shstrtab or .strtab"); 1193 1193 + return NULL; 1194 1194 + } 1195 1195 + } 1196 1196 + sec->sh.sh_name = elf_add_string(elf, shstrtab, sec->name); 1197 1197 + if (sec->sh.sh_name == -1) 1198 1198 + return NULL; 1199 1199 + 1200 1200 + elf_hash_add(section_name, &sec->name_hash, str_hash(sec->name)); 1457 1201 } 1458 1458 - sec->sh.sh_name = elf_add_string(elf, shstrtab, sec->name); 1459 1459 - if (sec->sh.sh_name == -1) 1460 1460 - return NULL; 1461 1202 1203 1203 + add: 1462 1204 list_add_tail(&sec->list, &elf->sections); 1463 1205 elf_hash_add(section, &sec->hash, sec->idx); 1464 1464 - elf_hash_add(section_name, &sec->name_hash, str_hash(sec->name)); 1465 1206 1466 1207 mark_sec_changed(elf, sec, true); 1467 1208 1468 1209 return sec; 1469 1210 } 1470 1211 1471 1471 - static struct section *elf_create_rela_section(struct elf *elf, 1472 1472 - struct section *sec, 1473 1473 - unsigned int reloc_nr) 1212 1212 + static int elf_alloc_reloc(struct elf *elf, struct section *rsec) 1213 1213 + { 1214 1214 + struct reloc *old_relocs, *old_relocs_end, *new_relocs; 1215 1215 + unsigned int nr_relocs_old = sec_num_entries(rsec); 1216 1216 + unsigned int nr_relocs_new = nr_relocs_old + 1; 1217 1217 + unsigned long nr_alloc; 1218 1218 + struct symbol *sym; 1219 1219 + 1220 1220 + if (!rsec->data) { 1221 1221 + rsec->data = elf_newdata(elf_getscn(elf->elf, rsec->idx)); 1222 1222 + if (!rsec->data) { 1223 1223 + ERROR_ELF("elf_newdata"); 1224 1224 + return -1; 1225 1225 + } 1226 1226 + 1227 1227 + rsec->data->d_align = 1; 1228 1228 + rsec->data->d_type = ELF_T_RELA; 1229 1229 + rsec->data->d_buf = NULL; 1230 1230 + } 1231 1231 + 1232 1232 + rsec->data->d_size = nr_relocs_new * elf_rela_size(elf); 1233 1233 + rsec->sh.sh_size = rsec->data->d_size; 1234 1234 + 1235 1235 + nr_alloc = MAX(64, ALIGN_UP_POW2(nr_relocs_new)); 1236 1236 + if (nr_alloc <= rsec->nr_alloc_relocs) 1237 1237 + return 0; 1238 1238 + 1239 1239 + if (rsec->data->d_buf && !rsec->nr_alloc_relocs) { 1240 1240 + void *orig_buf = rsec->data->d_buf; 1241 1241 + 1242 1242 + /* 1243 1243 + * The original d_buf is owned by libelf so it can't be 1244 1244 + * realloced. 1245 1245 + */ 1246 1246 + rsec->data->d_buf = malloc(nr_alloc * elf_rela_size(elf)); 1247 1247 + if (!rsec->data->d_buf) { 1248 1248 + ERROR_GLIBC("malloc"); 1249 1249 + return -1; 1250 1250 + } 1251 1251 + memcpy(rsec->data->d_buf, orig_buf, 1252 1252 + nr_relocs_old * elf_rela_size(elf)); 1253 1253 + } else { 1254 1254 + rsec->data->d_buf = realloc(rsec->data->d_buf, 1255 1255 + nr_alloc * elf_rela_size(elf)); 1256 1256 + if (!rsec->data->d_buf) { 1257 1257 + ERROR_GLIBC("realloc"); 1258 1258 + return -1; 1259 1259 + } 1260 1260 + } 1261 1261 + 1262 1262 + rsec->nr_alloc_relocs = nr_alloc; 1263 1263 + 1264 1264 + old_relocs = rsec->relocs; 1265 1265 + new_relocs = calloc(nr_alloc, sizeof(struct reloc)); 1266 1266 + if (!new_relocs) { 1267 1267 + ERROR_GLIBC("calloc"); 1268 1268 + return -1; 1269 1269 + } 1270 1270 + 1271 1271 + if (!old_relocs) 1272 1272 + goto done; 1273 1273 + 1274 1274 + /* 1275 1275 + * The struct reloc's address has changed. Update all the symbols and 1276 1276 + * relocs which reference it. 1277 1277 + */ 1278 1278 + 1279 1279 + old_relocs_end = &old_relocs[nr_relocs_old]; 1280 1280 + for_each_sym(elf, sym) { 1281 1281 + struct reloc *reloc; 1282 1282 + 1283 1283 + reloc = sym->relocs; 1284 1284 + if (!reloc) 1285 1285 + continue; 1286 1286 + 1287 1287 + if (reloc >= old_relocs && reloc < old_relocs_end) 1288 1288 + sym->relocs = &new_relocs[reloc - old_relocs]; 1289 1289 + 1290 1290 + while (1) { 1291 1291 + struct reloc *next_reloc = sym_next_reloc(reloc); 1292 1292 + 1293 1293 + if (!next_reloc) 1294 1294 + break; 1295 1295 + 1296 1296 + if (next_reloc >= old_relocs && next_reloc < old_relocs_end) 1297 1297 + set_sym_next_reloc(reloc, &new_relocs[next_reloc - old_relocs]); 1298 1298 + 1299 1299 + reloc = next_reloc; 1300 1300 + } 1301 1301 + } 1302 1302 + 1303 1303 + memcpy(new_relocs, old_relocs, nr_relocs_old * sizeof(struct reloc)); 1304 1304 + 1305 1305 + for (int i = 0; i < nr_relocs_old; i++) { 1306 1306 + struct reloc *old = &old_relocs[i]; 1307 1307 + struct reloc *new = &new_relocs[i]; 1308 1308 + u32 key = reloc_hash(old); 1309 1309 + 1310 1310 + elf_hash_del(reloc, &old->hash, key); 1311 1311 + elf_hash_add(reloc, &new->hash, key); 1312 1312 + } 1313 1313 + 1314 1314 + free(old_relocs); 1315 1315 + done: 1316 1316 + rsec->relocs = new_relocs; 1317 1317 + return 0; 1318 1318 + } 1319 1319 + 1320 1320 + struct section *elf_create_rela_section(struct elf *elf, struct section *sec, 1321 1321 + unsigned int nr_relocs) 1474 1322 { 1475 1323 struct section *rsec; 1476 1324 char *rsec_name; ··· 1601 1213 strcpy(rsec_name, ".rela"); 1602 1214 strcat(rsec_name, sec->name); 1603 1215 1604 1604 - rsec = elf_create_section(elf, rsec_name, elf_rela_size(elf), reloc_nr); 1216 1216 + rsec = elf_create_section(elf, rsec_name, nr_relocs * elf_rela_size(elf), 1217 1217 + elf_rela_size(elf), SHT_RELA, elf_addr_size(elf), 1218 1218 + SHF_INFO_LINK); 1605 1219 free(rsec_name); 1606 1220 if (!rsec) 1607 1221 return NULL; 1608 1222 1609 1609 - rsec->data->d_type = ELF_T_RELA; 1610 1610 - rsec->sh.sh_type = SHT_RELA; 1611 1611 - rsec->sh.sh_addralign = elf_addr_size(elf); 1223 1223 + if (nr_relocs) { 1224 1224 + rsec->data->d_type = ELF_T_RELA; 1225 1225 + 1226 1226 + rsec->nr_alloc_relocs = nr_relocs; 1227 1227 + rsec->relocs = calloc(nr_relocs, sizeof(struct reloc)); 1228 1228 + if (!rsec->relocs) { 1229 1229 + ERROR_GLIBC("calloc"); 1230 1230 + return NULL; 1231 1231 + } 1232 1232 + } 1233 1233 + 1612 1234 rsec->sh.sh_link = find_section_by_name(elf, ".symtab")->idx; 1613 1235 rsec->sh.sh_info = sec->idx; 1614 1614 - rsec->sh.sh_flags = SHF_INFO_LINK; 1615 1615 - 1616 1616 - rsec->relocs = calloc(sec_num_entries(rsec), sizeof(struct reloc)); 1617 1617 - if (!rsec->relocs) { 1618 1618 - ERROR_GLIBC("calloc"); 1619 1619 - return NULL; 1620 1620 - } 1621 1236 1622 1237 sec->rsec = rsec; 1623 1238 rsec->base = sec; ··· 1628 1237 return rsec; 1629 1238 } 1630 1239 1240 1240 + struct reloc *elf_create_reloc(struct elf *elf, struct section *sec, 1241 1241 + unsigned long offset, 1242 1242 + struct symbol *sym, s64 addend, 1243 1243 + unsigned int type) 1244 1244 + { 1245 1245 + struct section *rsec = sec->rsec; 1246 1246 + 1247 1247 + if (!rsec) { 1248 1248 + rsec = elf_create_rela_section(elf, sec, 0); 1249 1249 + if (!rsec) 1250 1250 + return NULL; 1251 1251 + } 1252 1252 + 1253 1253 + if (find_reloc_by_dest(elf, sec, offset)) { 1254 1254 + ERROR_FUNC(sec, offset, "duplicate reloc"); 1255 1255 + return NULL; 1256 1256 + } 1257 1257 + 1258 1258 + if (elf_alloc_reloc(elf, rsec)) 1259 1259 + return NULL; 1260 1260 + 1261 1261 + mark_sec_changed(elf, rsec, true); 1262 1262 + 1263 1263 + return elf_init_reloc(elf, rsec, sec_num_entries(rsec) - 1, offset, sym, 1264 1264 + addend, type); 1265 1265 + } 1266 1266 + 1631 1267 struct section *elf_create_section_pair(struct elf *elf, const char *name, 1632 1268 size_t entsize, unsigned int nr, 1633 1633 - unsigned int reloc_nr) 1269 1269 + unsigned int nr_relocs) 1634 1270 { 1635 1271 struct section *sec; 1636 1272 1637 1637 - sec = elf_create_section(elf, name, entsize, nr); 1273 1273 + sec = elf_create_section(elf, name, nr * entsize, entsize, 1274 1274 + SHT_PROGBITS, 1, SHF_ALLOC); 1638 1275 if (!sec) 1639 1276 return NULL; 1640 1277 1641 1641 - if (!elf_create_rela_section(elf, sec, reloc_nr)) 1278 1278 + if (!elf_create_rela_section(elf, sec, nr_relocs)) 1642 1279 return NULL; 1643 1280 1644 1281 return sec; ··· 1701 1282 */ 1702 1283 static int elf_truncate_section(struct elf *elf, struct section *sec) 1703 1284 { 1704 1704 - u64 size = sec->sh.sh_size; 1285 1285 + u64 size = sec_size(sec); 1705 1286 bool truncated = false; 1706 1287 Elf_Data *data = NULL; 1707 1288 Elf_Scn *s; ··· 1715 1296 for (;;) { 1716 1297 /* get next data descriptor for the relevant section */ 1717 1298 data = elf_getdata(s, data); 1718 1718 - 1719 1299 if (!data) { 1720 1300 if (size) { 1721 1301 ERROR("end of section data but non-zero size left\n"); ··· 1750 1332 1751 1333 /* Update changed relocation sections and section headers: */ 1752 1334 list_for_each_entry(sec, &elf->sections, list) { 1753 1753 - if (sec->truncate) 1754 1754 - elf_truncate_section(elf, sec); 1335 1335 + if (sec->truncate && elf_truncate_section(elf, sec)) 1336 1336 + return -1; 1755 1337 1756 1338 if (sec_changed(sec)) { 1757 1339 s = elf_getscn(elf->elf, sec->idx); ··· 1784 1366 return 0; 1785 1367 } 1786 1368 1787 1787 - void elf_close(struct elf *elf) 1369 1369 + int elf_close(struct elf *elf) 1788 1370 { 1789 1371 if (elf->elf) 1790 1372 elf_end(elf->elf); ··· 1792 1374 if (elf->fd > 0) 1793 1375 close(elf->fd); 1794 1376 1377 1377 + if (elf->tmp_name && rename(elf->tmp_name, elf->name)) 1378 1378 + return -1; 1379 1379 + 1795 1380 /* 1796 1381 * NOTE: All remaining allocations are leaked on purpose. Objtool is 1797 1382 * about to exit anyway. 1798 1383 */ 1384 1384 + return 0; 1799 1385 }

+14 -2

tools/objtool/include/objtool/arch.h

reviewed

··· 71 71 72 72 struct instruction; 73 73 74 74 - int arch_ftrace_match(char *name); 74 74 + int arch_ftrace_match(const char *name); 75 75 76 76 void arch_initial_func_cfi_state(struct cfi_init_state *state); 77 77 ··· 83 83 84 84 unsigned long arch_jump_destination(struct instruction *insn); 85 85 86 86 - unsigned long arch_dest_reloc_offset(int addend); 86 86 + s64 arch_insn_adjusted_addend(struct instruction *insn, struct reloc *reloc); 87 87 + u64 arch_adjusted_addend(struct reloc *reloc); 87 88 88 89 const char *arch_nop_insn(int len); 89 90 const char *arch_ret_insn(int len); ··· 102 101 103 102 unsigned int arch_reloc_size(struct reloc *reloc); 104 103 unsigned long arch_jump_table_sym_offset(struct reloc *reloc, struct reloc *table); 104 104 + 105 105 + extern const char *arch_reg_name[CFI_NUM_REGS]; 106 106 + 107 107 + #ifdef DISAS 108 108 + 109 109 + #include <bfd.h> 110 110 + #include <dis-asm.h> 111 111 + 112 112 + int arch_disas_info_init(struct disassemble_info *dinfo); 113 113 + 114 114 + #endif /* DISAS */ 105 115 106 116 #endif /* _ARCH_H */

+11 -3

tools/objtool/include/objtool/builtin.h

reviewed

··· 9 9 10 10 struct opts { 11 11 /* actions: */ 12 12 + bool cfi; 13 13 + bool checksum; 12 14 bool dump_orc; 13 15 bool hack_jump_label; 14 16 bool hack_noinstr; 15 17 bool hack_skylake; 16 18 bool ibt; 17 19 bool mcount; 20 20 + bool noabs; 18 21 bool noinstr; 19 22 bool orc; 20 23 bool retpoline; ··· 28 25 bool static_call; 29 26 bool uaccess; 30 27 int prefix; 31 31 - bool cfi; 32 32 - bool noabs; 28 28 + const char *disas; 33 29 34 30 /* options: */ 35 31 bool backtrace; 32 32 + bool backup; 33 33 + const char *debug_checksum; 36 34 bool dryrun; 37 35 bool link; 38 36 bool mnop; ··· 42 38 const char *output; 43 39 bool sec_address; 44 40 bool stats; 41 41 + const char *trace; 45 42 bool verbose; 46 43 bool werror; 44 44 + bool wide; 47 45 }; 48 46 49 47 extern struct opts opts; ··· 54 48 55 49 int objtool_run(int argc, const char **argv); 56 50 57 57 - void print_args(void); 51 51 + int make_backup(void); 52 52 + 53 53 + int cmd_klp(int argc, const char **argv); 58 54 59 55 #endif /* _BUILTIN_H */

+37 -2

tools/objtool/include/objtool/check.h

reviewed

··· 36 36 struct cfi_state **cfi; 37 37 38 38 bool ignore; 39 39 + unsigned int feature; 40 40 + }; 41 41 + 42 42 + enum alternative_type { 43 43 + ALT_TYPE_INSTRUCTIONS, 44 44 + ALT_TYPE_JUMP_TABLE, 45 45 + ALT_TYPE_EX_TABLE, 46 46 + }; 47 47 + 48 48 + struct alternative { 49 49 + struct alternative *next; 50 50 + struct instruction *insn; 51 51 + enum alternative_type type; 39 52 }; 40 53 41 54 #define INSN_CHUNK_BITS 8 ··· 77 64 noendbr : 1, 78 65 unret : 1, 79 66 visited : 4, 80 80 - no_reloc : 1; 81 81 - /* 10 bit hole */ 67 67 + no_reloc : 1, 68 68 + hole : 1, 69 69 + fake : 1, 70 70 + trace : 1; 71 71 + /* 9 bit hole */ 82 72 83 73 struct alt_group *alt_group; 84 74 struct instruction *jump_dest; ··· 131 115 return is_static_jump(insn) || is_dynamic_jump(insn); 132 116 } 133 117 118 118 + static inline struct symbol *insn_call_dest(struct instruction *insn) 119 119 + { 120 120 + if (insn->type == INSN_JUMP_DYNAMIC || 121 121 + insn->type == INSN_CALL_DYNAMIC) 122 122 + return NULL; 123 123 + 124 124 + return insn->_call_dest; 125 125 + } 126 126 + 134 127 struct instruction *find_insn(struct objtool_file *file, 135 128 struct section *sec, unsigned long offset); 136 129 ··· 149 124 for (insn = find_insn(file, _sec, 0); \ 150 125 insn && insn->sec == _sec; \ 151 126 insn = next_insn_same_sec(file, insn)) 127 127 + 128 128 + #define sym_for_each_insn(file, sym, insn) \ 129 129 + for (insn = find_insn(file, sym->sec, sym->offset); \ 130 130 + insn && insn->offset < sym->offset + sym->len; \ 131 131 + insn = next_insn_same_sec(file, insn)) 132 132 + 133 133 + const char *objtool_disas_insn(struct instruction *insn); 134 134 + 135 135 + extern size_t sym_name_max_len; 136 136 + extern struct disas_context *objtool_disas_ctx; 152 137 153 138 #endif /* _CHECK_H */

+43

tools/objtool/include/objtool/checksum.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 + #ifndef _OBJTOOL_CHECKSUM_H 3 3 + #define _OBJTOOL_CHECKSUM_H 4 4 + 5 5 + #include <objtool/elf.h> 6 6 + 7 7 + #ifdef BUILD_KLP 8 8 + 9 9 + static inline void checksum_init(struct symbol *func) 10 10 + { 11 11 + if (func && !func->csum.state) { 12 12 + func->csum.state = XXH3_createState(); 13 13 + XXH3_64bits_reset(func->csum.state); 14 14 + } 15 15 + } 16 16 + 17 17 + static inline void checksum_update(struct symbol *func, 18 18 + struct instruction *insn, 19 19 + const void *data, size_t size) 20 20 + { 21 21 + XXH3_64bits_update(func->csum.state, data, size); 22 22 + dbg_checksum(func, insn, XXH3_64bits_digest(func->csum.state)); 23 23 + } 24 24 + 25 25 + static inline void checksum_finish(struct symbol *func) 26 26 + { 27 27 + if (func && func->csum.state) { 28 28 + func->csum.checksum = XXH3_64bits_digest(func->csum.state); 29 29 + func->csum.state = NULL; 30 30 + } 31 31 + } 32 32 + 33 33 + #else /* !BUILD_KLP */ 34 34 + 35 35 + static inline void checksum_init(struct symbol *func) {} 36 36 + static inline void checksum_update(struct symbol *func, 37 37 + struct instruction *insn, 38 38 + const void *data, size_t size) {} 39 39 + static inline void checksum_finish(struct symbol *func) {} 40 40 + 41 41 + #endif /* !BUILD_KLP */ 42 42 + 43 43 + #endif /* _OBJTOOL_CHECKSUM_H */

+25

tools/objtool/include/objtool/checksum_types.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 + #ifndef _OBJTOOL_CHECKSUM_TYPES_H 3 3 + #define _OBJTOOL_CHECKSUM_TYPES_H 4 4 + 5 5 + struct sym_checksum { 6 6 + u64 addr; 7 7 + u64 checksum; 8 8 + }; 9 9 + 10 10 + #ifdef BUILD_KLP 11 11 + 12 12 + #include <xxhash.h> 13 13 + 14 14 + struct checksum { 15 15 + XXH3_state_t *state; 16 16 + XXH64_hash_t checksum; 17 17 + }; 18 18 + 19 19 + #else /* !BUILD_KLP */ 20 20 + 21 21 + struct checksum {}; 22 22 + 23 23 + #endif /* !BUILD_KLP */ 24 24 + 25 25 + #endif /* _OBJTOOL_CHECKSUM_TYPES_H */

+81

tools/objtool/include/objtool/disas.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 + /* 3 3 + * Copyright (c) 2025, Oracle and/or its affiliates. 4 4 + */ 5 5 + 6 6 + #ifndef _DISAS_H 7 7 + #define _DISAS_H 8 8 + 9 9 + struct alternative; 10 10 + struct disas_context; 11 11 + struct disassemble_info; 12 12 + 13 13 + #ifdef DISAS 14 14 + 15 15 + struct disas_context *disas_context_create(struct objtool_file *file); 16 16 + void disas_context_destroy(struct disas_context *dctx); 17 17 + void disas_warned_funcs(struct disas_context *dctx); 18 18 + void disas_funcs(struct disas_context *dctx); 19 19 + int disas_info_init(struct disassemble_info *dinfo, 20 20 + int arch, int mach32, int mach64, 21 21 + const char *options); 22 22 + size_t disas_insn(struct disas_context *dctx, struct instruction *insn); 23 23 + char *disas_result(struct disas_context *dctx); 24 24 + void disas_print_info(FILE *stream, struct instruction *insn, int depth, 25 25 + const char *format, ...); 26 26 + void disas_print_insn(FILE *stream, struct disas_context *dctx, 27 27 + struct instruction *insn, int depth, 28 28 + const char *format, ...); 29 29 + char *disas_alt_name(struct alternative *alt); 30 30 + const char *disas_alt_type_name(struct instruction *insn); 31 31 + 32 32 + #else /* DISAS */ 33 33 + 34 34 + #include <objtool/warn.h> 35 35 + 36 36 + static inline struct disas_context *disas_context_create(struct objtool_file *file) 37 37 + { 38 38 + WARN("Rebuild with libopcodes for disassembly support"); 39 39 + return NULL; 40 40 + } 41 41 + 42 42 + static inline void disas_context_destroy(struct disas_context *dctx) {} 43 43 + static inline void disas_warned_funcs(struct disas_context *dctx) {} 44 44 + static inline void disas_funcs(struct disas_context *dctx) {} 45 45 + 46 46 + static inline int disas_info_init(struct disassemble_info *dinfo, 47 47 + int arch, int mach32, int mach64, 48 48 + const char *options) 49 49 + { 50 50 + return -1; 51 51 + } 52 52 + 53 53 + static inline size_t disas_insn(struct disas_context *dctx, 54 54 + struct instruction *insn) 55 55 + { 56 56 + return -1; 57 57 + } 58 58 + 59 59 + static inline char *disas_result(struct disas_context *dctx) 60 60 + { 61 61 + return NULL; 62 62 + } 63 63 + 64 64 + static inline void disas_print_info(FILE *stream, struct instruction *insn, 65 65 + int depth, const char *format, ...) {} 66 66 + static inline void disas_print_insn(FILE *stream, struct disas_context *dctx, 67 67 + struct instruction *insn, int depth, 68 68 + const char *format, ...) {} 69 69 + static inline char *disas_alt_name(struct alternative *alt) 70 70 + { 71 71 + return NULL; 72 72 + } 73 73 + 74 74 + static inline const char *disas_alt_type_name(struct instruction *insn) 75 75 + { 76 76 + return NULL; 77 77 + } 78 78 + 79 79 + #endif /* DISAS */ 80 80 + 81 81 + #endif /* _DISAS_H */

+174 -24

tools/objtool/include/objtool/elf.h

reviewed

··· 8 8 9 9 #include <stdio.h> 10 10 #include <gelf.h> 11 11 + #include <linux/string.h> 11 12 #include <linux/list.h> 12 13 #include <linux/hashtable.h> 13 14 #include <linux/rbtree.h> 14 15 #include <linux/jhash.h> 16 16 + 17 17 + #include <objtool/endianness.h> 18 18 + #include <objtool/checksum_types.h> 15 19 #include <arch/elf.h> 20 20 + 21 21 + #define SEC_NAME_LEN 1024 22 22 + #define SYM_NAME_LEN 512 23 23 + 24 24 + #define bswap_if_needed(elf, val) __bswap_if_needed(&elf->ehdr, val) 16 25 17 26 #ifdef LIBELF_USE_DEPRECATED 18 27 # define elf_getshdrnum elf_getshnum ··· 49 40 struct section *base, *rsec; 50 41 struct symbol *sym; 51 42 Elf_Data *data; 52 52 - char *name; 43 43 + const char *name; 53 44 int idx; 54 45 bool _changed, text, rodata, noinstr, init, truncate; 55 46 struct reloc *relocs; 47 47 + unsigned long nr_alloc_relocs; 48 48 + struct section *twin; 56 49 }; 57 50 58 51 struct symbol { 59 52 struct list_head list; 53 53 + struct list_head global_list; 60 54 struct rb_node node; 61 55 struct elf_hash_node hash; 62 56 struct elf_hash_node name_hash; 63 57 GElf_Sym sym; 64 58 struct section *sec; 65 65 - char *name; 59 59 + const char *name, *demangled_name; 66 60 unsigned int idx, len; 67 61 unsigned long offset; 68 62 unsigned long __subtree_last; 69 69 - struct symbol *pfunc, *cfunc, *alias; 63 63 + struct symbol *pfunc, *cfunc, *alias, *file; 70 64 unsigned char bind, type; 71 65 u8 uaccess_safe : 1; 72 66 u8 static_call_tramp : 1; ··· 83 71 u8 frame_pointer : 1; 84 72 u8 ignore : 1; 85 73 u8 nocfi : 1; 74 74 + u8 cold : 1; 75 75 + u8 prefix : 1; 76 76 + u8 debug_checksum : 1; 77 77 + u8 changed : 1; 78 78 + u8 included : 1; 79 79 + u8 klp : 1; 86 80 struct list_head pv_target; 87 81 struct reloc *relocs; 88 82 struct section *group_sec; 83 83 + struct checksum csum; 84 84 + struct symbol *twin, *clone; 89 85 }; 90 86 91 87 struct reloc { ··· 108 88 GElf_Ehdr ehdr; 109 89 int fd; 110 90 bool changed; 111 111 - char *name; 91 91 + const char *name, *tmp_name; 112 92 unsigned int num_files; 113 93 struct list_head sections; 94 94 + struct list_head symbols; 114 95 unsigned long num_relocs; 115 96 116 97 int symbol_bits; ··· 131 110 }; 132 111 133 112 struct elf *elf_open_read(const char *name, int flags); 113 113 + struct elf *elf_create_file(GElf_Ehdr *ehdr, const char *name); 134 114 135 115 struct section *elf_create_section(struct elf *elf, const char *name, 136 136 - size_t entsize, unsigned int nr); 116 116 + size_t size, size_t entsize, 117 117 + unsigned int type, unsigned int align, 118 118 + unsigned int flags); 137 119 struct section *elf_create_section_pair(struct elf *elf, const char *name, 138 120 size_t entsize, unsigned int nr, 139 121 unsigned int reloc_nr); 140 122 141 141 - struct symbol *elf_create_prefix_symbol(struct elf *elf, struct symbol *orig, long size); 123 123 + struct section *elf_create_rela_section(struct elf *elf, struct section *sec, 124 124 + unsigned int reloc_nr); 125 125 + 126 126 + struct symbol *elf_create_symbol(struct elf *elf, const char *name, 127 127 + struct section *sec, unsigned int bind, 128 128 + unsigned int type, unsigned long offset, 129 129 + size_t size); 130 130 + struct symbol *elf_create_section_symbol(struct elf *elf, struct section *sec); 131 131 + 132 132 + void *elf_add_data(struct elf *elf, struct section *sec, const void *data, 133 133 + size_t size); 134 134 + 135 135 + unsigned int elf_add_string(struct elf *elf, struct section *strtab, const char *str); 136 136 + 137 137 + struct reloc *elf_create_reloc(struct elf *elf, struct section *sec, 138 138 + unsigned long offset, struct symbol *sym, 139 139 + s64 addend, unsigned int type); 140 140 + 141 141 + struct reloc *elf_init_reloc(struct elf *elf, struct section *rsec, 142 142 + unsigned int reloc_idx, unsigned long offset, 143 143 + struct symbol *sym, s64 addend, unsigned int type); 142 144 143 145 struct reloc *elf_init_reloc_text_sym(struct elf *elf, struct section *sec, 144 146 unsigned long offset, ··· 175 131 struct symbol *sym, 176 132 s64 addend); 177 133 178 178 - int elf_write_insn(struct elf *elf, struct section *sec, 179 179 - unsigned long offset, unsigned int len, 180 180 - const char *insn); 134 134 + int elf_write_insn(struct elf *elf, struct section *sec, unsigned long offset, 135 135 + unsigned int len, const char *insn); 136 136 + 181 137 int elf_write(struct elf *elf); 182 182 - void elf_close(struct elf *elf); 138 138 + int elf_close(struct elf *elf); 183 139 184 140 struct section *find_section_by_name(const struct elf *elf, const char *name); 185 141 struct symbol *find_func_by_offset(struct section *sec, unsigned long offset); 186 142 struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset); 187 143 struct symbol *find_symbol_by_name(const struct elf *elf, const char *name); 144 144 + struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *name); 188 145 struct symbol *find_symbol_containing(const struct section *sec, unsigned long offset); 189 146 int find_symbol_hole_containing(const struct section *sec, unsigned long offset); 190 147 struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, unsigned long offset); ··· 223 178 return elf_addr_size(elf) == 4 ? R_TEXT32 : R_TEXT64; 224 179 } 225 180 181 181 + static inline bool is_undef_sym(struct symbol *sym) 182 182 + { 183 183 + return !sym->sec->idx; 184 184 + } 185 185 + 186 186 + static inline bool is_null_sym(struct symbol *sym) 187 187 + { 188 188 + return !sym->idx; 189 189 + } 190 190 + 191 191 + static inline bool is_sec_sym(struct symbol *sym) 192 192 + { 193 193 + return sym->type == STT_SECTION; 194 194 + } 195 195 + 196 196 + static inline bool is_object_sym(struct symbol *sym) 197 197 + { 198 198 + return sym->type == STT_OBJECT; 199 199 + } 200 200 + 201 201 + static inline bool is_func_sym(struct symbol *sym) 202 202 + { 203 203 + return sym->type == STT_FUNC; 204 204 + } 205 205 + 206 206 + static inline bool is_file_sym(struct symbol *sym) 207 207 + { 208 208 + return sym->type == STT_FILE; 209 209 + } 210 210 + 211 211 + static inline bool is_notype_sym(struct symbol *sym) 212 212 + { 213 213 + return sym->type == STT_NOTYPE; 214 214 + } 215 215 + 216 216 + static inline bool is_global_sym(struct symbol *sym) 217 217 + { 218 218 + return sym->bind == STB_GLOBAL; 219 219 + } 220 220 + 221 221 + static inline bool is_weak_sym(struct symbol *sym) 222 222 + { 223 223 + return sym->bind == STB_WEAK; 224 224 + } 225 225 + 226 226 + static inline bool is_local_sym(struct symbol *sym) 227 227 + { 228 228 + return sym->bind == STB_LOCAL; 229 229 + } 230 230 + 231 231 + static inline bool is_prefix_func(struct symbol *sym) 232 232 + { 233 233 + return sym->prefix; 234 234 + } 235 235 + 226 236 static inline bool is_reloc_sec(struct section *sec) 227 237 { 228 238 return sec->sh.sh_type == SHT_RELA || sec->sh.sh_type == SHT_REL; 239 239 + } 240 240 + 241 241 + static inline bool is_string_sec(struct section *sec) 242 242 + { 243 243 + return sec->sh.sh_flags & SHF_STRINGS; 244 244 + } 245 245 + 246 246 + static inline bool is_text_sec(struct section *sec) 247 247 + { 248 248 + return sec->sh.sh_flags & SHF_EXECINSTR; 229 249 } 230 250 231 251 static inline bool sec_changed(struct section *sec) ··· 331 221 * Elf64_Rela: 24 bytes 332 222 */ 333 223 return reloc->sec->sh.sh_entsize < 16; 224 224 + } 225 225 + 226 226 + static inline unsigned long sec_size(struct section *sec) 227 227 + { 228 228 + return sec->sh.sh_size; 334 229 } 335 230 336 231 #define __get_reloc_field(reloc, field) \ ··· 415 300 mark_sec_changed(elf, reloc->sec, true); 416 301 } 417 302 303 303 + static inline unsigned int annotype(struct elf *elf, struct section *sec, 304 304 + struct reloc *reloc) 305 305 + { 306 306 + unsigned int type; 307 307 + 308 308 + type = *(u32 *)(sec->data->d_buf + (reloc_idx(reloc) * 8) + 4); 309 309 + return bswap_if_needed(elf, type); 310 310 + } 311 311 + 418 312 #define RELOC_JUMP_TABLE_BIT 1UL 419 313 420 314 /* Does reloc mark the beginning of a jump table? */ ··· 449 325 reloc->_sym_next_reloc = (unsigned long)next | bit; 450 326 } 451 327 452 452 - #define for_each_sec(file, sec) \ 453 453 - list_for_each_entry(sec, &file->elf->sections, list) 328 328 + #define for_each_sec(elf, sec) \ 329 329 + list_for_each_entry(sec, &elf->sections, list) 454 330 455 331 #define sec_for_each_sym(sec, sym) \ 456 332 list_for_each_entry(sym, &sec->symbol_list, list) 457 333 458 458 - #define for_each_sym(file, sym) \ 459 459 - for (struct section *__sec, *__fake = (struct section *)1; \ 460 460 - __fake; __fake = NULL) \ 461 461 - for_each_sec(file, __sec) \ 462 462 - sec_for_each_sym(__sec, sym) 334 334 + #define sec_prev_sym(sym) \ 335 335 + sym->sec && sym->list.prev != &sym->sec->symbol_list ? \ 336 336 + list_prev_entry(sym, list) : NULL 337 337 + 338 338 + #define for_each_sym(elf, sym) \ 339 339 + list_for_each_entry(sym, &elf->symbols, global_list) 340 340 + 341 341 + #define for_each_sym_continue(elf, sym) \ 342 342 + list_for_each_entry_continue(sym, &elf->symbols, global_list) 343 343 + 344 344 + #define rsec_next_reloc(rsec, reloc) \ 345 345 + reloc_idx(reloc) < sec_num_entries(rsec) - 1 ? reloc + 1 : NULL 463 346 464 347 #define for_each_reloc(rsec, reloc) \ 465 465 - for (int __i = 0, __fake = 1; __fake; __fake = 0) \ 466 466 - for (reloc = rsec->relocs; \ 467 467 - __i < sec_num_entries(rsec); \ 468 468 - __i++, reloc++) 348 348 + for (reloc = rsec->relocs; reloc; reloc = rsec_next_reloc(rsec, reloc)) 469 349 470 350 #define for_each_reloc_from(rsec, reloc) \ 471 471 - for (int __i = reloc_idx(reloc); \ 472 472 - __i < sec_num_entries(rsec); \ 473 473 - __i++, reloc++) 351 351 + for (; reloc; reloc = rsec_next_reloc(rsec, reloc)) 352 352 + 353 353 + #define for_each_reloc_continue(rsec, reloc) \ 354 354 + for (reloc = rsec_next_reloc(rsec, reloc); reloc; \ 355 355 + reloc = rsec_next_reloc(rsec, reloc)) 356 356 + 357 357 + #define sym_for_each_reloc(elf, sym, reloc) \ 358 358 + for (reloc = find_reloc_by_dest_range(elf, sym->sec, \ 359 359 + sym->offset, sym->len); \ 360 360 + reloc && reloc_offset(reloc) < sym->offset + sym->len; \ 361 361 + reloc = rsec_next_reloc(sym->sec->rsec, reloc)) 362 362 + 363 363 + static inline struct symbol *get_func_prefix(struct symbol *func) 364 364 + { 365 365 + struct symbol *prev; 366 366 + 367 367 + if (!is_func_sym(func)) 368 368 + return NULL; 369 369 + 370 370 + prev = sec_prev_sym(func); 371 371 + if (prev && is_prefix_func(prev)) 372 372 + return prev; 373 373 + 374 374 + return NULL; 375 375 + } 474 376 475 377 #define OFFSET_STRIDE_BITS 4 476 378 #define OFFSET_STRIDE (1UL << OFFSET_STRIDE_BITS)

+4 -5

tools/objtool/include/objtool/endianness.h

reviewed

··· 4 4 5 5 #include <linux/kernel.h> 6 6 #include <endian.h> 7 7 - #include <objtool/elf.h> 8 7 9 8 /* 10 9 * Does a byte swap if target file endianness doesn't match the host, i.e. cross ··· 11 12 * To be used for multi-byte values conversion, which are read from / about 12 13 * to be written to a target native endianness ELF file. 13 14 */ 14 14 - static inline bool need_bswap(struct elf *elf) 15 15 + static inline bool need_bswap(GElf_Ehdr *ehdr) 15 16 { 16 17 return (__BYTE_ORDER == __LITTLE_ENDIAN) ^ 17 17 - (elf->ehdr.e_ident[EI_DATA] == ELFDATA2LSB); 18 18 + (ehdr->e_ident[EI_DATA] == ELFDATA2LSB); 18 19 } 19 20 20 20 - #define bswap_if_needed(elf, val) \ 21 21 + #define __bswap_if_needed(ehdr, val) \ 21 22 ({ \ 22 23 __typeof__(val) __ret; \ 23 23 - bool __need_bswap = need_bswap(elf); \ 24 24 + bool __need_bswap = need_bswap(ehdr); \ 24 25 switch (sizeof(val)) { \ 25 26 case 8: \ 26 27 __ret = __need_bswap ? bswap_64(val) : (val); break; \

+35

tools/objtool/include/objtool/klp.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 + #ifndef _OBJTOOL_KLP_H 3 3 + #define _OBJTOOL_KLP_H 4 4 + 5 5 + #define SHF_RELA_LIVEPATCH 0x00100000 6 6 + #define SHN_LIVEPATCH 0xff20 7 7 + 8 8 + /* 9 9 + * __klp_objects and __klp_funcs are created by klp diff and used by the patch 10 10 + * module init code to build the klp_patch, klp_object and klp_func structs 11 11 + * needed by the livepatch API. 12 12 + */ 13 13 + #define KLP_OBJECTS_SEC "__klp_objects" 14 14 + #define KLP_FUNCS_SEC "__klp_funcs" 15 15 + 16 16 + /* 17 17 + * __klp_relocs is an intermediate section which are created by klp diff and 18 18 + * converted into KLP symbols/relas by "objtool klp post-link". This is needed 19 19 + * to work around the linker, which doesn't preserve SHN_LIVEPATCH or 20 20 + * SHF_RELA_LIVEPATCH, nor does it support having two RELA sections for a 21 21 + * single PROGBITS section. 22 22 + */ 23 23 + #define KLP_RELOCS_SEC "__klp_relocs" 24 24 + #define KLP_STRINGS_SEC ".rodata.klp.str1.1" 25 25 + 26 26 + struct klp_reloc { 27 27 + void *offset; 28 28 + void *sym; 29 29 + u32 type; 30 30 + }; 31 31 + 32 32 + int cmd_klp_diff(int argc, const char **argv); 33 33 + int cmd_klp_post_link(int argc, const char **argv); 34 34 + 35 35 + #endif /* _OBJTOOL_KLP_H */

+3 -1

tools/objtool/include/objtool/objtool.h

reviewed

··· 28 28 struct list_head mcount_loc_list; 29 29 struct list_head endbr_list; 30 30 struct list_head call_list; 31 31 - bool ignore_unreachables, hints, rodata; 31 31 + bool ignore_unreachables, hints, rodata, klp; 32 32 33 33 unsigned int nr_endbr; 34 34 unsigned int nr_endbr_int; ··· 38 38 39 39 struct pv_state *pv_ops; 40 40 }; 41 41 + 42 42 + char *top_level_dir(const char *file); 41 43 42 44 struct objtool_file *objtool_open_read(const char *_objname); 43 45

+3 -1

tools/objtool/include/objtool/special.h

reviewed

··· 25 25 struct section *new_sec; 26 26 unsigned long new_off; 27 27 28 28 - unsigned int orig_len, new_len; /* group only */ 28 28 + unsigned int orig_len, new_len, feature; /* group only */ 29 29 }; 30 30 31 31 int special_get_alts(struct elf *elf, struct list_head *alts); ··· 38 38 struct reloc *arch_find_switch_table(struct objtool_file *file, 39 39 struct instruction *insn, 40 40 unsigned long *table_size); 41 41 + const char *arch_cpu_feature_name(int feature_number); 42 42 + 41 43 #endif /* _SPECIAL_H */

+141

tools/objtool/include/objtool/trace.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 + /* 3 3 + * Copyright (c) 2025, Oracle and/or its affiliates. 4 4 + */ 5 5 + 6 6 + #ifndef _TRACE_H 7 7 + #define _TRACE_H 8 8 + 9 9 + #include <objtool/check.h> 10 10 + #include <objtool/disas.h> 11 11 + 12 12 + #ifdef DISAS 13 13 + 14 14 + extern bool trace; 15 15 + extern int trace_depth; 16 16 + 17 17 + #define TRACE(fmt, ...) \ 18 18 + ({ if (trace) \ 19 19 + fprintf(stderr, fmt, ##__VA_ARGS__); \ 20 20 + }) 21 21 + 22 22 + /* 23 23 + * Print the instruction address and a message. The instruction 24 24 + * itself is not printed. 25 25 + */ 26 26 + #define TRACE_ADDR(insn, fmt, ...) \ 27 27 + ({ \ 28 28 + if (trace) { \ 29 29 + disas_print_info(stderr, insn, trace_depth - 1, \ 30 30 + fmt "\n", ##__VA_ARGS__); \ 31 31 + } \ 32 32 + }) 33 33 + 34 34 + /* 35 35 + * Print the instruction address, the instruction and a message. 36 36 + */ 37 37 + #define TRACE_INSN(insn, fmt, ...) \ 38 38 + ({ \ 39 39 + if (trace) { \ 40 40 + disas_print_insn(stderr, objtool_disas_ctx, \ 41 41 + insn, trace_depth - 1, \ 42 42 + fmt, ##__VA_ARGS__); \ 43 43 + fprintf(stderr, "\n"); \ 44 44 + insn->trace = 1; \ 45 45 + } \ 46 46 + }) 47 47 + 48 48 + #define TRACE_INSN_STATE(insn, sprev, snext) \ 49 49 + ({ \ 50 50 + if (trace) \ 51 51 + trace_insn_state(insn, sprev, snext); \ 52 52 + }) 53 53 + 54 54 + #define TRACE_ALT_FMT(pfx, fmt) pfx "<%s.%lx> " fmt 55 55 + #define TRACE_ALT_ARG(insn) disas_alt_type_name(insn), (insn)->offset 56 56 + 57 57 + #define TRACE_ALT(insn, fmt, ...) \ 58 58 + TRACE_INSN(insn, TRACE_ALT_FMT("", fmt), \ 59 59 + TRACE_ALT_ARG(insn), ##__VA_ARGS__) 60 60 + 61 61 + #define TRACE_ALT_INFO(insn, pfx, fmt, ...) \ 62 62 + TRACE_ADDR(insn, TRACE_ALT_FMT(pfx, fmt), \ 63 63 + TRACE_ALT_ARG(insn), ##__VA_ARGS__) 64 64 + 65 65 + #define TRACE_ALT_INFO_NOADDR(insn, pfx, fmt, ...) \ 66 66 + TRACE_ADDR(NULL, TRACE_ALT_FMT(pfx, fmt), \ 67 67 + TRACE_ALT_ARG(insn), ##__VA_ARGS__) 68 68 + 69 69 + #define TRACE_ALT_BEGIN(insn, alt, alt_name) \ 70 70 + ({ \ 71 71 + if (trace) { \ 72 72 + alt_name = disas_alt_name(alt); \ 73 73 + trace_alt_begin(insn, alt, alt_name); \ 74 74 + } \ 75 75 + }) 76 76 + 77 77 + #define TRACE_ALT_END(insn, alt, alt_name) \ 78 78 + ({ \ 79 79 + if (trace) { \ 80 80 + trace_alt_end(insn, alt, alt_name); \ 81 81 + free(alt_name); \ 82 82 + } \ 83 83 + }) 84 84 + 85 85 + static inline void trace_enable(void) 86 86 + { 87 87 + trace = true; 88 88 + trace_depth = 0; 89 89 + } 90 90 + 91 91 + static inline void trace_disable(void) 92 92 + { 93 93 + trace = false; 94 94 + } 95 95 + 96 96 + static inline void trace_depth_inc(void) 97 97 + { 98 98 + if (trace) 99 99 + trace_depth++; 100 100 + } 101 101 + 102 102 + static inline void trace_depth_dec(void) 103 103 + { 104 104 + if (trace) 105 105 + trace_depth--; 106 106 + } 107 107 + 108 108 + void trace_insn_state(struct instruction *insn, struct insn_state *sprev, 109 109 + struct insn_state *snext); 110 110 + void trace_alt_begin(struct instruction *orig_insn, struct alternative *alt, 111 111 + char *alt_name); 112 112 + void trace_alt_end(struct instruction *orig_insn, struct alternative *alt, 113 113 + char *alt_name); 114 114 + 115 115 + #else /* DISAS */ 116 116 + 117 117 + #define TRACE(fmt, ...) ({}) 118 118 + #define TRACE_ADDR(insn, fmt, ...) ({}) 119 119 + #define TRACE_INSN(insn, fmt, ...) ({}) 120 120 + #define TRACE_INSN_STATE(insn, sprev, snext) ({}) 121 121 + #define TRACE_ALT(insn, fmt, ...) ({}) 122 122 + #define TRACE_ALT_INFO(insn, fmt, ...) ({}) 123 123 + #define TRACE_ALT_INFO_NOADDR(insn, fmt, ...) ({}) 124 124 + #define TRACE_ALT_BEGIN(insn, alt, alt_name) ({}) 125 125 + #define TRACE_ALT_END(insn, alt, alt_name) ({}) 126 126 + 127 127 + 128 128 + static inline void trace_enable(void) {} 129 129 + static inline void trace_disable(void) {} 130 130 + static inline void trace_depth_inc(void) {} 131 131 + static inline void trace_depth_dec(void) {} 132 132 + static inline void trace_alt_begin(struct instruction *orig_insn, 133 133 + struct alternative *alt, 134 134 + char *alt_name) {}; 135 135 + static inline void trace_alt_end(struct instruction *orig_insn, 136 136 + struct alternative *alt, 137 137 + char *alt_name) {}; 138 138 + 139 139 + #endif 140 140 + 141 141 + #endif /* _TRACE_H */

+19

tools/objtool/include/objtool/util.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 + #ifndef _UTIL_H 3 3 + #define _UTIL_H 4 4 + 5 5 + #include <objtool/warn.h> 6 6 + 7 7 + #define snprintf_check(str, size, format, args...) \ 8 8 + ({ \ 9 9 + int __ret = snprintf(str, size, format, args); \ 10 10 + if (__ret < 0) \ 11 11 + ERROR_GLIBC("snprintf"); \ 12 12 + else if (__ret >= size) \ 13 13 + ERROR("snprintf() failed for '" format "'", args); \ 14 14 + else \ 15 15 + __ret = 0; \ 16 16 + __ret; \ 17 17 + }) 18 18 + 19 19 + #endif /* _UTIL_H */

+61 -5

tools/objtool/include/objtool/warn.h

reviewed

··· 77 77 #define WARN_INSN(insn, format, ...) \ 78 78 ({ \ 79 79 struct instruction *_insn = (insn); \ 80 80 - if (!_insn->sym || !_insn->sym->warned) \ 80 80 + if (!_insn->sym || !_insn->sym->warned) { \ 81 81 WARN_FUNC(_insn->sec, _insn->offset, format, \ 82 82 ##__VA_ARGS__); \ 83 83 + BT_INSN(_insn, ""); \ 84 84 + } \ 83 85 if (_insn->sym) \ 84 86 _insn->sym->warned = 1; \ 85 87 }) ··· 89 87 #define BT_INSN(insn, format, ...) \ 90 88 ({ \ 91 89 if (opts.verbose || opts.backtrace) { \ 92 92 - struct instruction *_insn = (insn); \ 93 93 - char *_str = offstr(_insn->sec, _insn->offset); \ 94 94 - WARN(" %s: " format, _str, ##__VA_ARGS__); \ 95 95 - free(_str); \ 90 90 + struct instruction *__insn = (insn); \ 91 91 + char *_str = offstr(__insn->sec, __insn->offset); \ 92 92 + const char *_istr = objtool_disas_insn(__insn); \ 93 93 + int _len; \ 94 94 + _len = snprintf(NULL, 0, " %s: " format, _str, ##__VA_ARGS__); \ 95 95 + _len = (_len < 50) ? 50 - _len : 0; \ 96 96 + WARN(" %s: " format " %*s%s", _str, ##__VA_ARGS__, _len, "", _istr); \ 97 97 + free(_str); \ 98 98 + __insn->trace = 1; \ 96 99 } \ 97 100 }) 98 101 ··· 108 101 #define ERROR_GLIBC(format, ...) __WARN_GLIBC(ERROR_STR, format, ##__VA_ARGS__) 109 102 #define ERROR_FUNC(sec, offset, format, ...) __WARN_FUNC(ERROR_STR, sec, offset, format, ##__VA_ARGS__) 110 103 #define ERROR_INSN(insn, format, ...) WARN_FUNC(insn->sec, insn->offset, format, ##__VA_ARGS__) 104 104 + 105 105 + extern bool debug; 106 106 + extern int indent; 107 107 + 108 108 + static inline void unindent(int *unused) { indent--; } 109 109 + 110 110 + /* 111 111 + * Clang prior to 17 is being silly and considers many __cleanup() variables 112 112 + * as unused (because they are, their sole purpose is to go out of scope). 113 113 + * 114 114 + * https://github.com/llvm/llvm-project/commit/877210faa447f4cc7db87812f8ed80e398fedd61 115 115 + */ 116 116 + #undef __cleanup 117 117 + #define __cleanup(func) __maybe_unused __attribute__((__cleanup__(func))) 118 118 + 119 119 + #define __dbg(format, ...) \ 120 120 + fprintf(stderr, \ 121 121 + "DEBUG: %s%s" format "\n", \ 122 122 + objname ?: "", \ 123 123 + objname ? ": " : "", \ 124 124 + ##__VA_ARGS__) 125 125 + 126 126 + #define dbg(args...) \ 127 127 + ({ \ 128 128 + if (unlikely(debug)) \ 129 129 + __dbg(args); \ 130 130 + }) 131 131 + 132 132 + #define __dbg_indent(format, ...) \ 133 133 + ({ \ 134 134 + if (unlikely(debug)) \ 135 135 + __dbg("%*s" format, indent * 8, "", ##__VA_ARGS__); \ 136 136 + }) 137 137 + 138 138 + #define dbg_indent(args...) \ 139 139 + int __cleanup(unindent) __dummy_##__COUNTER__; \ 140 140 + __dbg_indent(args); \ 141 141 + indent++ 142 142 + 143 143 + #define dbg_checksum(func, insn, checksum) \ 144 144 + ({ \ 145 145 + if (unlikely(insn->sym && insn->sym->pfunc && \ 146 146 + insn->sym->pfunc->debug_checksum)) { \ 147 147 + char *insn_off = offstr(insn->sec, insn->offset); \ 148 148 + __dbg("checksum: %s %s %016lx", \ 149 149 + func->name, insn_off, checksum); \ 150 150 + free(insn_off); \ 151 151 + } \ 152 152 + }) 111 153 112 154 #endif /* _WARN_H */

+1723

tools/objtool/klp-diff.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + #define _GNU_SOURCE /* memmem() */ 3 3 + #include <subcmd/parse-options.h> 4 4 + #include <stdlib.h> 5 5 + #include <string.h> 6 6 + #include <libgen.h> 7 7 + #include <stdio.h> 8 8 + #include <ctype.h> 9 9 + 10 10 + #include <objtool/objtool.h> 11 11 + #include <objtool/warn.h> 12 12 + #include <objtool/arch.h> 13 13 + #include <objtool/klp.h> 14 14 + #include <objtool/util.h> 15 15 + #include <arch/special.h> 16 16 + 17 17 + #include <linux/objtool_types.h> 18 18 + #include <linux/livepatch_external.h> 19 19 + #include <linux/stringify.h> 20 20 + #include <linux/string.h> 21 21 + #include <linux/jhash.h> 22 22 + 23 23 + #define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) 24 24 + 25 25 + struct elfs { 26 26 + struct elf *orig, *patched, *out; 27 27 + const char *modname; 28 28 + }; 29 29 + 30 30 + struct export { 31 31 + struct hlist_node hash; 32 32 + char *mod, *sym; 33 33 + }; 34 34 + 35 35 + static const char * const klp_diff_usage[] = { 36 36 + "objtool klp diff [<options>] <in1.o> <in2.o> <out.o>", 37 37 + NULL, 38 38 + }; 39 39 + 40 40 + static const struct option klp_diff_options[] = { 41 41 + OPT_GROUP("Options:"), 42 42 + OPT_BOOLEAN('d', "debug", &debug, "enable debug output"), 43 43 + OPT_END(), 44 44 + }; 45 45 + 46 46 + static DEFINE_HASHTABLE(exports, 15); 47 47 + 48 48 + static inline u32 str_hash(const char *str) 49 49 + { 50 50 + return jhash(str, strlen(str), 0); 51 51 + } 52 52 + 53 53 + static char *escape_str(const char *orig) 54 54 + { 55 55 + size_t len = 0; 56 56 + const char *a; 57 57 + char *b, *new; 58 58 + 59 59 + for (a = orig; *a; a++) { 60 60 + switch (*a) { 61 61 + case '\001': len += 5; break; 62 62 + case '\n': 63 63 + case '\t': len += 2; break; 64 64 + default: len++; 65 65 + } 66 66 + } 67 67 + 68 68 + new = malloc(len + 1); 69 69 + if (!new) 70 70 + return NULL; 71 71 + 72 72 + for (a = orig, b = new; *a; a++) { 73 73 + switch (*a) { 74 74 + case '\001': memcpy(b, "<SOH>", 5); b += 5; break; 75 75 + case '\n': *b++ = '\\'; *b++ = 'n'; break; 76 76 + case '\t': *b++ = '\\'; *b++ = 't'; break; 77 77 + default: *b++ = *a; 78 78 + } 79 79 + } 80 80 + 81 81 + *b = '\0'; 82 82 + return new; 83 83 + } 84 84 + 85 85 + static int read_exports(void) 86 86 + { 87 87 + const char *symvers = "Module.symvers"; 88 88 + char line[1024], *path = NULL; 89 89 + unsigned int line_num = 1; 90 90 + FILE *file; 91 91 + 92 92 + file = fopen(symvers, "r"); 93 93 + if (!file) { 94 94 + path = top_level_dir(symvers); 95 95 + if (!path) { 96 96 + ERROR("can't open '%s', \"objtool diff\" should be run from the kernel tree", symvers); 97 97 + return -1; 98 98 + } 99 99 + 100 100 + file = fopen(path, "r"); 101 101 + if (!file) { 102 102 + ERROR_GLIBC("fopen"); 103 103 + return -1; 104 104 + } 105 105 + } 106 106 + 107 107 + while (fgets(line, 1024, file)) { 108 108 + char *sym, *mod, *type; 109 109 + struct export *export; 110 110 + 111 111 + sym = strchr(line, '\t'); 112 112 + if (!sym) { 113 113 + ERROR("malformed Module.symvers (sym) at line %d", line_num); 114 114 + return -1; 115 115 + } 116 116 + 117 117 + *sym++ = '\0'; 118 118 + 119 119 + mod = strchr(sym, '\t'); 120 120 + if (!mod) { 121 121 + ERROR("malformed Module.symvers (mod) at line %d", line_num); 122 122 + return -1; 123 123 + } 124 124 + 125 125 + *mod++ = '\0'; 126 126 + 127 127 + type = strchr(mod, '\t'); 128 128 + if (!type) { 129 129 + ERROR("malformed Module.symvers (type) at line %d", line_num); 130 130 + return -1; 131 131 + } 132 132 + 133 133 + *type++ = '\0'; 134 134 + 135 135 + if (*sym == '\0' || *mod == '\0') { 136 136 + ERROR("malformed Module.symvers at line %d", line_num); 137 137 + return -1; 138 138 + } 139 139 + 140 140 + export = calloc(1, sizeof(*export)); 141 141 + if (!export) { 142 142 + ERROR_GLIBC("calloc"); 143 143 + return -1; 144 144 + } 145 145 + 146 146 + export->mod = strdup(mod); 147 147 + if (!export->mod) { 148 148 + ERROR_GLIBC("strdup"); 149 149 + return -1; 150 150 + } 151 151 + 152 152 + export->sym = strdup(sym); 153 153 + if (!export->sym) { 154 154 + ERROR_GLIBC("strdup"); 155 155 + return -1; 156 156 + } 157 157 + 158 158 + hash_add(exports, &export->hash, str_hash(sym)); 159 159 + } 160 160 + 161 161 + free(path); 162 162 + fclose(file); 163 163 + 164 164 + return 0; 165 165 + } 166 166 + 167 167 + static int read_sym_checksums(struct elf *elf) 168 168 + { 169 169 + struct section *sec; 170 170 + 171 171 + sec = find_section_by_name(elf, ".discard.sym_checksum"); 172 172 + if (!sec) { 173 173 + ERROR("'%s' missing .discard.sym_checksum section, file not processed by 'objtool --checksum'?", 174 174 + elf->name); 175 175 + return -1; 176 176 + } 177 177 + 178 178 + if (!sec->rsec) { 179 179 + ERROR("missing reloc section for .discard.sym_checksum"); 180 180 + return -1; 181 181 + } 182 182 + 183 183 + if (sec_size(sec) % sizeof(struct sym_checksum)) { 184 184 + ERROR("struct sym_checksum size mismatch"); 185 185 + return -1; 186 186 + } 187 187 + 188 188 + for (int i = 0; i < sec_size(sec) / sizeof(struct sym_checksum); i++) { 189 189 + struct sym_checksum *sym_checksum; 190 190 + struct reloc *reloc; 191 191 + struct symbol *sym; 192 192 + 193 193 + sym_checksum = (struct sym_checksum *)sec->data->d_buf + i; 194 194 + 195 195 + reloc = find_reloc_by_dest(elf, sec, i * sizeof(*sym_checksum)); 196 196 + if (!reloc) { 197 197 + ERROR("can't find reloc for sym_checksum[%d]", i); 198 198 + return -1; 199 199 + } 200 200 + 201 201 + sym = reloc->sym; 202 202 + 203 203 + if (is_sec_sym(sym)) { 204 204 + ERROR("not sure how to handle section %s", sym->name); 205 205 + return -1; 206 206 + } 207 207 + 208 208 + if (is_func_sym(sym)) 209 209 + sym->csum.checksum = sym_checksum->checksum; 210 210 + } 211 211 + 212 212 + return 0; 213 213 + } 214 214 + 215 215 + static struct symbol *first_file_symbol(struct elf *elf) 216 216 + { 217 217 + struct symbol *sym; 218 218 + 219 219 + for_each_sym(elf, sym) { 220 220 + if (is_file_sym(sym)) 221 221 + return sym; 222 222 + } 223 223 + 224 224 + return NULL; 225 225 + } 226 226 + 227 227 + static struct symbol *next_file_symbol(struct elf *elf, struct symbol *sym) 228 228 + { 229 229 + for_each_sym_continue(elf, sym) { 230 230 + if (is_file_sym(sym)) 231 231 + return sym; 232 232 + } 233 233 + 234 234 + return NULL; 235 235 + } 236 236 + 237 237 + /* 238 238 + * Certain static local variables should never be correlated. They will be 239 239 + * used in place rather than referencing the originals. 240 240 + */ 241 241 + static bool is_uncorrelated_static_local(struct symbol *sym) 242 242 + { 243 243 + static const char * const vars[] = { 244 244 + "__already_done.", 245 245 + "__func__.", 246 246 + "__key.", 247 247 + "__warned.", 248 248 + "_entry.", 249 249 + "_entry_ptr.", 250 250 + "_rs.", 251 251 + "descriptor.", 252 252 + "CSWTCH.", 253 253 + }; 254 254 + 255 255 + if (!is_object_sym(sym) || !is_local_sym(sym)) 256 256 + return false; 257 257 + 258 258 + if (!strcmp(sym->sec->name, ".data.once")) 259 259 + return true; 260 260 + 261 261 + for (int i = 0; i < ARRAY_SIZE(vars); i++) { 262 262 + if (strstarts(sym->name, vars[i])) 263 263 + return true; 264 264 + } 265 265 + 266 266 + return false; 267 267 + } 268 268 + 269 269 + /* 270 270 + * Clang emits several useless .Ltmp_* code labels. 271 271 + */ 272 272 + static bool is_clang_tmp_label(struct symbol *sym) 273 273 + { 274 274 + return sym->type == STT_NOTYPE && 275 275 + is_text_sec(sym->sec) && 276 276 + strstarts(sym->name, ".Ltmp") && 277 277 + isdigit(sym->name[5]); 278 278 + } 279 279 + 280 280 + static bool is_special_section(struct section *sec) 281 281 + { 282 282 + static const char * const specials[] = { 283 283 + ".altinstructions", 284 284 + ".smp_locks", 285 285 + "__bug_table", 286 286 + "__ex_table", 287 287 + "__jump_table", 288 288 + "__mcount_loc", 289 289 + 290 290 + /* 291 291 + * Extract .static_call_sites here to inherit non-module 292 292 + * preferential treatment. The later static call processing 293 293 + * during klp module build will be skipped when it sees this 294 294 + * section already exists. 295 295 + */ 296 296 + ".static_call_sites", 297 297 + }; 298 298 + 299 299 + static const char * const non_special_discards[] = { 300 300 + ".discard.addressable", 301 301 + ".discard.sym_checksum", 302 302 + }; 303 303 + 304 304 + if (is_text_sec(sec)) 305 305 + return false; 306 306 + 307 307 + for (int i = 0; i < ARRAY_SIZE(specials); i++) { 308 308 + if (!strcmp(sec->name, specials[i])) 309 309 + return true; 310 310 + } 311 311 + 312 312 + /* Most .discard data sections are special */ 313 313 + for (int i = 0; i < ARRAY_SIZE(non_special_discards); i++) { 314 314 + if (!strcmp(sec->name, non_special_discards[i])) 315 315 + return false; 316 316 + } 317 317 + 318 318 + return strstarts(sec->name, ".discard."); 319 319 + } 320 320 + 321 321 + /* 322 322 + * These sections are referenced by special sections but aren't considered 323 323 + * special sections themselves. 324 324 + */ 325 325 + static bool is_special_section_aux(struct section *sec) 326 326 + { 327 327 + static const char * const specials_aux[] = { 328 328 + ".altinstr_replacement", 329 329 + ".altinstr_aux", 330 330 + }; 331 331 + 332 332 + for (int i = 0; i < ARRAY_SIZE(specials_aux); i++) { 333 333 + if (!strcmp(sec->name, specials_aux[i])) 334 334 + return true; 335 335 + } 336 336 + 337 337 + return false; 338 338 + } 339 339 + 340 340 + /* 341 341 + * These symbols should never be correlated, so their local patched versions 342 342 + * are used instead of linking to the originals. 343 343 + */ 344 344 + static bool dont_correlate(struct symbol *sym) 345 345 + { 346 346 + return is_file_sym(sym) || 347 347 + is_null_sym(sym) || 348 348 + is_sec_sym(sym) || 349 349 + is_prefix_func(sym) || 350 350 + is_uncorrelated_static_local(sym) || 351 351 + is_clang_tmp_label(sym) || 352 352 + is_string_sec(sym->sec) || 353 353 + is_special_section(sym->sec) || 354 354 + is_special_section_aux(sym->sec) || 355 355 + strstarts(sym->name, "__initcall__"); 356 356 + } 357 357 + 358 358 + /* 359 359 + * For each symbol in the original kernel, find its corresponding "twin" in the 360 360 + * patched kernel. 361 361 + */ 362 362 + static int correlate_symbols(struct elfs *e) 363 363 + { 364 364 + struct symbol *file1_sym, *file2_sym; 365 365 + struct symbol *sym1, *sym2; 366 366 + 367 367 + /* Correlate locals */ 368 368 + for (file1_sym = first_file_symbol(e->orig), 369 369 + file2_sym = first_file_symbol(e->patched); ; 370 370 + file1_sym = next_file_symbol(e->orig, file1_sym), 371 371 + file2_sym = next_file_symbol(e->patched, file2_sym)) { 372 372 + 373 373 + if (!file1_sym && file2_sym) { 374 374 + ERROR("FILE symbol mismatch: NULL != %s", file2_sym->name); 375 375 + return -1; 376 376 + } 377 377 + 378 378 + if (file1_sym && !file2_sym) { 379 379 + ERROR("FILE symbol mismatch: %s != NULL", file1_sym->name); 380 380 + return -1; 381 381 + } 382 382 + 383 383 + if (!file1_sym) 384 384 + break; 385 385 + 386 386 + if (strcmp(file1_sym->name, file2_sym->name)) { 387 387 + ERROR("FILE symbol mismatch: %s != %s", file1_sym->name, file2_sym->name); 388 388 + return -1; 389 389 + } 390 390 + 391 391 + file1_sym->twin = file2_sym; 392 392 + file2_sym->twin = file1_sym; 393 393 + 394 394 + sym1 = file1_sym; 395 395 + 396 396 + for_each_sym_continue(e->orig, sym1) { 397 397 + if (is_file_sym(sym1) || !is_local_sym(sym1)) 398 398 + break; 399 399 + 400 400 + if (dont_correlate(sym1)) 401 401 + continue; 402 402 + 403 403 + sym2 = file2_sym; 404 404 + for_each_sym_continue(e->patched, sym2) { 405 405 + if (is_file_sym(sym2) || !is_local_sym(sym2)) 406 406 + break; 407 407 + 408 408 + if (sym2->twin || dont_correlate(sym2)) 409 409 + continue; 410 410 + 411 411 + if (strcmp(sym1->demangled_name, sym2->demangled_name)) 412 412 + continue; 413 413 + 414 414 + sym1->twin = sym2; 415 415 + sym2->twin = sym1; 416 416 + break; 417 417 + } 418 418 + } 419 419 + } 420 420 + 421 421 + /* Correlate globals */ 422 422 + for_each_sym(e->orig, sym1) { 423 423 + if (sym1->bind == STB_LOCAL) 424 424 + continue; 425 425 + 426 426 + sym2 = find_global_symbol_by_name(e->patched, sym1->name); 427 427 + 428 428 + if (sym2 && !sym2->twin && !strcmp(sym1->name, sym2->name)) { 429 429 + sym1->twin = sym2; 430 430 + sym2->twin = sym1; 431 431 + } 432 432 + } 433 433 + 434 434 + for_each_sym(e->orig, sym1) { 435 435 + if (sym1->twin || dont_correlate(sym1)) 436 436 + continue; 437 437 + WARN("no correlation: %s", sym1->name); 438 438 + } 439 439 + 440 440 + return 0; 441 441 + } 442 442 + 443 443 + /* "sympos" is used by livepatch to disambiguate duplicate symbol names */ 444 444 + static unsigned long find_sympos(struct elf *elf, struct symbol *sym) 445 445 + { 446 446 + bool vmlinux = str_ends_with(objname, "vmlinux.o"); 447 447 + unsigned long sympos = 0, nr_matches = 0; 448 448 + bool has_dup = false; 449 449 + struct symbol *s; 450 450 + 451 451 + if (sym->bind != STB_LOCAL) 452 452 + return 0; 453 453 + 454 454 + if (vmlinux && sym->type == STT_FUNC) { 455 455 + /* 456 456 + * HACK: Unfortunately, symbol ordering can differ between 457 457 + * vmlinux.o and vmlinux due to the linker script emitting 458 458 + * .text.unlikely* before .text*. Count .text.unlikely* first. 459 459 + * 460 460 + * TODO: Disambiguate symbols more reliably (checksums?) 461 461 + */ 462 462 + for_each_sym(elf, s) { 463 463 + if (strstarts(s->sec->name, ".text.unlikely") && 464 464 + !strcmp(s->name, sym->name)) { 465 465 + nr_matches++; 466 466 + if (s == sym) 467 467 + sympos = nr_matches; 468 468 + else 469 469 + has_dup = true; 470 470 + } 471 471 + } 472 472 + for_each_sym(elf, s) { 473 473 + if (!strstarts(s->sec->name, ".text.unlikely") && 474 474 + !strcmp(s->name, sym->name)) { 475 475 + nr_matches++; 476 476 + if (s == sym) 477 477 + sympos = nr_matches; 478 478 + else 479 479 + has_dup = true; 480 480 + } 481 481 + } 482 482 + } else { 483 483 + for_each_sym(elf, s) { 484 484 + if (!strcmp(s->name, sym->name)) { 485 485 + nr_matches++; 486 486 + if (s == sym) 487 487 + sympos = nr_matches; 488 488 + else 489 489 + has_dup = true; 490 490 + } 491 491 + } 492 492 + } 493 493 + 494 494 + if (!sympos) { 495 495 + ERROR("can't find sympos for %s", sym->name); 496 496 + return ULONG_MAX; 497 497 + } 498 498 + 499 499 + return has_dup ? sympos : 0; 500 500 + } 501 501 + 502 502 + static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym); 503 503 + 504 504 + static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym, 505 505 + bool data_too) 506 506 + { 507 507 + struct section *out_sec = NULL; 508 508 + unsigned long offset = 0; 509 509 + struct symbol *out_sym; 510 510 + 511 511 + if (data_too && !is_undef_sym(patched_sym)) { 512 512 + struct section *patched_sec = patched_sym->sec; 513 513 + 514 514 + out_sec = find_section_by_name(elf, patched_sec->name); 515 515 + if (!out_sec) { 516 516 + out_sec = elf_create_section(elf, patched_sec->name, 0, 517 517 + patched_sec->sh.sh_entsize, 518 518 + patched_sec->sh.sh_type, 519 519 + patched_sec->sh.sh_addralign, 520 520 + patched_sec->sh.sh_flags); 521 521 + if (!out_sec) 522 522 + return NULL; 523 523 + } 524 524 + 525 525 + if (is_string_sec(patched_sym->sec)) { 526 526 + out_sym = elf_create_section_symbol(elf, out_sec); 527 527 + if (!out_sym) 528 528 + return NULL; 529 529 + 530 530 + goto sym_created; 531 531 + } 532 532 + 533 533 + if (!is_sec_sym(patched_sym)) 534 534 + offset = sec_size(out_sec); 535 535 + 536 536 + if (patched_sym->len || is_sec_sym(patched_sym)) { 537 537 + void *data = NULL; 538 538 + size_t size; 539 539 + 540 540 + /* bss doesn't have data */ 541 541 + if (patched_sym->sec->data->d_buf) 542 542 + data = patched_sym->sec->data->d_buf + patched_sym->offset; 543 543 + 544 544 + if (is_sec_sym(patched_sym)) 545 545 + size = sec_size(patched_sym->sec); 546 546 + else 547 547 + size = patched_sym->len; 548 548 + 549 549 + if (!elf_add_data(elf, out_sec, data, size)) 550 550 + return NULL; 551 551 + } 552 552 + } 553 553 + 554 554 + out_sym = elf_create_symbol(elf, patched_sym->name, out_sec, 555 555 + patched_sym->bind, patched_sym->type, 556 556 + offset, patched_sym->len); 557 557 + if (!out_sym) 558 558 + return NULL; 559 559 + 560 560 + sym_created: 561 561 + patched_sym->clone = out_sym; 562 562 + out_sym->clone = patched_sym; 563 563 + 564 564 + return out_sym; 565 565 + } 566 566 + 567 567 + static const char *sym_type(struct symbol *sym) 568 568 + { 569 569 + switch (sym->type) { 570 570 + case STT_NOTYPE: return "NOTYPE"; 571 571 + case STT_OBJECT: return "OBJECT"; 572 572 + case STT_FUNC: return "FUNC"; 573 573 + case STT_SECTION: return "SECTION"; 574 574 + case STT_FILE: return "FILE"; 575 575 + default: return "UNKNOWN"; 576 576 + } 577 577 + } 578 578 + 579 579 + static const char *sym_bind(struct symbol *sym) 580 580 + { 581 581 + switch (sym->bind) { 582 582 + case STB_LOCAL: return "LOCAL"; 583 583 + case STB_GLOBAL: return "GLOBAL"; 584 584 + case STB_WEAK: return "WEAK"; 585 585 + default: return "UNKNOWN"; 586 586 + } 587 587 + } 588 588 + 589 589 + /* 590 590 + * Copy a symbol to the output object, optionally including its data and 591 591 + * relocations. 592 592 + */ 593 593 + static struct symbol *clone_symbol(struct elfs *e, struct symbol *patched_sym, 594 594 + bool data_too) 595 595 + { 596 596 + struct symbol *pfx; 597 597 + 598 598 + if (patched_sym->clone) 599 599 + return patched_sym->clone; 600 600 + 601 601 + dbg_indent("%s%s", patched_sym->name, data_too ? " [+DATA]" : ""); 602 602 + 603 603 + /* Make sure the prefix gets cloned first */ 604 604 + if (is_func_sym(patched_sym) && data_too) { 605 605 + pfx = get_func_prefix(patched_sym); 606 606 + if (pfx) 607 607 + clone_symbol(e, pfx, true); 608 608 + } 609 609 + 610 610 + if (!__clone_symbol(e->out, patched_sym, data_too)) 611 611 + return NULL; 612 612 + 613 613 + if (data_too && clone_sym_relocs(e, patched_sym)) 614 614 + return NULL; 615 615 + 616 616 + return patched_sym->clone; 617 617 + } 618 618 + 619 619 + static void mark_included_function(struct symbol *func) 620 620 + { 621 621 + struct symbol *pfx; 622 622 + 623 623 + func->included = 1; 624 624 + 625 625 + /* Include prefix function */ 626 626 + pfx = get_func_prefix(func); 627 627 + if (pfx) 628 628 + pfx->included = 1; 629 629 + 630 630 + /* Make sure .cold parent+child always stay together */ 631 631 + if (func->cfunc && func->cfunc != func) 632 632 + func->cfunc->included = 1; 633 633 + if (func->pfunc && func->pfunc != func) 634 634 + func->pfunc->included = 1; 635 635 + } 636 636 + 637 637 + /* 638 638 + * Copy all changed functions (and their dependencies) from the patched object 639 639 + * to the output object. 640 640 + */ 641 641 + static int mark_changed_functions(struct elfs *e) 642 642 + { 643 643 + struct symbol *sym_orig, *patched_sym; 644 644 + bool changed = false; 645 645 + 646 646 + /* Find changed functions */ 647 647 + for_each_sym(e->orig, sym_orig) { 648 648 + if (!is_func_sym(sym_orig) || is_prefix_func(sym_orig)) 649 649 + continue; 650 650 + 651 651 + patched_sym = sym_orig->twin; 652 652 + if (!patched_sym) 653 653 + continue; 654 654 + 655 655 + if (sym_orig->csum.checksum != patched_sym->csum.checksum) { 656 656 + patched_sym->changed = 1; 657 657 + mark_included_function(patched_sym); 658 658 + changed = true; 659 659 + } 660 660 + } 661 661 + 662 662 + /* Find added functions and print them */ 663 663 + for_each_sym(e->patched, patched_sym) { 664 664 + if (!is_func_sym(patched_sym) || is_prefix_func(patched_sym)) 665 665 + continue; 666 666 + 667 667 + if (!patched_sym->twin) { 668 668 + printf("%s: new function: %s\n", objname, patched_sym->name); 669 669 + mark_included_function(patched_sym); 670 670 + changed = true; 671 671 + } 672 672 + } 673 673 + 674 674 + /* Print changed functions */ 675 675 + for_each_sym(e->patched, patched_sym) { 676 676 + if (patched_sym->changed) 677 677 + printf("%s: changed function: %s\n", objname, patched_sym->name); 678 678 + } 679 679 + 680 680 + return !changed ? -1 : 0; 681 681 + } 682 682 + 683 683 + static int clone_included_functions(struct elfs *e) 684 684 + { 685 685 + struct symbol *patched_sym; 686 686 + 687 687 + for_each_sym(e->patched, patched_sym) { 688 688 + if (patched_sym->included) { 689 689 + if (!clone_symbol(e, patched_sym, true)) 690 690 + return -1; 691 691 + } 692 692 + } 693 693 + 694 694 + return 0; 695 695 + } 696 696 + 697 697 + /* 698 698 + * Determine whether a relocation should reference the section rather than the 699 699 + * underlying symbol. 700 700 + */ 701 701 + static bool section_reference_needed(struct section *sec) 702 702 + { 703 703 + /* 704 704 + * String symbols are zero-length and uncorrelated. It's easier to 705 705 + * deal with them as section symbols. 706 706 + */ 707 707 + if (is_string_sec(sec)) 708 708 + return true; 709 709 + 710 710 + /* 711 711 + * .rodata has mostly anonymous data so there's no way to determine the 712 712 + * length of a needed reference. just copy the whole section if needed. 713 713 + */ 714 714 + if (strstarts(sec->name, ".rodata")) 715 715 + return true; 716 716 + 717 717 + /* UBSAN anonymous data */ 718 718 + if (strstarts(sec->name, ".data..Lubsan") || /* GCC */ 719 719 + strstarts(sec->name, ".data..L__unnamed_")) /* Clang */ 720 720 + return true; 721 721 + 722 722 + return false; 723 723 + } 724 724 + 725 725 + static bool is_reloc_allowed(struct reloc *reloc) 726 726 + { 727 727 + return section_reference_needed(reloc->sym->sec) == is_sec_sym(reloc->sym); 728 728 + } 729 729 + 730 730 + static struct export *find_export(struct symbol *sym) 731 731 + { 732 732 + struct export *export; 733 733 + 734 734 + hash_for_each_possible(exports, export, hash, str_hash(sym->name)) { 735 735 + if (!strcmp(export->sym, sym->name)) 736 736 + return export; 737 737 + } 738 738 + 739 739 + return NULL; 740 740 + } 741 741 + 742 742 + static const char *__find_modname(struct elfs *e) 743 743 + { 744 744 + struct section *sec; 745 745 + char *name; 746 746 + 747 747 + sec = find_section_by_name(e->orig, ".modinfo"); 748 748 + if (!sec) { 749 749 + ERROR("missing .modinfo section"); 750 750 + return NULL; 751 751 + } 752 752 + 753 753 + name = memmem(sec->data->d_buf, sec_size(sec), "\0name=", 6); 754 754 + if (name) 755 755 + return name + 6; 756 756 + 757 757 + name = strdup(e->orig->name); 758 758 + if (!name) { 759 759 + ERROR_GLIBC("strdup"); 760 760 + return NULL; 761 761 + } 762 762 + 763 763 + for (char *c = name; *c; c++) { 764 764 + if (*c == '/') 765 765 + name = c + 1; 766 766 + else if (*c == '-') 767 767 + *c = '_'; 768 768 + else if (*c == '.') { 769 769 + *c = '\0'; 770 770 + break; 771 771 + } 772 772 + } 773 773 + 774 774 + return name; 775 775 + } 776 776 + 777 777 + /* Get the object's module name as defined by the kernel (and klp_object) */ 778 778 + static const char *find_modname(struct elfs *e) 779 779 + { 780 780 + const char *modname; 781 781 + 782 782 + if (e->modname) 783 783 + return e->modname; 784 784 + 785 785 + modname = __find_modname(e); 786 786 + e->modname = modname; 787 787 + return modname; 788 788 + } 789 789 + 790 790 + /* 791 791 + * Copying a function from its native compiled environment to a kernel module 792 792 + * removes its natural access to local functions/variables and unexported 793 793 + * globals. References to such symbols need to be converted to KLP relocs so 794 794 + * the kernel arch relocation code knows to apply them and where to find the 795 795 + * symbols. Particularly, duplicate static symbols need to be disambiguated. 796 796 + */ 797 797 + static bool klp_reloc_needed(struct reloc *patched_reloc) 798 798 + { 799 799 + struct symbol *patched_sym = patched_reloc->sym; 800 800 + struct export *export; 801 801 + 802 802 + /* no external symbol to reference */ 803 803 + if (dont_correlate(patched_sym)) 804 804 + return false; 805 805 + 806 806 + /* For included functions, a regular reloc will do. */ 807 807 + if (patched_sym->included) 808 808 + return false; 809 809 + 810 810 + /* 811 811 + * If exported by a module, it has to be a klp reloc. Thanks to the 812 812 + * clusterfunk that is late module patching, the patch module is 813 813 + * allowed to be loaded before any modules it depends on. 814 814 + * 815 815 + * If exported by vmlinux, a normal reloc will do. 816 816 + */ 817 817 + export = find_export(patched_sym); 818 818 + if (export) 819 819 + return strcmp(export->mod, "vmlinux"); 820 820 + 821 821 + if (!patched_sym->twin) { 822 822 + /* 823 823 + * Presumably the symbol and its reference were added by the 824 824 + * patch. The symbol could be defined in this .o or in another 825 825 + * .o in the patch module. 826 826 + * 827 827 + * This check needs to be *after* the export check due to the 828 828 + * possibility of the patch adding a new UNDEF reference to an 829 829 + * exported symbol. 830 830 + */ 831 831 + return false; 832 832 + } 833 833 + 834 834 + /* Unexported symbol which lives in the original vmlinux or module. */ 835 835 + return true; 836 836 + } 837 837 + 838 838 + static int convert_reloc_sym_to_secsym(struct elf *elf, struct reloc *reloc) 839 839 + { 840 840 + struct symbol *sym = reloc->sym; 841 841 + struct section *sec = sym->sec; 842 842 + 843 843 + if (!sec->sym && !elf_create_section_symbol(elf, sec)) 844 844 + return -1; 845 845 + 846 846 + reloc->sym = sec->sym; 847 847 + set_reloc_sym(elf, reloc, sym->idx); 848 848 + set_reloc_addend(elf, reloc, sym->offset + reloc_addend(reloc)); 849 849 + return 0; 850 850 + } 851 851 + 852 852 + static int convert_reloc_secsym_to_sym(struct elf *elf, struct reloc *reloc) 853 853 + { 854 854 + struct symbol *sym = reloc->sym; 855 855 + struct section *sec = sym->sec; 856 856 + 857 857 + /* If the symbol has a dedicated section, it's easy to find */ 858 858 + sym = find_symbol_by_offset(sec, 0); 859 859 + if (sym && sym->len == sec_size(sec)) 860 860 + goto found_sym; 861 861 + 862 862 + /* No dedicated section; find the symbol manually */ 863 863 + sym = find_symbol_containing(sec, arch_adjusted_addend(reloc)); 864 864 + if (!sym) { 865 865 + /* 866 866 + * This can happen for special section references to weak code 867 867 + * whose symbol has been stripped by the linker. 868 868 + */ 869 869 + return -1; 870 870 + } 871 871 + 872 872 + found_sym: 873 873 + reloc->sym = sym; 874 874 + set_reloc_sym(elf, reloc, sym->idx); 875 875 + set_reloc_addend(elf, reloc, reloc_addend(reloc) - sym->offset); 876 876 + return 0; 877 877 + } 878 878 + 879 879 + /* 880 880 + * Convert a relocation symbol reference to the needed format: either a section 881 881 + * symbol or the underlying symbol itself. 882 882 + */ 883 883 + static int convert_reloc_sym(struct elf *elf, struct reloc *reloc) 884 884 + { 885 885 + if (is_reloc_allowed(reloc)) 886 886 + return 0; 887 887 + 888 888 + if (section_reference_needed(reloc->sym->sec)) 889 889 + return convert_reloc_sym_to_secsym(elf, reloc); 890 890 + else 891 891 + return convert_reloc_secsym_to_sym(elf, reloc); 892 892 + } 893 893 + 894 894 + /* 895 895 + * Convert a regular relocation to a klp relocation (sort of). 896 896 + */ 897 897 + static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, 898 898 + struct section *sec, unsigned long offset, 899 899 + struct export *export) 900 900 + { 901 901 + struct symbol *patched_sym = patched_reloc->sym; 902 902 + s64 addend = reloc_addend(patched_reloc); 903 903 + const char *sym_modname, *sym_orig_name; 904 904 + static struct section *klp_relocs; 905 905 + struct symbol *sym, *klp_sym; 906 906 + unsigned long klp_reloc_off; 907 907 + char sym_name[SYM_NAME_LEN]; 908 908 + struct klp_reloc klp_reloc; 909 909 + unsigned long sympos; 910 910 + 911 911 + if (!patched_sym->twin) { 912 912 + ERROR("unexpected klp reloc for new symbol %s", patched_sym->name); 913 913 + return -1; 914 914 + } 915 915 + 916 916 + /* 917 917 + * Keep the original reloc intact for now to avoid breaking objtool run 918 918 + * which relies on proper relocations for many of its features. This 919 919 + * will be disabled later by "objtool klp post-link". 920 920 + * 921 921 + * Convert it to UNDEF (and WEAK to avoid modpost warnings). 922 922 + */ 923 923 + 924 924 + sym = patched_sym->clone; 925 925 + if (!sym) { 926 926 + /* STB_WEAK: avoid modpost undefined symbol warnings */ 927 927 + sym = elf_create_symbol(e->out, patched_sym->name, NULL, 928 928 + STB_WEAK, patched_sym->type, 0, 0); 929 929 + if (!sym) 930 930 + return -1; 931 931 + 932 932 + patched_sym->clone = sym; 933 933 + sym->clone = patched_sym; 934 934 + } 935 935 + 936 936 + if (!elf_create_reloc(e->out, sec, offset, sym, addend, reloc_type(patched_reloc))) 937 937 + return -1; 938 938 + 939 939 + /* 940 940 + * Create the KLP symbol. 941 941 + */ 942 942 + 943 943 + if (export) { 944 944 + sym_modname = export->mod; 945 945 + sym_orig_name = export->sym; 946 946 + sympos = 0; 947 947 + } else { 948 948 + sym_modname = find_modname(e); 949 949 + if (!sym_modname) 950 950 + return -1; 951 951 + 952 952 + sym_orig_name = patched_sym->twin->name; 953 953 + sympos = find_sympos(e->orig, patched_sym->twin); 954 954 + if (sympos == ULONG_MAX) 955 955 + return -1; 956 956 + } 957 957 + 958 958 + /* symbol format: .klp.sym.modname.sym_name,sympos */ 959 959 + if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_SYM_PREFIX "%s.%s,%ld", 960 960 + sym_modname, sym_orig_name, sympos)) 961 961 + return -1; 962 962 + 963 963 + klp_sym = find_symbol_by_name(e->out, sym_name); 964 964 + if (!klp_sym) { 965 965 + __dbg_indent("%s", sym_name); 966 966 + 967 967 + /* STB_WEAK: avoid modpost undefined symbol warnings */ 968 968 + klp_sym = elf_create_symbol(e->out, sym_name, NULL, 969 969 + STB_WEAK, patched_sym->type, 0, 0); 970 970 + if (!klp_sym) 971 971 + return -1; 972 972 + } 973 973 + 974 974 + /* 975 975 + * Create the __klp_relocs entry. This will be converted to an actual 976 976 + * KLP rela by "objtool klp post-link". 977 977 + * 978 978 + * This intermediate step is necessary to prevent corruption by the 979 979 + * linker, which doesn't know how to properly handle two rela sections 980 980 + * applying to the same base section. 981 981 + */ 982 982 + 983 983 + if (!klp_relocs) { 984 984 + klp_relocs = elf_create_section(e->out, KLP_RELOCS_SEC, 0, 985 985 + 0, SHT_PROGBITS, 8, SHF_ALLOC); 986 986 + if (!klp_relocs) 987 987 + return -1; 988 988 + } 989 989 + 990 990 + klp_reloc_off = sec_size(klp_relocs); 991 991 + memset(&klp_reloc, 0, sizeof(klp_reloc)); 992 992 + 993 993 + klp_reloc.type = reloc_type(patched_reloc); 994 994 + if (!elf_add_data(e->out, klp_relocs, &klp_reloc, sizeof(klp_reloc))) 995 995 + return -1; 996 996 + 997 997 + /* klp_reloc.offset */ 998 998 + if (!sec->sym && !elf_create_section_symbol(e->out, sec)) 999 999 + return -1; 1000 1000 + 1001 1001 + if (!elf_create_reloc(e->out, klp_relocs, 1002 1002 + klp_reloc_off + offsetof(struct klp_reloc, offset), 1003 1003 + sec->sym, offset, R_ABS64)) 1004 1004 + return -1; 1005 1005 + 1006 1006 + /* klp_reloc.sym */ 1007 1007 + if (!elf_create_reloc(e->out, klp_relocs, 1008 1008 + klp_reloc_off + offsetof(struct klp_reloc, sym), 1009 1009 + klp_sym, addend, R_ABS64)) 1010 1010 + return -1; 1011 1011 + 1012 1012 + return 0; 1013 1013 + } 1014 1014 + 1015 1015 + #define dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp) \ 1016 1016 + dbg_indent("%s+0x%lx: %s%s0x%lx [%s%s%s%s%s%s]", \ 1017 1017 + sec->name, offset, patched_sym->name, \ 1018 1018 + addend >= 0 ? "+" : "-", labs(addend), \ 1019 1019 + sym_type(patched_sym), \ 1020 1020 + patched_sym->type == STT_SECTION ? "" : " ", \ 1021 1021 + patched_sym->type == STT_SECTION ? "" : sym_bind(patched_sym), \ 1022 1022 + is_undef_sym(patched_sym) ? " UNDEF" : "", \ 1023 1023 + export ? " EXPORTED" : "", \ 1024 1024 + klp ? " KLP" : "") 1025 1025 + 1026 1026 + /* Copy a reloc and its symbol to the output object */ 1027 1027 + static int clone_reloc(struct elfs *e, struct reloc *patched_reloc, 1028 1028 + struct section *sec, unsigned long offset) 1029 1029 + { 1030 1030 + struct symbol *patched_sym = patched_reloc->sym; 1031 1031 + struct export *export = find_export(patched_sym); 1032 1032 + long addend = reloc_addend(patched_reloc); 1033 1033 + struct symbol *out_sym; 1034 1034 + bool klp; 1035 1035 + 1036 1036 + if (!is_reloc_allowed(patched_reloc)) { 1037 1037 + ERROR_FUNC(patched_reloc->sec->base, reloc_offset(patched_reloc), 1038 1038 + "missing symbol for reference to %s+%ld", 1039 1039 + patched_sym->name, addend); 1040 1040 + return -1; 1041 1041 + } 1042 1042 + 1043 1043 + klp = klp_reloc_needed(patched_reloc); 1044 1044 + 1045 1045 + dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp); 1046 1046 + 1047 1047 + if (klp) { 1048 1048 + if (clone_reloc_klp(e, patched_reloc, sec, offset, export)) 1049 1049 + return -1; 1050 1050 + 1051 1051 + return 0; 1052 1052 + } 1053 1053 + 1054 1054 + /* 1055 1055 + * Why !export sets 'data_too': 1056 1056 + * 1057 1057 + * Unexported non-klp symbols need to live in the patch module, 1058 1058 + * otherwise there will be unresolved symbols. Notably, this includes: 1059 1059 + * 1060 1060 + * - New functions/data 1061 1061 + * - String sections 1062 1062 + * - Special section entries 1063 1063 + * - Uncorrelated static local variables 1064 1064 + * - UBSAN sections 1065 1065 + */ 1066 1066 + out_sym = clone_symbol(e, patched_sym, patched_sym->included || !export); 1067 1067 + if (!out_sym) 1068 1068 + return -1; 1069 1069 + 1070 1070 + /* 1071 1071 + * For strings, all references use section symbols, thanks to 1072 1072 + * section_reference_needed(). clone_symbol() has cloned an empty 1073 1073 + * version of the string section. Now copy the string itself. 1074 1074 + */ 1075 1075 + if (is_string_sec(patched_sym->sec)) { 1076 1076 + const char *str = patched_sym->sec->data->d_buf + addend; 1077 1077 + 1078 1078 + __dbg_indent("\"%s\"", escape_str(str)); 1079 1079 + 1080 1080 + addend = elf_add_string(e->out, out_sym->sec, str); 1081 1081 + if (addend == -1) 1082 1082 + return -1; 1083 1083 + } 1084 1084 + 1085 1085 + if (!elf_create_reloc(e->out, sec, offset, out_sym, addend, 1086 1086 + reloc_type(patched_reloc))) 1087 1087 + return -1; 1088 1088 + 1089 1089 + return 0; 1090 1090 + } 1091 1091 + 1092 1092 + /* Copy all relocs needed for a symbol's contents */ 1093 1093 + static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym) 1094 1094 + { 1095 1095 + struct section *patched_rsec = patched_sym->sec->rsec; 1096 1096 + struct reloc *patched_reloc; 1097 1097 + unsigned long start, end; 1098 1098 + struct symbol *out_sym; 1099 1099 + 1100 1100 + out_sym = patched_sym->clone; 1101 1101 + if (!out_sym) { 1102 1102 + ERROR("no clone for %s", patched_sym->name); 1103 1103 + return -1; 1104 1104 + } 1105 1105 + 1106 1106 + if (!patched_rsec) 1107 1107 + return 0; 1108 1108 + 1109 1109 + if (!is_sec_sym(patched_sym) && !patched_sym->len) 1110 1110 + return 0; 1111 1111 + 1112 1112 + if (is_string_sec(patched_sym->sec)) 1113 1113 + return 0; 1114 1114 + 1115 1115 + if (is_sec_sym(patched_sym)) { 1116 1116 + start = 0; 1117 1117 + end = sec_size(patched_sym->sec); 1118 1118 + } else { 1119 1119 + start = patched_sym->offset; 1120 1120 + end = start + patched_sym->len; 1121 1121 + } 1122 1122 + 1123 1123 + for_each_reloc(patched_rsec, patched_reloc) { 1124 1124 + unsigned long offset; 1125 1125 + 1126 1126 + if (reloc_offset(patched_reloc) < start || 1127 1127 + reloc_offset(patched_reloc) >= end) 1128 1128 + continue; 1129 1129 + 1130 1130 + /* 1131 1131 + * Skip any reloc referencing .altinstr_aux. Its code is 1132 1132 + * always patched by alternatives. See ALTERNATIVE_TERNARY(). 1133 1133 + */ 1134 1134 + if (patched_reloc->sym->sec && 1135 1135 + !strcmp(patched_reloc->sym->sec->name, ".altinstr_aux")) 1136 1136 + continue; 1137 1137 + 1138 1138 + if (convert_reloc_sym(e->patched, patched_reloc)) { 1139 1139 + ERROR_FUNC(patched_rsec->base, reloc_offset(patched_reloc), 1140 1140 + "failed to convert reloc sym '%s' to its proper format", 1141 1141 + patched_reloc->sym->name); 1142 1142 + return -1; 1143 1143 + } 1144 1144 + 1145 1145 + offset = out_sym->offset + (reloc_offset(patched_reloc) - patched_sym->offset); 1146 1146 + 1147 1147 + if (clone_reloc(e, patched_reloc, out_sym->sec, offset)) 1148 1148 + return -1; 1149 1149 + } 1150 1150 + return 0; 1151 1151 + 1152 1152 + } 1153 1153 + 1154 1154 + static int create_fake_symbol(struct elf *elf, struct section *sec, 1155 1155 + unsigned long offset, size_t size) 1156 1156 + { 1157 1157 + char name[SYM_NAME_LEN]; 1158 1158 + unsigned int type; 1159 1159 + static int ctr; 1160 1160 + char *c; 1161 1161 + 1162 1162 + if (snprintf_check(name, SYM_NAME_LEN, "%s_%d", sec->name, ctr++)) 1163 1163 + return -1; 1164 1164 + 1165 1165 + for (c = name; *c; c++) 1166 1166 + if (*c == '.') 1167 1167 + *c = '_'; 1168 1168 + 1169 1169 + /* 1170 1170 + * STT_NOTYPE: Prevent objtool from validating .altinstr_replacement 1171 1171 + * while still allowing objdump to disassemble it. 1172 1172 + */ 1173 1173 + type = is_text_sec(sec) ? STT_NOTYPE : STT_OBJECT; 1174 1174 + return elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size) ? 0 : -1; 1175 1175 + } 1176 1176 + 1177 1177 + /* 1178 1178 + * Special sections (alternatives, etc) are basically arrays of structs. 1179 1179 + * For all the special sections, create a symbol for each struct entry. This 1180 1180 + * is a bit cumbersome, but it makes the extracting of the individual entries 1181 1181 + * much more straightforward. 1182 1182 + * 1183 1183 + * There are three ways to identify the entry sizes for a special section: 1184 1184 + * 1185 1185 + * 1) ELF section header sh_entsize: Ideally this would be used almost 1186 1186 + * everywhere. But unfortunately the toolchains make it difficult. The 1187 1187 + * assembler .[push]section directive syntax only takes entsize when 1188 1188 + * combined with SHF_MERGE. But Clang disallows combining SHF_MERGE with 1189 1189 + * SHF_WRITE. And some special sections do need to be writable. 1190 1190 + * 1191 1191 + * Another place this wouldn't work is .altinstr_replacement, whose entries 1192 1192 + * don't have a fixed size. 1193 1193 + * 1194 1194 + * 2) ANNOTATE_DATA_SPECIAL: This is a lightweight objtool annotation which 1195 1195 + * points to the beginning of each entry. The size of the entry is then 1196 1196 + * inferred by the location of the subsequent annotation (or end of 1197 1197 + * section). 1198 1198 + * 1199 1199 + * 3) Simple array of pointers: If the special section is just a basic array of 1200 1200 + * pointers, the entry size can be inferred by the number of relocations. 1201 1201 + * No annotations needed. 1202 1202 + * 1203 1203 + * Note I also tried to create per-entry symbols at the time of creation, in 1204 1204 + * the original [inline] asm. Unfortunately, creating uniquely named symbols 1205 1205 + * is trickier than one might think, especially with Clang inline asm. I 1206 1206 + * eventually just gave up trying to make that work, in favor of using 1207 1207 + * ANNOTATE_DATA_SPECIAL and creating the symbols here after the fact. 1208 1208 + */ 1209 1209 + static int create_fake_symbols(struct elf *elf) 1210 1210 + { 1211 1211 + struct section *sec; 1212 1212 + struct reloc *reloc; 1213 1213 + 1214 1214 + /* 1215 1215 + * 1) Make symbols for all the ANNOTATE_DATA_SPECIAL entries: 1216 1216 + */ 1217 1217 + 1218 1218 + sec = find_section_by_name(elf, ".discard.annotate_data"); 1219 1219 + if (!sec || !sec->rsec) 1220 1220 + return 0; 1221 1221 + 1222 1222 + for_each_reloc(sec->rsec, reloc) { 1223 1223 + unsigned long offset, size; 1224 1224 + struct reloc *next_reloc; 1225 1225 + 1226 1226 + if (annotype(elf, sec, reloc) != ANNOTYPE_DATA_SPECIAL) 1227 1227 + continue; 1228 1228 + 1229 1229 + offset = reloc_addend(reloc); 1230 1230 + 1231 1231 + size = 0; 1232 1232 + next_reloc = reloc; 1233 1233 + for_each_reloc_continue(sec->rsec, next_reloc) { 1234 1234 + if (annotype(elf, sec, next_reloc) != ANNOTYPE_DATA_SPECIAL || 1235 1235 + next_reloc->sym->sec != reloc->sym->sec) 1236 1236 + continue; 1237 1237 + 1238 1238 + size = reloc_addend(next_reloc) - offset; 1239 1239 + break; 1240 1240 + } 1241 1241 + 1242 1242 + if (!size) 1243 1243 + size = sec_size(reloc->sym->sec) - offset; 1244 1244 + 1245 1245 + if (create_fake_symbol(elf, reloc->sym->sec, offset, size)) 1246 1246 + return -1; 1247 1247 + } 1248 1248 + 1249 1249 + /* 1250 1250 + * 2) Make symbols for sh_entsize, and simple arrays of pointers: 1251 1251 + */ 1252 1252 + 1253 1253 + for_each_sec(elf, sec) { 1254 1254 + unsigned int entry_size; 1255 1255 + unsigned long offset; 1256 1256 + 1257 1257 + if (!is_special_section(sec) || find_symbol_by_offset(sec, 0)) 1258 1258 + continue; 1259 1259 + 1260 1260 + if (!sec->rsec) { 1261 1261 + ERROR("%s: missing special section relocations", sec->name); 1262 1262 + return -1; 1263 1263 + } 1264 1264 + 1265 1265 + entry_size = sec->sh.sh_entsize; 1266 1266 + if (!entry_size) { 1267 1267 + entry_size = arch_reloc_size(sec->rsec->relocs); 1268 1268 + if (sec_size(sec) != entry_size * sec_num_entries(sec->rsec)) { 1269 1269 + ERROR("%s: missing special section entsize or annotations", sec->name); 1270 1270 + return -1; 1271 1271 + } 1272 1272 + } 1273 1273 + 1274 1274 + for (offset = 0; offset < sec_size(sec); offset += entry_size) { 1275 1275 + if (create_fake_symbol(elf, sec, offset, entry_size)) 1276 1276 + return -1; 1277 1277 + } 1278 1278 + } 1279 1279 + 1280 1280 + return 0; 1281 1281 + } 1282 1282 + 1283 1283 + /* Keep a special section entry if it references an included function */ 1284 1284 + static bool should_keep_special_sym(struct elf *elf, struct symbol *sym) 1285 1285 + { 1286 1286 + struct reloc *reloc; 1287 1287 + 1288 1288 + if (is_sec_sym(sym) || !sym->sec->rsec) 1289 1289 + return false; 1290 1290 + 1291 1291 + sym_for_each_reloc(elf, sym, reloc) { 1292 1292 + if (convert_reloc_sym(elf, reloc)) 1293 1293 + continue; 1294 1294 + 1295 1295 + if (is_func_sym(reloc->sym) && reloc->sym->included) 1296 1296 + return true; 1297 1297 + } 1298 1298 + 1299 1299 + return false; 1300 1300 + } 1301 1301 + 1302 1302 + /* 1303 1303 + * Klp relocations aren't allowed for __jump_table and .static_call_sites if 1304 1304 + * the referenced symbol lives in a kernel module, because such klp relocs may 1305 1305 + * be applied after static branch/call init, resulting in code corruption. 1306 1306 + * 1307 1307 + * Validate a special section entry to avoid that. Note that an inert 1308 1308 + * tracepoint is harmless enough, in that case just skip the entry and print a 1309 1309 + * warning. Otherwise, return an error. 1310 1310 + * 1311 1311 + * This is only a temporary limitation which will be fixed when livepatch adds 1312 1312 + * support for submodules: fully self-contained modules which are embedded in 1313 1313 + * the top-level livepatch module's data and which can be loaded on demand when 1314 1314 + * their corresponding to-be-patched module gets loaded. Then klp relocs can 1315 1315 + * be retired. 1316 1316 + * 1317 1317 + * Return: 1318 1318 + * -1: error: validation failed 1319 1319 + * 1: warning: tracepoint skipped 1320 1320 + * 0: success 1321 1321 + */ 1322 1322 + static int validate_special_section_klp_reloc(struct elfs *e, struct symbol *sym) 1323 1323 + { 1324 1324 + bool static_branch = !strcmp(sym->sec->name, "__jump_table"); 1325 1325 + bool static_call = !strcmp(sym->sec->name, ".static_call_sites"); 1326 1326 + struct symbol *code_sym = NULL; 1327 1327 + unsigned long code_offset = 0; 1328 1328 + struct reloc *reloc; 1329 1329 + int ret = 0; 1330 1330 + 1331 1331 + if (!static_branch && !static_call) 1332 1332 + return 0; 1333 1333 + 1334 1334 + sym_for_each_reloc(e->patched, sym, reloc) { 1335 1335 + const char *sym_modname; 1336 1336 + struct export *export; 1337 1337 + 1338 1338 + /* Static branch/call keys are always STT_OBJECT */ 1339 1339 + if (reloc->sym->type != STT_OBJECT) { 1340 1340 + 1341 1341 + /* Save code location which can be printed below */ 1342 1342 + if (reloc->sym->type == STT_FUNC && !code_sym) { 1343 1343 + code_sym = reloc->sym; 1344 1344 + code_offset = reloc_addend(reloc); 1345 1345 + } 1346 1346 + 1347 1347 + continue; 1348 1348 + } 1349 1349 + 1350 1350 + if (!klp_reloc_needed(reloc)) 1351 1351 + continue; 1352 1352 + 1353 1353 + export = find_export(reloc->sym); 1354 1354 + if (export) { 1355 1355 + sym_modname = export->mod; 1356 1356 + } else { 1357 1357 + sym_modname = find_modname(e); 1358 1358 + if (!sym_modname) 1359 1359 + return -1; 1360 1360 + } 1361 1361 + 1362 1362 + /* vmlinux keys are ok */ 1363 1363 + if (!strcmp(sym_modname, "vmlinux")) 1364 1364 + continue; 1365 1365 + 1366 1366 + if (static_branch) { 1367 1367 + if (strstarts(reloc->sym->name, "__tracepoint_")) { 1368 1368 + WARN("%s: disabling unsupported tracepoint %s", 1369 1369 + code_sym->name, reloc->sym->name + 13); 1370 1370 + ret = 1; 1371 1371 + continue; 1372 1372 + } 1373 1373 + 1374 1374 + ERROR("%s+0x%lx: unsupported static branch key %s. Use static_key_enabled() instead", 1375 1375 + code_sym->name, code_offset, reloc->sym->name); 1376 1376 + return -1; 1377 1377 + } 1378 1378 + 1379 1379 + /* static call */ 1380 1380 + if (strstarts(reloc->sym->name, "__SCK__tp_func_")) { 1381 1381 + ret = 1; 1382 1382 + continue; 1383 1383 + } 1384 1384 + 1385 1385 + ERROR("%s()+0x%lx: unsupported static call key %s. Use KLP_STATIC_CALL() instead", 1386 1386 + code_sym->name, code_offset, reloc->sym->name); 1387 1387 + return -1; 1388 1388 + } 1389 1389 + 1390 1390 + return ret; 1391 1391 + } 1392 1392 + 1393 1393 + static int clone_special_section(struct elfs *e, struct section *patched_sec) 1394 1394 + { 1395 1395 + struct symbol *patched_sym; 1396 1396 + 1397 1397 + /* 1398 1398 + * Extract all special section symbols (and their dependencies) which 1399 1399 + * reference included functions. 1400 1400 + */ 1401 1401 + sec_for_each_sym(patched_sec, patched_sym) { 1402 1402 + int ret; 1403 1403 + 1404 1404 + if (!is_object_sym(patched_sym)) 1405 1405 + continue; 1406 1406 + 1407 1407 + if (!should_keep_special_sym(e->patched, patched_sym)) 1408 1408 + continue; 1409 1409 + 1410 1410 + ret = validate_special_section_klp_reloc(e, patched_sym); 1411 1411 + if (ret < 0) 1412 1412 + return -1; 1413 1413 + if (ret > 0) 1414 1414 + continue; 1415 1415 + 1416 1416 + if (!clone_symbol(e, patched_sym, true)) 1417 1417 + return -1; 1418 1418 + } 1419 1419 + 1420 1420 + return 0; 1421 1421 + } 1422 1422 + 1423 1423 + /* Extract only the needed bits from special sections */ 1424 1424 + static int clone_special_sections(struct elfs *e) 1425 1425 + { 1426 1426 + struct section *patched_sec; 1427 1427 + 1428 1428 + if (create_fake_symbols(e->patched)) 1429 1429 + return -1; 1430 1430 + 1431 1431 + for_each_sec(e->patched, patched_sec) { 1432 1432 + if (is_special_section(patched_sec)) { 1433 1433 + if (clone_special_section(e, patched_sec)) 1434 1434 + return -1; 1435 1435 + } 1436 1436 + } 1437 1437 + 1438 1438 + return 0; 1439 1439 + } 1440 1440 + 1441 1441 + /* 1442 1442 + * Create __klp_objects and __klp_funcs sections which are intermediate 1443 1443 + * sections provided as input to the patch module's init code for building the 1444 1444 + * klp_patch, klp_object and klp_func structs for the livepatch API. 1445 1445 + */ 1446 1446 + static int create_klp_sections(struct elfs *e) 1447 1447 + { 1448 1448 + size_t obj_size = sizeof(struct klp_object_ext); 1449 1449 + size_t func_size = sizeof(struct klp_func_ext); 1450 1450 + struct section *obj_sec, *funcs_sec, *str_sec; 1451 1451 + struct symbol *funcs_sym, *str_sym, *sym; 1452 1452 + char sym_name[SYM_NAME_LEN]; 1453 1453 + unsigned int nr_funcs = 0; 1454 1454 + const char *modname; 1455 1455 + void *obj_data; 1456 1456 + s64 addend; 1457 1457 + 1458 1458 + obj_sec = elf_create_section_pair(e->out, KLP_OBJECTS_SEC, obj_size, 0, 0); 1459 1459 + if (!obj_sec) 1460 1460 + return -1; 1461 1461 + 1462 1462 + funcs_sec = elf_create_section_pair(e->out, KLP_FUNCS_SEC, func_size, 0, 0); 1463 1463 + if (!funcs_sec) 1464 1464 + return -1; 1465 1465 + 1466 1466 + funcs_sym = elf_create_section_symbol(e->out, funcs_sec); 1467 1467 + if (!funcs_sym) 1468 1468 + return -1; 1469 1469 + 1470 1470 + str_sec = elf_create_section(e->out, KLP_STRINGS_SEC, 0, 0, 1471 1471 + SHT_PROGBITS, 1, 1472 1472 + SHF_ALLOC | SHF_STRINGS | SHF_MERGE); 1473 1473 + if (!str_sec) 1474 1474 + return -1; 1475 1475 + 1476 1476 + if (elf_add_string(e->out, str_sec, "") == -1) 1477 1477 + return -1; 1478 1478 + 1479 1479 + str_sym = elf_create_section_symbol(e->out, str_sec); 1480 1480 + if (!str_sym) 1481 1481 + return -1; 1482 1482 + 1483 1483 + /* allocate klp_object_ext */ 1484 1484 + obj_data = elf_add_data(e->out, obj_sec, NULL, obj_size); 1485 1485 + if (!obj_data) 1486 1486 + return -1; 1487 1487 + 1488 1488 + modname = find_modname(e); 1489 1489 + if (!modname) 1490 1490 + return -1; 1491 1491 + 1492 1492 + /* klp_object_ext.name */ 1493 1493 + if (strcmp(modname, "vmlinux")) { 1494 1494 + addend = elf_add_string(e->out, str_sec, modname); 1495 1495 + if (addend == -1) 1496 1496 + return -1; 1497 1497 + 1498 1498 + if (!elf_create_reloc(e->out, obj_sec, 1499 1499 + offsetof(struct klp_object_ext, name), 1500 1500 + str_sym, addend, R_ABS64)) 1501 1501 + return -1; 1502 1502 + } 1503 1503 + 1504 1504 + /* klp_object_ext.funcs */ 1505 1505 + if (!elf_create_reloc(e->out, obj_sec, offsetof(struct klp_object_ext, funcs), 1506 1506 + funcs_sym, 0, R_ABS64)) 1507 1507 + return -1; 1508 1508 + 1509 1509 + for_each_sym(e->out, sym) { 1510 1510 + unsigned long offset = nr_funcs * func_size; 1511 1511 + unsigned long sympos; 1512 1512 + void *func_data; 1513 1513 + 1514 1514 + if (!is_func_sym(sym) || sym->cold || !sym->clone || !sym->clone->changed) 1515 1515 + continue; 1516 1516 + 1517 1517 + /* allocate klp_func_ext */ 1518 1518 + func_data = elf_add_data(e->out, funcs_sec, NULL, func_size); 1519 1519 + if (!func_data) 1520 1520 + return -1; 1521 1521 + 1522 1522 + /* klp_func_ext.old_name */ 1523 1523 + addend = elf_add_string(e->out, str_sec, sym->clone->twin->name); 1524 1524 + if (addend == -1) 1525 1525 + return -1; 1526 1526 + 1527 1527 + if (!elf_create_reloc(e->out, funcs_sec, 1528 1528 + offset + offsetof(struct klp_func_ext, old_name), 1529 1529 + str_sym, addend, R_ABS64)) 1530 1530 + return -1; 1531 1531 + 1532 1532 + /* klp_func_ext.new_func */ 1533 1533 + if (!elf_create_reloc(e->out, funcs_sec, 1534 1534 + offset + offsetof(struct klp_func_ext, new_func), 1535 1535 + sym, 0, R_ABS64)) 1536 1536 + return -1; 1537 1537 + 1538 1538 + /* klp_func_ext.sympos */ 1539 1539 + BUILD_BUG_ON(sizeof(sympos) != sizeof_field(struct klp_func_ext, sympos)); 1540 1540 + sympos = find_sympos(e->orig, sym->clone->twin); 1541 1541 + if (sympos == ULONG_MAX) 1542 1542 + return -1; 1543 1543 + memcpy(func_data + offsetof(struct klp_func_ext, sympos), &sympos, 1544 1544 + sizeof_field(struct klp_func_ext, sympos)); 1545 1545 + 1546 1546 + nr_funcs++; 1547 1547 + } 1548 1548 + 1549 1549 + /* klp_object_ext.nr_funcs */ 1550 1550 + BUILD_BUG_ON(sizeof(nr_funcs) != sizeof_field(struct klp_object_ext, nr_funcs)); 1551 1551 + memcpy(obj_data + offsetof(struct klp_object_ext, nr_funcs), &nr_funcs, 1552 1552 + sizeof_field(struct klp_object_ext, nr_funcs)); 1553 1553 + 1554 1554 + /* 1555 1555 + * Find callback pointers created by KLP_PRE_PATCH_CALLBACK() and 1556 1556 + * friends, and add them to the klp object. 1557 1557 + */ 1558 1558 + 1559 1559 + if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_PRE_PATCH_PREFIX "%s", modname)) 1560 1560 + return -1; 1561 1561 + 1562 1562 + sym = find_symbol_by_name(e->out, sym_name); 1563 1563 + if (sym) { 1564 1564 + struct reloc *reloc; 1565 1565 + 1566 1566 + reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset); 1567 1567 + 1568 1568 + if (!elf_create_reloc(e->out, obj_sec, 1569 1569 + offsetof(struct klp_object_ext, callbacks) + 1570 1570 + offsetof(struct klp_callbacks, pre_patch), 1571 1571 + reloc->sym, reloc_addend(reloc), R_ABS64)) 1572 1572 + return -1; 1573 1573 + } 1574 1574 + 1575 1575 + if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_POST_PATCH_PREFIX "%s", modname)) 1576 1576 + return -1; 1577 1577 + 1578 1578 + sym = find_symbol_by_name(e->out, sym_name); 1579 1579 + if (sym) { 1580 1580 + struct reloc *reloc; 1581 1581 + 1582 1582 + reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset); 1583 1583 + 1584 1584 + if (!elf_create_reloc(e->out, obj_sec, 1585 1585 + offsetof(struct klp_object_ext, callbacks) + 1586 1586 + offsetof(struct klp_callbacks, post_patch), 1587 1587 + reloc->sym, reloc_addend(reloc), R_ABS64)) 1588 1588 + return -1; 1589 1589 + } 1590 1590 + 1591 1591 + if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_PRE_UNPATCH_PREFIX "%s", modname)) 1592 1592 + return -1; 1593 1593 + 1594 1594 + sym = find_symbol_by_name(e->out, sym_name); 1595 1595 + if (sym) { 1596 1596 + struct reloc *reloc; 1597 1597 + 1598 1598 + reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset); 1599 1599 + 1600 1600 + if (!elf_create_reloc(e->out, obj_sec, 1601 1601 + offsetof(struct klp_object_ext, callbacks) + 1602 1602 + offsetof(struct klp_callbacks, pre_unpatch), 1603 1603 + reloc->sym, reloc_addend(reloc), R_ABS64)) 1604 1604 + return -1; 1605 1605 + } 1606 1606 + 1607 1607 + if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_POST_UNPATCH_PREFIX "%s", modname)) 1608 1608 + return -1; 1609 1609 + 1610 1610 + sym = find_symbol_by_name(e->out, sym_name); 1611 1611 + if (sym) { 1612 1612 + struct reloc *reloc; 1613 1613 + 1614 1614 + reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset); 1615 1615 + 1616 1616 + if (!elf_create_reloc(e->out, obj_sec, 1617 1617 + offsetof(struct klp_object_ext, callbacks) + 1618 1618 + offsetof(struct klp_callbacks, post_unpatch), 1619 1619 + reloc->sym, reloc_addend(reloc), R_ABS64)) 1620 1620 + return -1; 1621 1621 + } 1622 1622 + 1623 1623 + return 0; 1624 1624 + } 1625 1625 + 1626 1626 + /* 1627 1627 + * Copy all .modinfo import_ns= tags to ensure all namespaced exported symbols 1628 1628 + * can be accessed via normal relocs. 1629 1629 + */ 1630 1630 + static int copy_import_ns(struct elfs *e) 1631 1631 + { 1632 1632 + struct section *patched_sec, *out_sec = NULL; 1633 1633 + char *import_ns, *data_end; 1634 1634 + 1635 1635 + patched_sec = find_section_by_name(e->patched, ".modinfo"); 1636 1636 + if (!patched_sec) 1637 1637 + return 0; 1638 1638 + 1639 1639 + import_ns = patched_sec->data->d_buf; 1640 1640 + if (!import_ns) 1641 1641 + return 0; 1642 1642 + 1643 1643 + for (data_end = import_ns + sec_size(patched_sec); 1644 1644 + import_ns < data_end; 1645 1645 + import_ns += strlen(import_ns) + 1) { 1646 1646 + 1647 1647 + import_ns = memmem(import_ns, data_end - import_ns, "import_ns=", 10); 1648 1648 + if (!import_ns) 1649 1649 + return 0; 1650 1650 + 1651 1651 + if (!out_sec) { 1652 1652 + out_sec = find_section_by_name(e->out, ".modinfo"); 1653 1653 + if (!out_sec) { 1654 1654 + out_sec = elf_create_section(e->out, ".modinfo", 0, 1655 1655 + patched_sec->sh.sh_entsize, 1656 1656 + patched_sec->sh.sh_type, 1657 1657 + patched_sec->sh.sh_addralign, 1658 1658 + patched_sec->sh.sh_flags); 1659 1659 + if (!out_sec) 1660 1660 + return -1; 1661 1661 + } 1662 1662 + } 1663 1663 + 1664 1664 + if (!elf_add_data(e->out, out_sec, import_ns, strlen(import_ns) + 1)) 1665 1665 + return -1; 1666 1666 + } 1667 1667 + 1668 1668 + return 0; 1669 1669 + } 1670 1670 + 1671 1671 + int cmd_klp_diff(int argc, const char **argv) 1672 1672 + { 1673 1673 + struct elfs e = {0}; 1674 1674 + 1675 1675 + argc = parse_options(argc, argv, klp_diff_options, klp_diff_usage, 0); 1676 1676 + if (argc != 3) 1677 1677 + usage_with_options(klp_diff_usage, klp_diff_options); 1678 1678 + 1679 1679 + objname = argv[0]; 1680 1680 + 1681 1681 + e.orig = elf_open_read(argv[0], O_RDONLY); 1682 1682 + e.patched = elf_open_read(argv[1], O_RDONLY); 1683 1683 + e.out = NULL; 1684 1684 + 1685 1685 + if (!e.orig || !e.patched) 1686 1686 + return -1; 1687 1687 + 1688 1688 + if (read_exports()) 1689 1689 + return -1; 1690 1690 + 1691 1691 + if (read_sym_checksums(e.orig)) 1692 1692 + return -1; 1693 1693 + 1694 1694 + if (read_sym_checksums(e.patched)) 1695 1695 + return -1; 1696 1696 + 1697 1697 + if (correlate_symbols(&e)) 1698 1698 + return -1; 1699 1699 + 1700 1700 + if (mark_changed_functions(&e)) 1701 1701 + return 0; 1702 1702 + 1703 1703 + e.out = elf_create_file(&e.orig->ehdr, argv[2]); 1704 1704 + if (!e.out) 1705 1705 + return -1; 1706 1706 + 1707 1707 + if (clone_included_functions(&e)) 1708 1708 + return -1; 1709 1709 + 1710 1710 + if (clone_special_sections(&e)) 1711 1711 + return -1; 1712 1712 + 1713 1713 + if (create_klp_sections(&e)) 1714 1714 + return -1; 1715 1715 + 1716 1716 + if (copy_import_ns(&e)) 1717 1717 + return -1; 1718 1718 + 1719 1719 + if (elf_write(e.out)) 1720 1720 + return -1; 1721 1721 + 1722 1722 + return elf_close(e.out); 1723 1723 + }

+168

tools/objtool/klp-post-link.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + /* 3 3 + * Read the intermediate KLP reloc/symbol representations created by klp diff 4 4 + * and convert them to the proper format required by livepatch. This needs to 5 5 + * run last to avoid linker wreckage. Linkers don't tend to handle the "two 6 6 + * rela sections for a single base section" case very well, nor do they like 7 7 + * SHN_LIVEPATCH. 8 8 + * 9 9 + * This is the final tool in the livepatch module generation pipeline: 10 10 + * 11 11 + * kernel builds -> objtool klp diff -> module link -> objtool klp post-link 12 12 + */ 13 13 + 14 14 + #include <fcntl.h> 15 15 + #include <gelf.h> 16 16 + #include <objtool/objtool.h> 17 17 + #include <objtool/warn.h> 18 18 + #include <objtool/klp.h> 19 19 + #include <objtool/util.h> 20 20 + #include <linux/livepatch_external.h> 21 21 + 22 22 + static int fix_klp_relocs(struct elf *elf) 23 23 + { 24 24 + struct section *symtab, *klp_relocs; 25 25 + 26 26 + klp_relocs = find_section_by_name(elf, KLP_RELOCS_SEC); 27 27 + if (!klp_relocs) 28 28 + return 0; 29 29 + 30 30 + symtab = find_section_by_name(elf, ".symtab"); 31 31 + if (!symtab) { 32 32 + ERROR("missing .symtab"); 33 33 + return -1; 34 34 + } 35 35 + 36 36 + for (int i = 0; i < sec_size(klp_relocs) / sizeof(struct klp_reloc); i++) { 37 37 + struct klp_reloc *klp_reloc; 38 38 + unsigned long klp_reloc_off; 39 39 + struct section *sec, *tmp, *klp_rsec; 40 40 + unsigned long offset; 41 41 + struct reloc *reloc; 42 42 + char sym_modname[64]; 43 43 + char rsec_name[SEC_NAME_LEN]; 44 44 + u64 addend; 45 45 + struct symbol *sym, *klp_sym; 46 46 + 47 47 + klp_reloc_off = i * sizeof(*klp_reloc); 48 48 + klp_reloc = klp_relocs->data->d_buf + klp_reloc_off; 49 49 + 50 50 + /* 51 51 + * Read __klp_relocs[i]: 52 52 + */ 53 53 + 54 54 + /* klp_reloc.sec_offset */ 55 55 + reloc = find_reloc_by_dest(elf, klp_relocs, 56 56 + klp_reloc_off + offsetof(struct klp_reloc, offset)); 57 57 + if (!reloc) { 58 58 + ERROR("malformed " KLP_RELOCS_SEC " section"); 59 59 + return -1; 60 60 + } 61 61 + 62 62 + sec = reloc->sym->sec; 63 63 + offset = reloc_addend(reloc); 64 64 + 65 65 + /* klp_reloc.sym */ 66 66 + reloc = find_reloc_by_dest(elf, klp_relocs, 67 67 + klp_reloc_off + offsetof(struct klp_reloc, sym)); 68 68 + if (!reloc) { 69 69 + ERROR("malformed " KLP_RELOCS_SEC " section"); 70 70 + return -1; 71 71 + } 72 72 + 73 73 + klp_sym = reloc->sym; 74 74 + addend = reloc_addend(reloc); 75 75 + 76 76 + /* symbol format: .klp.sym.modname.sym_name,sympos */ 77 77 + if (sscanf(klp_sym->name + strlen(KLP_SYM_PREFIX), "%55[^.]", sym_modname) != 1) 78 78 + ERROR("can't find modname in klp symbol '%s'", klp_sym->name); 79 79 + 80 80 + /* 81 81 + * Create the KLP rela: 82 82 + */ 83 83 + 84 84 + /* section format: .klp.rela.sec_objname.section_name */ 85 85 + if (snprintf_check(rsec_name, SEC_NAME_LEN, 86 86 + KLP_RELOC_SEC_PREFIX "%s.%s", 87 87 + sym_modname, sec->name)) 88 88 + return -1; 89 89 + 90 90 + klp_rsec = find_section_by_name(elf, rsec_name); 91 91 + if (!klp_rsec) { 92 92 + klp_rsec = elf_create_section(elf, rsec_name, 0, 93 93 + elf_rela_size(elf), 94 94 + SHT_RELA, elf_addr_size(elf), 95 95 + SHF_ALLOC | SHF_INFO_LINK | SHF_RELA_LIVEPATCH); 96 96 + if (!klp_rsec) 97 97 + return -1; 98 98 + 99 99 + klp_rsec->sh.sh_link = symtab->idx; 100 100 + klp_rsec->sh.sh_info = sec->idx; 101 101 + klp_rsec->base = sec; 102 102 + } 103 103 + 104 104 + tmp = sec->rsec; 105 105 + sec->rsec = klp_rsec; 106 106 + if (!elf_create_reloc(elf, sec, offset, klp_sym, addend, klp_reloc->type)) 107 107 + return -1; 108 108 + sec->rsec = tmp; 109 109 + 110 110 + /* 111 111 + * Fix up the corresponding KLP symbol: 112 112 + */ 113 113 + 114 114 + klp_sym->sym.st_shndx = SHN_LIVEPATCH; 115 115 + if (!gelf_update_sym(symtab->data, klp_sym->idx, &klp_sym->sym)) { 116 116 + ERROR_ELF("gelf_update_sym"); 117 117 + return -1; 118 118 + } 119 119 + 120 120 + /* 121 121 + * Disable the original non-KLP reloc by converting it to R_*_NONE: 122 122 + */ 123 123 + 124 124 + reloc = find_reloc_by_dest(elf, sec, offset); 125 125 + sym = reloc->sym; 126 126 + sym->sym.st_shndx = SHN_LIVEPATCH; 127 127 + set_reloc_type(elf, reloc, 0); 128 128 + if (!gelf_update_sym(symtab->data, sym->idx, &sym->sym)) { 129 129 + ERROR_ELF("gelf_update_sym"); 130 130 + return -1; 131 131 + } 132 132 + } 133 133 + 134 134 + return 0; 135 135 + } 136 136 + 137 137 + /* 138 138 + * This runs on the livepatch module after all other linking has been done. It 139 139 + * converts the intermediate __klp_relocs section into proper KLP relocs to be 140 140 + * processed by livepatch. This needs to run last to avoid linker wreckage. 141 141 + * Linkers don't tend to handle the "two rela sections for a single base 142 142 + * section" case very well, nor do they appreciate SHN_LIVEPATCH. 143 143 + */ 144 144 + int cmd_klp_post_link(int argc, const char **argv) 145 145 + { 146 146 + struct elf *elf; 147 147 + 148 148 + argc--; 149 149 + argv++; 150 150 + 151 151 + if (argc != 1) { 152 152 + fprintf(stderr, "%d\n", argc); 153 153 + fprintf(stderr, "usage: objtool link <file.ko>\n"); 154 154 + return -1; 155 155 + } 156 156 + 157 157 + elf = elf_open_read(argv[0], O_RDWR); 158 158 + if (!elf) 159 159 + return -1; 160 160 + 161 161 + if (fix_klp_relocs(elf)) 162 162 + return -1; 163 163 + 164 164 + if (elf_write(elf)) 165 165 + return -1; 166 166 + 167 167 + return elf_close(elf); 168 168 + }

+1

tools/objtool/noreturns.h

reviewed

··· 36 36 NORETURN(make_task_dead) 37 37 NORETURN(mpt_halt_firmware) 38 38 NORETURN(mwait_play_dead) 39 39 + NORETURN(native_play_dead) 39 40 NORETURN(nmi_panic_self_stop) 40 41 NORETURN(panic) 41 42 NORETURN(vpanic)

+41 -1

tools/objtool/objtool.c

reviewed

··· 16 16 #include <objtool/objtool.h> 17 17 #include <objtool/warn.h> 18 18 19 19 - bool help; 19 19 + bool debug; 20 20 + int indent; 20 21 21 22 static struct objtool_file file; 22 23 ··· 72 71 return 0; 73 72 } 74 73 74 74 + char *top_level_dir(const char *file) 75 75 + { 76 76 + ssize_t len, self_len, file_len; 77 77 + char self[PATH_MAX], *str; 78 78 + int i; 79 79 + 80 80 + len = readlink("/proc/self/exe", self, sizeof(self) - 1); 81 81 + if (len <= 0) 82 82 + return NULL; 83 83 + self[len] = '\0'; 84 84 + 85 85 + for (i = 0; i < 3; i++) { 86 86 + char *s = strrchr(self, '/'); 87 87 + if (!s) 88 88 + return NULL; 89 89 + *s = '\0'; 90 90 + } 91 91 + 92 92 + self_len = strlen(self); 93 93 + file_len = strlen(file); 94 94 + 95 95 + str = malloc(self_len + file_len + 2); 96 96 + if (!str) 97 97 + return NULL; 98 98 + 99 99 + memcpy(str, self, self_len); 100 100 + str[self_len] = '/'; 101 101 + strcpy(str + self_len + 1, file); 102 102 + 103 103 + return str; 104 104 + } 105 105 + 106 106 + 75 107 int main(int argc, const char **argv) 76 108 { 77 109 static const char *UNUSED = "OBJTOOL_NOT_IMPLEMENTED"; ··· 112 78 /* libsubcmd init */ 113 79 exec_cmd_init("objtool", UNUSED, UNUSED, UNUSED); 114 80 pager_init(UNUSED); 81 81 + 82 82 + if (argc > 1 && !strcmp(argv[1], "klp")) { 83 83 + argc--; 84 84 + argv++; 85 85 + return cmd_klp(argc, argv); 86 86 + } 115 87 116 88 return objtool_run(argc, argv); 117 89 }

-1

tools/objtool/orc_dump.c

reviewed

··· 8 8 #include <objtool/objtool.h> 9 9 #include <objtool/orc.h> 10 10 #include <objtool/warn.h> 11 11 - #include <objtool/endianness.h> 12 11 13 12 int orc_dump(const char *filename) 14 13 {

+6 -3

tools/objtool/orc_gen.c

reviewed

··· 12 12 #include <objtool/check.h> 13 13 #include <objtool/orc.h> 14 14 #include <objtool/warn.h> 15 15 - #include <objtool/endianness.h> 16 15 17 16 struct orc_list_entry { 18 17 struct list_head list; ··· 56 57 57 58 /* Build a deduplicated list of ORC entries: */ 58 59 INIT_LIST_HEAD(&orc_list); 59 59 - for_each_sec(file, sec) { 60 60 + for_each_sec(file->elf, sec) { 60 61 struct orc_entry orc, prev_orc = {0}; 61 62 struct instruction *insn; 62 63 bool empty = true; ··· 126 127 return -1; 127 128 } 128 129 orc_sec = elf_create_section(file->elf, ".orc_unwind", 129 129 - sizeof(struct orc_entry), nr); 130 130 + nr * sizeof(struct orc_entry), 131 131 + sizeof(struct orc_entry), 132 132 + SHT_PROGBITS, 133 133 + 1, 134 134 + SHF_ALLOC); 130 135 if (!orc_sec) 131 136 return -1; 132 137

+7 -9

tools/objtool/special.c

reviewed

··· 15 15 #include <objtool/builtin.h> 16 16 #include <objtool/special.h> 17 17 #include <objtool/warn.h> 18 18 - #include <objtool/endianness.h> 19 18 20 19 struct special_entry { 21 20 const char *sec; ··· 81 82 entry->orig_len); 82 83 alt->new_len = *(unsigned char *)(sec->data->d_buf + offset + 83 84 entry->new_len); 85 85 + alt->feature = *(unsigned int *)(sec->data->d_buf + offset + 86 86 + entry->feature); 84 87 } 85 88 86 89 orig_reloc = find_reloc_by_dest(elf, sec, offset + entry->orig); ··· 134 133 struct section *sec; 135 134 unsigned int nr_entries; 136 135 struct special_alt *alt; 137 137 - int idx, ret; 136 136 + int idx; 138 137 139 138 INIT_LIST_HEAD(alts); 140 139 ··· 143 142 if (!sec) 144 143 continue; 145 144 146 146 - if (sec->sh.sh_size % entry->size != 0) { 145 145 + if (sec_size(sec) % entry->size != 0) { 147 146 ERROR("%s size not a multiple of %d", sec->name, entry->size); 148 147 return -1; 149 148 } 150 149 151 151 - nr_entries = sec->sh.sh_size / entry->size; 150 150 + nr_entries = sec_size(sec) / entry->size; 152 151 153 152 for (idx = 0; idx < nr_entries; idx++) { 154 153 alt = malloc(sizeof(*alt)); ··· 158 157 } 159 158 memset(alt, 0, sizeof(*alt)); 160 159 161 161 - ret = get_alt_entry(elf, entry, sec, idx, alt); 162 162 - if (ret > 0) 163 163 - continue; 164 164 - if (ret < 0) 165 165 - return ret; 160 160 + if (get_alt_entry(elf, entry, sec, idx, alt)) 161 161 + return -1; 166 162 167 163 list_add_tail(&alt->list, alts); 168 164 }

+2

tools/objtool/sync-check.sh

reviewed

··· 16 16 arch/x86/include/asm/emulate_prefix.h 17 17 arch/x86/lib/x86-opcode-map.txt 18 18 arch/x86/tools/gen-insn-attr-x86.awk 19 19 + include/linux/interval_tree_generic.h 20 20 + include/linux/livepatch_external.h 19 21 include/linux/static_call_types.h 20 22 " 21 23

+203

tools/objtool/trace.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + /* 3 3 + * Copyright (c) 2025, Oracle and/or its affiliates. 4 4 + */ 5 5 + 6 6 + #include <objtool/trace.h> 7 7 + 8 8 + bool trace; 9 9 + int trace_depth; 10 10 + 11 11 + /* 12 12 + * Macros to trace CFI state attributes changes. 13 13 + */ 14 14 + 15 15 + #define TRACE_CFI_ATTR(attr, prev, next, fmt, ...) \ 16 16 + ({ \ 17 17 + if ((prev)->attr != (next)->attr) \ 18 18 + TRACE("%s=" fmt " ", #attr, __VA_ARGS__); \ 19 19 + }) 20 20 + 21 21 + #define TRACE_CFI_ATTR_BOOL(attr, prev, next) \ 22 22 + TRACE_CFI_ATTR(attr, prev, next, \ 23 23 + "%s", (next)->attr ? "true" : "false") 24 24 + 25 25 + #define TRACE_CFI_ATTR_NUM(attr, prev, next, fmt) \ 26 26 + TRACE_CFI_ATTR(attr, prev, next, fmt, (next)->attr) 27 27 + 28 28 + #define CFI_REG_NAME_MAXLEN 16 29 29 + 30 30 + /* 31 31 + * Return the name of a register. Note that the same static buffer 32 32 + * is returned if the name is dynamically generated. 33 33 + */ 34 34 + static const char *cfi_reg_name(unsigned int reg) 35 35 + { 36 36 + static char rname_buffer[CFI_REG_NAME_MAXLEN]; 37 37 + const char *rname; 38 38 + 39 39 + switch (reg) { 40 40 + case CFI_UNDEFINED: 41 41 + return "<undefined>"; 42 42 + case CFI_CFA: 43 43 + return "cfa"; 44 44 + case CFI_SP_INDIRECT: 45 45 + return "(sp)"; 46 46 + case CFI_BP_INDIRECT: 47 47 + return "(bp)"; 48 48 + } 49 49 + 50 50 + if (reg < CFI_NUM_REGS) { 51 51 + rname = arch_reg_name[reg]; 52 52 + if (rname) 53 53 + return rname; 54 54 + } 55 55 + 56 56 + if (snprintf(rname_buffer, CFI_REG_NAME_MAXLEN, "r%d", reg) == -1) 57 57 + return "<error>"; 58 58 + 59 59 + return (const char *)rname_buffer; 60 60 + } 61 61 + 62 62 + /* 63 63 + * Functions and macros to trace CFI registers changes. 64 64 + */ 65 65 + 66 66 + static void trace_cfi_reg(const char *prefix, int reg, const char *fmt, 67 67 + int base_prev, int offset_prev, 68 68 + int base_next, int offset_next) 69 69 + { 70 70 + char *rname; 71 71 + 72 72 + if (base_prev == base_next && offset_prev == offset_next) 73 73 + return; 74 74 + 75 75 + if (prefix) 76 76 + TRACE("%s:", prefix); 77 77 + 78 78 + if (base_next == CFI_UNDEFINED) { 79 79 + TRACE("%1$s=<undef> ", cfi_reg_name(reg)); 80 80 + } else { 81 81 + rname = strdup(cfi_reg_name(reg)); 82 82 + TRACE(fmt, rname, cfi_reg_name(base_next), offset_next); 83 83 + free(rname); 84 84 + } 85 85 + } 86 86 + 87 87 + static void trace_cfi_reg_val(const char *prefix, int reg, 88 88 + int base_prev, int offset_prev, 89 89 + int base_next, int offset_next) 90 90 + { 91 91 + trace_cfi_reg(prefix, reg, "%1$s=%2$s%3$+d ", 92 92 + base_prev, offset_prev, base_next, offset_next); 93 93 + } 94 94 + 95 95 + static void trace_cfi_reg_ref(const char *prefix, int reg, 96 96 + int base_prev, int offset_prev, 97 97 + int base_next, int offset_next) 98 98 + { 99 99 + trace_cfi_reg(prefix, reg, "%1$s=(%2$s%3$+d) ", 100 100 + base_prev, offset_prev, base_next, offset_next); 101 101 + } 102 102 + 103 103 + #define TRACE_CFI_REG_VAL(reg, prev, next) \ 104 104 + trace_cfi_reg_val(NULL, reg, prev.base, prev.offset, \ 105 105 + next.base, next.offset) 106 106 + 107 107 + #define TRACE_CFI_REG_REF(reg, prev, next) \ 108 108 + trace_cfi_reg_ref(NULL, reg, prev.base, prev.offset, \ 109 109 + next.base, next.offset) 110 110 + 111 111 + void trace_insn_state(struct instruction *insn, struct insn_state *sprev, 112 112 + struct insn_state *snext) 113 113 + { 114 114 + struct cfi_state *cprev, *cnext; 115 115 + int i; 116 116 + 117 117 + if (!memcmp(sprev, snext, sizeof(struct insn_state))) 118 118 + return; 119 119 + 120 120 + cprev = &sprev->cfi; 121 121 + cnext = &snext->cfi; 122 122 + 123 123 + disas_print_insn(stderr, objtool_disas_ctx, insn, 124 124 + trace_depth - 1, "state: "); 125 125 + 126 126 + /* print registers changes */ 127 127 + TRACE_CFI_REG_VAL(CFI_CFA, cprev->cfa, cnext->cfa); 128 128 + for (i = 0; i < CFI_NUM_REGS; i++) { 129 129 + TRACE_CFI_REG_VAL(i, cprev->vals[i], cnext->vals[i]); 130 130 + TRACE_CFI_REG_REF(i, cprev->regs[i], cnext->regs[i]); 131 131 + } 132 132 + 133 133 + /* print attributes changes */ 134 134 + TRACE_CFI_ATTR_NUM(stack_size, cprev, cnext, "%d"); 135 135 + TRACE_CFI_ATTR_BOOL(drap, cprev, cnext); 136 136 + if (cnext->drap) { 137 137 + trace_cfi_reg_val("drap", cnext->drap_reg, 138 138 + cprev->drap_reg, cprev->drap_offset, 139 139 + cnext->drap_reg, cnext->drap_offset); 140 140 + } 141 141 + TRACE_CFI_ATTR_BOOL(bp_scratch, cprev, cnext); 142 142 + TRACE_CFI_ATTR_NUM(instr, sprev, snext, "%d"); 143 143 + TRACE_CFI_ATTR_NUM(uaccess_stack, sprev, snext, "%u"); 144 144 + 145 145 + TRACE("\n"); 146 146 + 147 147 + insn->trace = 1; 148 148 + } 149 149 + 150 150 + void trace_alt_begin(struct instruction *orig_insn, struct alternative *alt, 151 151 + char *alt_name) 152 152 + { 153 153 + struct instruction *alt_insn; 154 154 + char suffix[2]; 155 155 + 156 156 + alt_insn = alt->insn; 157 157 + 158 158 + if (alt->type == ALT_TYPE_EX_TABLE) { 159 159 + /* 160 160 + * When there is an exception table then the instruction 161 161 + * at the original location is executed but it can cause 162 162 + * an exception. In that case, the execution will be 163 163 + * redirected to the alternative instruction. 164 164 + * 165 165 + * The instruction at the original location can have 166 166 + * instruction alternatives, so we just print the location 167 167 + * of the instruction that can cause the exception and 168 168 + * not the instruction itself. 169 169 + */ 170 170 + TRACE_ALT_INFO_NOADDR(orig_insn, "/ ", "%s for instruction at 0x%lx <%s+0x%lx>", 171 171 + alt_name, 172 172 + orig_insn->offset, orig_insn->sym->name, 173 173 + orig_insn->offset - orig_insn->sym->offset); 174 174 + } else { 175 175 + TRACE_ALT_INFO_NOADDR(orig_insn, "/ ", "%s", alt_name); 176 176 + } 177 177 + 178 178 + if (alt->type == ALT_TYPE_JUMP_TABLE) { 179 179 + /* 180 180 + * For a jump alternative, if the default instruction is 181 181 + * a NOP then it is replaced with the jmp instruction, 182 182 + * otherwise it is replaced with a NOP instruction. 183 183 + */ 184 184 + trace_depth++; 185 185 + if (orig_insn->type == INSN_NOP) { 186 186 + suffix[0] = (orig_insn->len == 5) ? 'q' : '\0'; 187 187 + TRACE_ADDR(orig_insn, "jmp%-3s %lx <%s+0x%lx>", suffix, 188 188 + alt_insn->offset, alt_insn->sym->name, 189 189 + alt_insn->offset - alt_insn->sym->offset); 190 190 + } else { 191 191 + TRACE_ADDR(orig_insn, "nop%d", orig_insn->len); 192 192 + trace_depth--; 193 193 + } 194 194 + } 195 195 + } 196 196 + 197 197 + void trace_alt_end(struct instruction *orig_insn, struct alternative *alt, 198 198 + char *alt_name) 199 199 + { 200 200 + if (alt->type == ALT_TYPE_JUMP_TABLE && orig_insn->type == INSN_NOP) 201 201 + trace_depth--; 202 202 + TRACE_ALT_INFO_NOADDR(orig_insn, "\\ ", "%s", alt_name); 203 203 + }

+7

tools/objtool/weak.c

reviewed

··· 8 8 #include <stdbool.h> 9 9 #include <errno.h> 10 10 #include <objtool/objtool.h> 11 11 + #include <objtool/arch.h> 12 12 + #include <objtool/builtin.h> 11 13 12 14 #define UNSUPPORTED(name) \ 13 15 ({ \ ··· 25 23 int __weak orc_create(struct objtool_file *file) 26 24 { 27 25 UNSUPPORTED("ORC"); 26 26 + } 27 27 + 28 28 + int __weak cmd_klp(int argc, const char **argv) 29 29 + { 30 30 + UNSUPPORTED("klp"); 28 31 }

+3 -3

tools/perf/Makefile.perf

reviewed

··· 234 234 # The fixdep build - we force fixdep tool to be built as 235 235 # the first target in the separate make session not to be 236 236 # disturbed by any parallel make jobs. Once fixdep is done 237 237 - # we issue the requested build with FIXDEP=1 variable. 237 237 + # we issue the requested build with FIXDEP_BUILT=1 variable. 238 238 # 239 239 # The fixdep build is disabled for $(NON_CONFIG_TARGETS) 240 240 # targets, because it's not necessary. 241 241 242 242 - ifdef FIXDEP 242 242 + ifdef FIXDEP_BUILT 243 243 force_fixdep := 0 244 244 else 245 245 force_fixdep := $(config) ··· 286 286 287 287 sub-make: fixdep 288 288 @./check-headers.sh 289 289 - $(Q)$(MAKE) FIXDEP=1 -f Makefile.perf $(goals) 289 289 + $(Q)$(MAKE) FIXDEP_BUILT=1 -f Makefile.perf $(goals) 290 290 291 291 else # force_fixdep 292 292