Merge tag 'x86_alternatives_for_v6.3_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

+75 -55

arch/x86/include/asm/alternative.h

··· 6 6 #include <linux/stringify.h> 7 7 #include <asm/asm.h> 8 8 9 - #define ALTINSTR_FLAG_INV (1 << 15) 10 - #define ALT_NOT(feat) ((feat) | ALTINSTR_FLAG_INV) 9 + #define ALT_FLAGS_SHIFT 16 10 + 11 + #define ALT_FLAG_NOT BIT(0) 12 + #define ALT_NOT(feature) ((ALT_FLAG_NOT << ALT_FLAGS_SHIFT) | (feature)) 11 13 12 14 #ifndef __ASSEMBLY__ 13 15 ··· 61 59 ".long 999b - .\n\t" \ 62 60 ".popsection\n\t" 63 61 62 + /* 63 + * The patching flags are part of the upper bits of the @ft_flags parameter when 64 + * specifying them. The split is currently like this: 65 + * 66 + * [31... flags ...16][15... CPUID feature bit ...0] 67 + * 68 + * but since this is all hidden in the macros argument being split, those fields can be 69 + * extended in the future to fit in a u64 or however the need arises. 70 + */ 64 71 struct alt_instr { 65 72 s32 instr_offset; /* original instruction */ 66 73 s32 repl_offset; /* offset to replacement instruction */ 67 - u16 cpuid; /* cpuid bit set for replacement */ 74 + 75 + union { 76 + struct { 77 + u32 cpuid: 16; /* CPUID bit set for replacement */ 78 + u32 flags: 16; /* patching control flags */ 79 + }; 80 + u32 ft_flags; 81 + }; 82 + 68 83 u8 instrlen; /* length of original instruction */ 69 84 u8 replacementlen; /* length of new instruction */ 70 85 } __packed; ··· 201 182 " - (" alt_slen ")), 0x90\n" \ 202 183 alt_end_marker ":\n" 203 184 204 - #define ALTINSTR_ENTRY(feature, num) \ 185 + #define ALTINSTR_ENTRY(ft_flags, num) \ 205 186 " .long 661b - .\n" /* label */ \ 206 187 " .long " b_replacement(num)"f - .\n" /* new instruction */ \ 207 - " .word " __stringify(feature) "\n" /* feature bit */ \ 188 + " .4byte " __stringify(ft_flags) "\n" /* feature + flags */ \ 208 189 " .byte " alt_total_slen "\n" /* source len */ \ 209 190 " .byte " alt_rlen(num) "\n" /* replacement len */ 210 191 ··· 213 194 b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n" 214 195 215 196 /* alternative assembly primitive: */ 216 - #define ALTERNATIVE(oldinstr, newinstr, feature) \ 197 + #define ALTERNATIVE(oldinstr, newinstr, ft_flags) \ 217 198 OLDINSTR(oldinstr, 1) \ 218 199 ".pushsection .altinstructions,\"a\"\n" \ 219 - ALTINSTR_ENTRY(feature, 1) \ 200 + ALTINSTR_ENTRY(ft_flags, 1) \ 220 201 ".popsection\n" \ 221 202 ".pushsection .altinstr_replacement, \"ax\"\n" \ 222 203 ALTINSTR_REPLACEMENT(newinstr, 1) \ 223 204 ".popsection\n" 224 205 225 - #define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ 206 + #define ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \ 226 207 OLDINSTR_2(oldinstr, 1, 2) \ 227 208 ".pushsection .altinstructions,\"a\"\n" \ 228 - ALTINSTR_ENTRY(feature1, 1) \ 229 - ALTINSTR_ENTRY(feature2, 2) \ 209 + ALTINSTR_ENTRY(ft_flags1, 1) \ 210 + ALTINSTR_ENTRY(ft_flags2, 2) \ 230 211 ".popsection\n" \ 231 212 ".pushsection .altinstr_replacement, \"ax\"\n" \ 232 213 ALTINSTR_REPLACEMENT(newinstr1, 1) \ ··· 234 215 ".popsection\n" 235 216 236 217 /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */ 237 - #define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \ 218 + #define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ 238 219 ALTERNATIVE_2(oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \ 239 - newinstr_yes, feature) 220 + newinstr_yes, ft_flags) 240 221 241 - #define ALTERNATIVE_3(oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3, feat3) \ 242 - OLDINSTR_3(oldinsn, 1, 2, 3) \ 243 - ".pushsection .altinstructions,\"a\"\n" \ 244 - ALTINSTR_ENTRY(feat1, 1) \ 245 - ALTINSTR_ENTRY(feat2, 2) \ 246 - ALTINSTR_ENTRY(feat3, 3) \ 247 - ".popsection\n" \ 248 - ".pushsection .altinstr_replacement, \"ax\"\n" \ 249 - ALTINSTR_REPLACEMENT(newinsn1, 1) \ 250 - ALTINSTR_REPLACEMENT(newinsn2, 2) \ 251 - ALTINSTR_REPLACEMENT(newinsn3, 3) \ 222 + #define ALTERNATIVE_3(oldinsn, newinsn1, ft_flags1, newinsn2, ft_flags2, \ 223 + newinsn3, ft_flags3) \ 224 + OLDINSTR_3(oldinsn, 1, 2, 3) \ 225 + ".pushsection .altinstructions,\"a\"\n" \ 226 + ALTINSTR_ENTRY(ft_flags1, 1) \ 227 + ALTINSTR_ENTRY(ft_flags2, 2) \ 228 + ALTINSTR_ENTRY(ft_flags3, 3) \ 229 + ".popsection\n" \ 230 + ".pushsection .altinstr_replacement, \"ax\"\n" \ 231 + ALTINSTR_REPLACEMENT(newinsn1, 1) \ 232 + ALTINSTR_REPLACEMENT(newinsn2, 2) \ 233 + ALTINSTR_REPLACEMENT(newinsn3, 3) \ 252 234 ".popsection\n" 253 235 254 236 /* ··· 264 244 * For non barrier like inlines please define new variants 265 245 * without volatile and memory clobber. 266 246 */ 267 - #define alternative(oldinstr, newinstr, feature) \ 268 - asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory") 247 + #define alternative(oldinstr, newinstr, ft_flags) \ 248 + asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) : : : "memory") 269 249 270 - #define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \ 271 - asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory") 250 + #define alternative_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \ 251 + asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) ::: "memory") 272 252 273 - #define alternative_ternary(oldinstr, feature, newinstr_yes, newinstr_no) \ 274 - asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) ::: "memory") 253 + #define alternative_ternary(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ 254 + asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) ::: "memory") 275 255 276 256 /* 277 257 * Alternative inline assembly with input. ··· 281 261 * Argument numbers start with 1. 282 262 * Leaving an unused argument 0 to keep API compatibility. 283 263 */ 284 - #define alternative_input(oldinstr, newinstr, feature, input...) \ 285 - asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ 264 + #define alternative_input(oldinstr, newinstr, ft_flags, input...) \ 265 + asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \ 286 266 : : "i" (0), ## input) 287 267 288 268 /* ··· 293 273 * Otherwise, if CPU has feature1, newinstr1 is used. 294 274 * Otherwise, oldinstr is used. 295 275 */ 296 - #define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2, \ 297 - feature2, input...) \ 298 - asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, \ 299 - newinstr2, feature2) \ 276 + #define alternative_input_2(oldinstr, newinstr1, ft_flags1, newinstr2, \ 277 + ft_flags2, input...) \ 278 + asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, \ 279 + newinstr2, ft_flags2) \ 300 280 : : "i" (0), ## input) 301 281 302 282 /* Like alternative_input, but with a single output argument */ 303 - #define alternative_io(oldinstr, newinstr, feature, output, input...) \ 304 - asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ 283 + #define alternative_io(oldinstr, newinstr, ft_flags, output, input...) \ 284 + asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \ 305 285 : output : "i" (0), ## input) 306 286 307 287 /* Like alternative_io, but for replacing a direct call with another one. */ 308 - #define alternative_call(oldfunc, newfunc, feature, output, input...) \ 309 - asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \ 288 + #define alternative_call(oldfunc, newfunc, ft_flags, output, input...) \ 289 + asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", ft_flags) \ 310 290 : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input) 311 291 312 292 /* ··· 315 295 * Otherwise, if CPU has feature1, function1 is used. 316 296 * Otherwise, old function is used. 317 297 */ 318 - #define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \ 298 + #define alternative_call_2(oldfunc, newfunc1, ft_flags1, newfunc2, ft_flags2, \ 319 299 output, input...) \ 320 - asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\ 321 - "call %P[new2]", feature2) \ 300 + asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", ft_flags1,\ 301 + "call %P[new2]", ft_flags2) \ 322 302 : output, ASM_CALL_CONSTRAINT \ 323 303 : [old] "i" (oldfunc), [new1] "i" (newfunc1), \ 324 304 [new2] "i" (newfunc2), ## input) ··· 367 347 * enough information for the alternatives patching code to patch an 368 348 * instruction. See apply_alternatives(). 369 349 */ 370 - .macro altinstruction_entry orig alt feature orig_len alt_len 350 + .macro altinstr_entry orig alt ft_flags orig_len alt_len 371 351 .long \orig - . 372 352 .long \alt - . 373 - .word \feature 353 + .4byte \ft_flags 374 354 .byte \orig_len 375 355 .byte \alt_len 376 356 .endm ··· 381 361 * @newinstr. ".skip" directive takes care of proper instruction padding 382 362 * in case @newinstr is longer than @oldinstr. 383 363 */ 384 - .macro ALTERNATIVE oldinstr, newinstr, feature 364 + .macro ALTERNATIVE oldinstr, newinstr, ft_flags 385 365 140: 386 366 \oldinstr 387 367 141: ··· 389 369 142: 390 370 391 371 .pushsection .altinstructions,"a" 392 - altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f 372 + altinstr_entry 140b,143f,\ft_flags,142b-140b,144f-143f 393 373 .popsection 394 374 395 375 .pushsection .altinstr_replacement,"ax" ··· 419 399 * has @feature1, it replaces @oldinstr with @newinstr1. If CPU has 420 400 * @feature2, it replaces @oldinstr with @feature2. 421 401 */ 422 - .macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2 402 + .macro ALTERNATIVE_2 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2 423 403 140: 424 404 \oldinstr 425 405 141: ··· 428 408 142: 429 409 430 410 .pushsection .altinstructions,"a" 431 - altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f 432 - altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f 411 + altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f 412 + altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f 433 413 .popsection 434 414 435 415 .pushsection .altinstr_replacement,"ax" ··· 441 421 .popsection 442 422 .endm 443 423 444 - .macro ALTERNATIVE_3 oldinstr, newinstr1, feature1, newinstr2, feature2, newinstr3, feature3 424 + .macro ALTERNATIVE_3 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2, newinstr3, ft_flags3 445 425 140: 446 426 \oldinstr 447 427 141: ··· 450 430 142: 451 431 452 432 .pushsection .altinstructions,"a" 453 - altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f 454 - altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f 455 - altinstruction_entry 140b,145f,\feature3,142b-140b,146f-145f 433 + altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f 434 + altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f 435 + altinstr_entry 140b,145f,\ft_flags3,142b-140b,146f-145f 456 436 .popsection 457 437 458 438 .pushsection .altinstr_replacement,"ax" ··· 467 447 .endm 468 448 469 449 /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */ 470 - #define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \ 450 + #define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ 471 451 ALTERNATIVE_2 oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \ 472 - newinstr_yes, feature 452 + newinstr_yes, ft_flags 473 453 474 454 #endif /* __ASSEMBLY__ */ 475 455

+31

arch/x86/include/asm/text-patching.h

··· 184 184 unsigned long ip = int3_emulate_pop(regs); 185 185 int3_emulate_jmp(regs, ip); 186 186 } 187 + 188 + static __always_inline 189 + void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp) 190 + { 191 + static const unsigned long jcc_mask[6] = { 192 + [0] = X86_EFLAGS_OF, 193 + [1] = X86_EFLAGS_CF, 194 + [2] = X86_EFLAGS_ZF, 195 + [3] = X86_EFLAGS_CF | X86_EFLAGS_ZF, 196 + [4] = X86_EFLAGS_SF, 197 + [5] = X86_EFLAGS_PF, 198 + }; 199 + 200 + bool invert = cc & 1; 201 + bool match; 202 + 203 + if (cc < 0xc) { 204 + match = regs->flags & jcc_mask[cc >> 1]; 205 + } else { 206 + match = ((regs->flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^ 207 + ((regs->flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT); 208 + if (cc >= 0xe) 209 + match = match || (regs->flags & X86_EFLAGS_ZF); 210 + } 211 + 212 + if ((match && !invert) || (!match && invert)) 213 + ip += disp; 214 + 215 + int3_emulate_jmp(regs, ip); 216 + } 217 + 187 218 #endif /* !CONFIG_UML_X86 */ 188 219 189 220 #endif /* _ASM_X86_TEXT_PATCHING_H */

+53 -23

arch/x86/kernel/alternative.c

··· 282 282 */ 283 283 for (a = start; a < end; a++) { 284 284 int insn_buff_sz = 0; 285 - /* Mask away "NOT" flag bit for feature to test. */ 286 - u16 feature = a->cpuid & ~ALTINSTR_FLAG_INV; 287 285 288 286 instr = (u8 *)&a->instr_offset + a->instr_offset; 289 287 replacement = (u8 *)&a->repl_offset + a->repl_offset; 290 288 BUG_ON(a->instrlen > sizeof(insn_buff)); 291 - BUG_ON(feature >= (NCAPINTS + NBUGINTS) * 32); 289 + BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32); 292 290 293 291 /* 294 292 * Patch if either: 295 293 * - feature is present 296 - * - feature not present but ALTINSTR_FLAG_INV is set to mean, 294 + * - feature not present but ALT_FLAG_NOT is set to mean, 297 295 * patch if feature is *NOT* present. 298 296 */ 299 - if (!boot_cpu_has(feature) == !(a->cpuid & ALTINSTR_FLAG_INV)) 297 + if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) 300 298 goto next; 301 299 302 300 DPRINTK("feat: %s%d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d)", 303 - (a->cpuid & ALTINSTR_FLAG_INV) ? "!" : "", 304 - feature >> 5, 305 - feature & 0x1f, 301 + (a->flags & ALT_FLAG_NOT) ? "!" : "", 302 + a->cpuid >> 5, 303 + a->cpuid & 0x1f, 306 304 instr, instr, a->instrlen, 307 305 replacement, a->replacementlen); 308 306 ··· 336 338 next: 337 339 optimize_nops(instr, a->instrlen); 338 340 } 341 + } 342 + 343 + static inline bool is_jcc32(struct insn *insn) 344 + { 345 + /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */ 346 + return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80; 339 347 } 340 348 341 349 #if defined(CONFIG_RETPOLINE) && defined(CONFIG_OBJTOOL) ··· 380 376 bytes[i++] = modrm; 381 377 382 378 return i; 383 - } 384 - 385 - static inline bool is_jcc32(struct insn *insn) 386 - { 387 - /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */ 388 - return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80; 389 379 } 390 380 391 381 static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes) ··· 1770 1772 on_each_cpu(do_sync_core, NULL, 1); 1771 1773 } 1772 1774 1775 + /* 1776 + * NOTE: crazy scheme to allow patching Jcc.d32 but not increase the size of 1777 + * this thing. When len == 6 everything is prefixed with 0x0f and we map 1778 + * opcode to Jcc.d8, using len to distinguish. 1779 + */ 1773 1780 struct text_poke_loc { 1774 1781 /* addr := _stext + rel_addr */ 1775 1782 s32 rel_addr; ··· 1896 1893 int3_emulate_jmp(regs, (long)ip + tp->disp); 1897 1894 break; 1898 1895 1896 + case 0x70 ... 0x7f: /* Jcc */ 1897 + int3_emulate_jcc(regs, tp->opcode & 0xf, (long)ip, tp->disp); 1898 + break; 1899 + 1899 1900 default: 1900 1901 BUG(); 1901 1902 } ··· 1973 1966 * Second step: update all but the first byte of the patched range. 1974 1967 */ 1975 1968 for (do_sync = 0, i = 0; i < nr_entries; i++) { 1976 - u8 old[POKE_MAX_OPCODE_SIZE] = { tp[i].old, }; 1969 + u8 old[POKE_MAX_OPCODE_SIZE+1] = { tp[i].old, }; 1970 + u8 _new[POKE_MAX_OPCODE_SIZE+1]; 1971 + const u8 *new = tp[i].text; 1977 1972 int len = tp[i].len; 1978 1973 1979 1974 if (len - INT3_INSN_SIZE > 0) { 1980 1975 memcpy(old + INT3_INSN_SIZE, 1981 1976 text_poke_addr(&tp[i]) + INT3_INSN_SIZE, 1982 1977 len - INT3_INSN_SIZE); 1978 + 1979 + if (len == 6) { 1980 + _new[0] = 0x0f; 1981 + memcpy(_new + 1, new, 5); 1982 + new = _new; 1983 + } 1984 + 1983 1985 text_poke(text_poke_addr(&tp[i]) + INT3_INSN_SIZE, 1984 - (const char *)tp[i].text + INT3_INSN_SIZE, 1986 + new + INT3_INSN_SIZE, 1985 1987 len - INT3_INSN_SIZE); 1988 + 1986 1989 do_sync++; 1987 1990 } 1988 1991 ··· 2020 2003 * The old instruction is recorded so that the event can be 2021 2004 * processed forwards or backwards. 2022 2005 */ 2023 - perf_event_text_poke(text_poke_addr(&tp[i]), old, len, 2024 - tp[i].text, len); 2006 + perf_event_text_poke(text_poke_addr(&tp[i]), old, len, new, len); 2025 2007 } 2026 2008 2027 2009 if (do_sync) { ··· 2037 2021 * replacing opcode. 2038 2022 */ 2039 2023 for (do_sync = 0, i = 0; i < nr_entries; i++) { 2040 - if (tp[i].text[0] == INT3_INSN_OPCODE) 2024 + u8 byte = tp[i].text[0]; 2025 + 2026 + if (tp[i].len == 6) 2027 + byte = 0x0f; 2028 + 2029 + if (byte == INT3_INSN_OPCODE) 2041 2030 continue; 2042 2031 2043 - text_poke(text_poke_addr(&tp[i]), tp[i].text, INT3_INSN_SIZE); 2032 + text_poke(text_poke_addr(&tp[i]), &byte, INT3_INSN_SIZE); 2044 2033 do_sync++; 2045 2034 } 2046 2035 ··· 2063 2042 const void *opcode, size_t len, const void *emulate) 2064 2043 { 2065 2044 struct insn insn; 2066 - int ret, i; 2045 + int ret, i = 0; 2067 2046 2068 - memcpy((void *)tp->text, opcode, len); 2047 + if (len == 6) 2048 + i = 1; 2049 + memcpy((void *)tp->text, opcode+i, len-i); 2069 2050 if (!emulate) 2070 2051 emulate = opcode; 2071 2052 ··· 2077 2054 tp->rel_addr = addr - (void *)_stext; 2078 2055 tp->len = len; 2079 2056 tp->opcode = insn.opcode.bytes[0]; 2057 + 2058 + if (is_jcc32(&insn)) { 2059 + /* 2060 + * Map Jcc.d32 onto Jcc.d8 and use len to distinguish. 2061 + */ 2062 + tp->opcode = insn.opcode.bytes[1] - 0x10; 2063 + } 2080 2064 2081 2065 switch (tp->opcode) { 2082 2066 case RET_INSN_OPCODE: ··· 2101 2071 BUG_ON(len != insn.length); 2102 2072 } 2103 2073 2104 - 2105 2074 switch (tp->opcode) { 2106 2075 case INT3_INSN_OPCODE: 2107 2076 case RET_INSN_OPCODE: ··· 2109 2080 case CALL_INSN_OPCODE: 2110 2081 case JMP32_INSN_OPCODE: 2111 2082 case JMP8_INSN_OPCODE: 2083 + case 0x70 ... 0x7f: /* Jcc */ 2112 2084 tp->disp = insn.immediate.value; 2113 2085 break; 2114 2086

+8 -30

arch/x86/kernel/kprobes/core.c

··· 464 464 } 465 465 NOKPROBE_SYMBOL(kprobe_emulate_call); 466 466 467 - static nokprobe_inline 468 - void __kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs, bool cond) 467 + static void kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs) 469 468 { 470 469 unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 471 470 472 - if (cond) 473 - ip += p->ainsn.rel32; 471 + ip += p->ainsn.rel32; 474 472 int3_emulate_jmp(regs, ip); 475 - } 476 - 477 - static void kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs) 478 - { 479 - __kprobe_emulate_jmp(p, regs, true); 480 473 } 481 474 NOKPROBE_SYMBOL(kprobe_emulate_jmp); 482 475 483 - static const unsigned long jcc_mask[6] = { 484 - [0] = X86_EFLAGS_OF, 485 - [1] = X86_EFLAGS_CF, 486 - [2] = X86_EFLAGS_ZF, 487 - [3] = X86_EFLAGS_CF | X86_EFLAGS_ZF, 488 - [4] = X86_EFLAGS_SF, 489 - [5] = X86_EFLAGS_PF, 490 - }; 491 - 492 476 static void kprobe_emulate_jcc(struct kprobe *p, struct pt_regs *regs) 493 477 { 494 - bool invert = p->ainsn.jcc.type & 1; 495 - bool match; 478 + unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 496 479 497 - if (p->ainsn.jcc.type < 0xc) { 498 - match = regs->flags & jcc_mask[p->ainsn.jcc.type >> 1]; 499 - } else { 500 - match = ((regs->flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^ 501 - ((regs->flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT); 502 - if (p->ainsn.jcc.type >= 0xe) 503 - match = match || (regs->flags & X86_EFLAGS_ZF); 504 - } 505 - __kprobe_emulate_jmp(p, regs, (match && !invert) || (!match && invert)); 480 + int3_emulate_jcc(regs, p->ainsn.jcc.type, ip, p->ainsn.rel32); 506 481 } 507 482 NOKPROBE_SYMBOL(kprobe_emulate_jcc); 508 483 509 484 static void kprobe_emulate_loop(struct kprobe *p, struct pt_regs *regs) 510 485 { 486 + unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 511 487 bool match; 512 488 513 489 if (p->ainsn.loop.type != 3) { /* LOOP* */ ··· 511 535 else if (p->ainsn.loop.type == 1) /* LOOPE */ 512 536 match = match && (regs->flags & X86_EFLAGS_ZF); 513 537 514 - __kprobe_emulate_jmp(p, regs, match); 538 + if (match) 539 + ip += p->ainsn.rel32; 540 + int3_emulate_jmp(regs, ip); 515 541 } 516 542 NOKPROBE_SYMBOL(kprobe_emulate_loop); 517 543

+47 -3

arch/x86/kernel/static_call.c

··· 9 9 NOP = 1, /* site cond-call */ 10 10 JMP = 2, /* tramp / site tail-call */ 11 11 RET = 3, /* tramp / site cond-tail-call */ 12 + JCC = 4, 12 13 }; 13 14 14 15 /* ··· 26 25 27 26 static const u8 retinsn[] = { RET_INSN_OPCODE, 0xcc, 0xcc, 0xcc, 0xcc }; 28 27 28 + static u8 __is_Jcc(u8 *insn) /* Jcc.d32 */ 29 + { 30 + u8 ret = 0; 31 + 32 + if (insn[0] == 0x0f) { 33 + u8 tmp = insn[1]; 34 + if ((tmp & 0xf0) == 0x80) 35 + ret = tmp; 36 + } 37 + 38 + return ret; 39 + } 40 + 41 + extern void __static_call_return(void); 42 + 43 + asm (".global __static_call_return\n\t" 44 + ".type __static_call_return, @function\n\t" 45 + ASM_FUNC_ALIGN "\n\t" 46 + "__static_call_return:\n\t" 47 + ANNOTATE_NOENDBR 48 + ANNOTATE_RETPOLINE_SAFE 49 + "ret; int3\n\t" 50 + ".size __static_call_return, . - __static_call_return \n\t"); 51 + 29 52 static void __ref __static_call_transform(void *insn, enum insn_type type, 30 53 void *func, bool modinit) 31 54 { 32 55 const void *emulate = NULL; 33 56 int size = CALL_INSN_SIZE; 34 57 const void *code; 58 + u8 op, buf[6]; 59 + 60 + if ((type == JMP || type == RET) && (op = __is_Jcc(insn))) 61 + type = JCC; 35 62 36 63 switch (type) { 37 64 case CALL: ··· 86 57 else 87 58 code = &retinsn; 88 59 break; 60 + 61 + case JCC: 62 + if (!func) { 63 + func = __static_call_return; 64 + if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) 65 + func = x86_return_thunk; 66 + } 67 + 68 + buf[0] = 0x0f; 69 + __text_gen_insn(buf+1, op, insn+1, func, 5); 70 + code = buf; 71 + size = 6; 72 + 73 + break; 89 74 } 90 75 91 76 if (memcmp(insn, code, size) == 0) ··· 111 68 text_poke_bp(insn, code, size, emulate); 112 69 } 113 70 114 - static void __static_call_validate(void *insn, bool tail, bool tramp) 71 + static void __static_call_validate(u8 *insn, bool tail, bool tramp) 115 72 { 116 - u8 opcode = *(u8 *)insn; 73 + u8 opcode = insn[0]; 117 74 118 75 if (tramp && memcmp(insn+5, tramp_ud, 3)) { 119 76 pr_err("trampoline signature fail"); ··· 122 79 123 80 if (tail) { 124 81 if (opcode == JMP32_INSN_OPCODE || 125 - opcode == RET_INSN_OPCODE) 82 + opcode == RET_INSN_OPCODE || 83 + __is_Jcc(insn)) 126 84 return; 127 85 } else { 128 86 if (opcode == CALL_INSN_OPCODE ||

+3 -3

tools/objtool/arch/x86/include/arch/special.h

··· 11 11 #define JUMP_NEW_OFFSET 4 12 12 #define JUMP_KEY_OFFSET 8 13 13 14 - #define ALT_ENTRY_SIZE 12 14 + #define ALT_ENTRY_SIZE 14 15 15 #define ALT_ORIG_OFFSET 0 16 16 #define ALT_NEW_OFFSET 4 17 17 #define ALT_FEATURE_OFFSET 8 18 - #define ALT_ORIG_LEN_OFFSET 10 19 - #define ALT_NEW_LEN_OFFSET 11 18 + #define ALT_ORIG_LEN_OFFSET 12 19 + #define ALT_NEW_LEN_OFFSET 13 20 20 21 21 #endif /* _X86_ARCH_SPECIAL_H */

Configure Feed

Configure Feed