Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
KVM: x86: Remove fastops
No more FASTOPs, remove the remains.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Sean Christopherson <seanjc@google.com>
Link: https://lkml.kernel.org/r/20250714103440.751192860@infradead.org
+1
-171
+1
-171
arch/x86/kvm/emulate.c
+1
-171
arch/x86/kvm/emulate.c
···
167
167
#define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
168
168
#define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
169
169
#define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
170
-
#define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
171
170
#define NoWrite ((u64)1 << 45) /* No writeback */
172
171
#define SrcWrite ((u64)1 << 46) /* Write back src operand */
173
172
#define NoMod ((u64)1 << 47) /* Mod field is ignored */
···
202
203
const struct escape *esc;
203
204
const struct instr_dual *idual;
204
205
const struct mode_dual *mdual;
205
-
void (*fastop)(struct fastop *fake);
206
206
} u;
207
207
int (*check_perm)(struct x86_emulate_ctxt *ctxt);
208
208
};
···
380
382
case 4: __EM_ASM_3(op##l, eax, edx, cl); break; \
381
383
ON64(case 8: __EM_ASM_3(op##q, rax, rdx, cl); break;) \
382
384
EM_ASM_END
383
-
384
-
385
-
/*
386
-
* fastop functions have a special calling convention:
387
-
*
388
-
* dst: rax (in/out)
389
-
* src: rdx (in/out)
390
-
* src2: rcx (in)
391
-
* flags: rflags (in/out)
392
-
* ex: rsi (in:fastop pointer, out:zero if exception)
393
-
*
394
-
* Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
395
-
* different operand sizes can be reached by calculation, rather than a jump
396
-
* table (which would be bigger than the code).
397
-
*
398
-
* The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR
399
-
* and 1 for the straight line speculation INT3, leaves 7 bytes for the
400
-
* body of the function. Currently none is larger than 4.
401
-
*/
402
-
static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
403
-
404
-
#define FASTOP_SIZE 16
405
-
406
-
#define __FOP_FUNC(name) \
407
-
".align " __stringify(FASTOP_SIZE) " \n\t" \
408
-
".type " name ", @function \n\t" \
409
-
name ":\n\t" \
410
-
ASM_ENDBR \
411
-
IBT_NOSEAL(name)
412
-
413
-
#define FOP_FUNC(name) \
414
-
__FOP_FUNC(#name)
415
-
416
-
#define __FOP_RET(name) \
417
-
"11: " ASM_RET \
418
-
".size " name ", .-" name "\n\t"
419
-
420
-
#define FOP_RET(name) \
421
-
__FOP_RET(#name)
422
-
423
-
#define __FOP_START(op, align) \
424
-
extern void em_##op(struct fastop *fake); \
425
-
asm(".pushsection .text, \"ax\" \n\t" \
426
-
".global em_" #op " \n\t" \
427
-
".align " __stringify(align) " \n\t" \
428
-
"em_" #op ":\n\t"
429
-
430
-
#define FOP_START(op) __FOP_START(op, FASTOP_SIZE)
431
-
432
-
#define FOP_END \
433
-
".popsection")
434
-
435
-
#define __FOPNOP(name) \
436
-
__FOP_FUNC(name) \
437
-
__FOP_RET(name)
438
-
439
-
#define FOPNOP() \
440
-
__FOPNOP(__stringify(__UNIQUE_ID(nop)))
441
-
442
-
#define FOP1E(op, dst) \
443
-
__FOP_FUNC(#op "_" #dst) \
444
-
"10: " #op " %" #dst " \n\t" \
445
-
__FOP_RET(#op "_" #dst)
446
-
447
-
#define FOP1EEX(op, dst) \
448
-
FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi)
449
-
450
-
#define FASTOP1(op) \
451
-
FOP_START(op) \
452
-
FOP1E(op##b, al) \
453
-
FOP1E(op##w, ax) \
454
-
FOP1E(op##l, eax) \
455
-
ON64(FOP1E(op##q, rax)) \
456
-
FOP_END
457
-
458
-
/* 1-operand, using src2 (for MUL/DIV r/m) */
459
-
#define FASTOP1SRC2(op, name) \
460
-
FOP_START(name) \
461
-
FOP1E(op, cl) \
462
-
FOP1E(op, cx) \
463
-
FOP1E(op, ecx) \
464
-
ON64(FOP1E(op, rcx)) \
465
-
FOP_END
466
-
467
-
/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
468
-
#define FASTOP1SRC2EX(op, name) \
469
-
FOP_START(name) \
470
-
FOP1EEX(op, cl) \
471
-
FOP1EEX(op, cx) \
472
-
FOP1EEX(op, ecx) \
473
-
ON64(FOP1EEX(op, rcx)) \
474
-
FOP_END
475
-
476
-
#define FOP2E(op, dst, src) \
477
-
__FOP_FUNC(#op "_" #dst "_" #src) \
478
-
#op " %" #src ", %" #dst " \n\t" \
479
-
__FOP_RET(#op "_" #dst "_" #src)
480
-
481
-
#define FASTOP2(op) \
482
-
FOP_START(op) \
483
-
FOP2E(op##b, al, dl) \
484
-
FOP2E(op##w, ax, dx) \
485
-
FOP2E(op##l, eax, edx) \
486
-
ON64(FOP2E(op##q, rax, rdx)) \
487
-
FOP_END
488
-
489
-
/* 2 operand, word only */
490
-
#define FASTOP2W(op) \
491
-
FOP_START(op) \
492
-
FOPNOP() \
493
-
FOP2E(op##w, ax, dx) \
494
-
FOP2E(op##l, eax, edx) \
495
-
ON64(FOP2E(op##q, rax, rdx)) \
496
-
FOP_END
497
-
498
-
/* 2 operand, src is CL */
499
-
#define FASTOP2CL(op) \
500
-
FOP_START(op) \
501
-
FOP2E(op##b, al, cl) \
502
-
FOP2E(op##w, ax, cl) \
503
-
FOP2E(op##l, eax, cl) \
504
-
ON64(FOP2E(op##q, rax, cl)) \
505
-
FOP_END
506
-
507
-
/* 2 operand, src and dest are reversed */
508
-
#define FASTOP2R(op, name) \
509
-
FOP_START(name) \
510
-
FOP2E(op##b, dl, al) \
511
-
FOP2E(op##w, dx, ax) \
512
-
FOP2E(op##l, edx, eax) \
513
-
ON64(FOP2E(op##q, rdx, rax)) \
514
-
FOP_END
515
-
516
-
#define FOP3E(op, dst, src, src2) \
517
-
__FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
518
-
#op " %" #src2 ", %" #src ", %" #dst " \n\t"\
519
-
__FOP_RET(#op "_" #dst "_" #src "_" #src2)
520
-
521
-
/* 3-operand, word-only, src2=cl */
522
-
#define FASTOP3WCL(op) \
523
-
FOP_START(op) \
524
-
FOPNOP() \
525
-
FOP3E(op##w, ax, dx, cl) \
526
-
FOP3E(op##l, eax, edx, cl) \
527
-
ON64(FOP3E(op##q, rax, rdx, cl)) \
528
-
FOP_END
529
385
530
386
static int em_salc(struct x86_emulate_ctxt *ctxt)
531
387
{
···
3904
4052
#define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
3905
4053
#define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3906
4054
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3907
-
#define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3908
4055
#define II(_f, _e, _i) \
3909
4056
{ .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3910
4057
#define IIP(_f, _e, _i, _p) \
···
5009
5158
kvm_read_mmx_reg(op->addr.mm, &op->mm_val);
5010
5159
}
5011
5160
5012
-
static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5013
-
{
5014
-
ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5015
-
5016
-
if (!(ctxt->d & ByteOp))
5017
-
fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5018
-
5019
-
asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5020
-
: "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5021
-
[thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5022
-
: "c"(ctxt->src2.val));
5023
-
5024
-
ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5025
-
if (!fop) /* exception is returned in fop variable */
5026
-
return emulate_de(ctxt);
5027
-
return X86EMUL_CONTINUE;
5028
-
}
5029
-
5030
5161
void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5031
5162
{
5032
5163
/* Clear fields that are set conditionally but read without a guard. */
···
5173
5340
ctxt->eflags &= ~X86_EFLAGS_RF;
5174
5341
5175
5342
if (ctxt->execute) {
5176
-
if (ctxt->d & Fastop)
5177
-
rc = fastop(ctxt, ctxt->fop);
5178
-
else
5179
-
rc = ctxt->execute(ctxt);
5343
+
rc = ctxt->execute(ctxt);
5180
5344
if (rc != X86EMUL_CONTINUE)
5181
5345
goto done;
5182
5346
goto writeback;