Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm
Pull kvm fixes from Paolo Bonzini:
"This is dominated by a couple large series for ARM and x86
respectively, but apart from that things are calm.
ARM:
- Large set of FP/SVE fixes for pKVM, addressing the fallout from the
per-CPU data rework and making sure that the host is not involved
in the FP/SVE switching any more
- Allow FEAT_BTI to be enabled with NV now that FEAT_PAUTH is
completely supported
- Fix for the respective priorities of Failed PAC, Illegal Execution
state and Instruction Abort exceptions
- Fix the handling of AArch32 instruction traps failing their
condition code, which was broken by the introduction of
ESR_EL2.ISS2
- Allow vcpus running in AArch32 state to be restored in System mode
- Fix AArch32 GPR restore that would lose the 64 bit state under some
conditions
RISC-V:
- No need to use mask when hart-index-bits is 0
- Fix incorrect reg_subtype labels in
kvm_riscv_vcpu_set_reg_isa_ext()
x86:
- Fixes and debugging help for the #VE sanity check.
Also disable it by default, even for CONFIG_DEBUG_KERNEL, because
it was found to trigger spuriously (most likely a processor erratum
as the exact symptoms vary by generation).
- Avoid WARN() when two NMIs arrive simultaneously during an
NMI-disabled situation (GIF=0 or interrupt shadow) when the
processor supports virtual NMI.
While generally KVM will not request an NMI window when virtual
NMIs are supported, in this case it *does* have to single-step over
the interrupt shadow or enable the STGI intercept, in order to
deliver the latched second NMI.
- Drop support for hand tuning APIC timer advancement from userspace.
Since we have adaptive tuning, and it has proved to work well, drop
the module parameter for manual configuration and with it a few
stupid bugs that it had"
* tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (32 commits)
KVM: x86/mmu: Don't save mmu_invalidate_seq after checking private attr
KVM: arm64: Ensure that SME controls are disabled in protected mode
KVM: arm64: Refactor CPACR trap bit setting/clearing to use ELx format
KVM: arm64: Consolidate initializing the host data's fpsimd_state/sve in pKVM
KVM: arm64: Eagerly restore host fpsimd/sve state in pKVM
KVM: arm64: Allocate memory mapped at hyp for host sve state in pKVM
KVM: arm64: Specialize handling of host fpsimd state on trap
KVM: arm64: Abstract set/clear of CPTR_EL2 bits behind helper
KVM: arm64: Fix prototype for __sve_save_state/__sve_restore_state
KVM: arm64: Reintroduce __sve_save_state
KVM: x86: Drop support for hand tuning APIC timer advancement from userspace
KVM: SEV-ES: Delegate LBR virtualization to the processor
KVM: SEV-ES: Disallow SEV-ES guests when X86_FEATURE_LBRV is absent
KVM: SEV-ES: Prevent MSR access post VMSA encryption
RISC-V: KVM: Fix incorrect reg_subtype labels in kvm_riscv_vcpu_set_reg_isa_ext function
RISC-V: KVM: No need to use mask when hart-index-bit is 0
KVM: arm64: nv: Expose BTI and CSV_frac to a guest hypervisor
KVM: arm64: nv: Fix relative priorities of exceptions generated by ERETAx
KVM: arm64: AArch32: Fix spurious trapping of conditional instructions
KVM: arm64: Allow AArch32 PSTATE.M to be restored as System mode
...
+559
-151
+3
-3
arch/arm64/include/asm/el2_setup.h
+3
-3
arch/arm64/include/asm/el2_setup.h
···
146
146
/* Coprocessor traps */
147
147
.macro __init_el2_cptr
148
148
__check_hvhe .LnVHE_\@, x1
149
-
mov x0, #(CPACR_EL1_FPEN_EL1EN | CPACR_EL1_FPEN_EL0EN)
149
+
mov x0, #CPACR_ELx_FPEN
150
150
msr cpacr_el1, x0
151
151
b .Lskip_set_cptr_\@
152
152
.LnVHE_\@:
···
277
277
278
278
// (h)VHE case
279
279
mrs x0, cpacr_el1 // Disable SVE traps
280
-
orr x0, x0, #(CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN)
280
+
orr x0, x0, #CPACR_ELx_ZEN
281
281
msr cpacr_el1, x0
282
282
b .Lskip_set_cptr_\@
283
283
···
298
298
299
299
// (h)VHE case
300
300
mrs x0, cpacr_el1 // Disable SME traps
301
-
orr x0, x0, #(CPACR_EL1_SMEN_EL0EN | CPACR_EL1_SMEN_EL1EN)
301
+
orr x0, x0, #CPACR_ELx_SMEN
302
302
msr cpacr_el1, x0
303
303
b .Lskip_set_cptr_sme_\@
304
304
+6
arch/arm64/include/asm/kvm_arm.h
+6
arch/arm64/include/asm/kvm_arm.h
···
305
305
GENMASK(19, 14) | \
306
306
BIT(11))
307
307
308
+
#define CPTR_VHE_EL2_RES0 (GENMASK(63, 32) | \
309
+
GENMASK(27, 26) | \
310
+
GENMASK(23, 22) | \
311
+
GENMASK(19, 18) | \
312
+
GENMASK(15, 0))
313
+
308
314
/* Hyp Debug Configuration Register bits */
309
315
#define MDCR_EL2_E2TB_MASK (UL(0x3))
310
316
#define MDCR_EL2_E2TB_SHIFT (UL(24))
+66
-5
arch/arm64/include/asm/kvm_emulate.h
+66
-5
arch/arm64/include/asm/kvm_emulate.h
···
557
557
vcpu_set_flag((v), e); \
558
558
} while (0)
559
559
560
+
#define __build_check_all_or_none(r, bits) \
561
+
BUILD_BUG_ON(((r) & (bits)) && ((r) & (bits)) != (bits))
562
+
563
+
#define __cpacr_to_cptr_clr(clr, set) \
564
+
({ \
565
+
u64 cptr = 0; \
566
+
\
567
+
if ((set) & CPACR_ELx_FPEN) \
568
+
cptr |= CPTR_EL2_TFP; \
569
+
if ((set) & CPACR_ELx_ZEN) \
570
+
cptr |= CPTR_EL2_TZ; \
571
+
if ((set) & CPACR_ELx_SMEN) \
572
+
cptr |= CPTR_EL2_TSM; \
573
+
if ((clr) & CPACR_ELx_TTA) \
574
+
cptr |= CPTR_EL2_TTA; \
575
+
if ((clr) & CPTR_EL2_TAM) \
576
+
cptr |= CPTR_EL2_TAM; \
577
+
if ((clr) & CPTR_EL2_TCPAC) \
578
+
cptr |= CPTR_EL2_TCPAC; \
579
+
\
580
+
cptr; \
581
+
})
582
+
583
+
#define __cpacr_to_cptr_set(clr, set) \
584
+
({ \
585
+
u64 cptr = 0; \
586
+
\
587
+
if ((clr) & CPACR_ELx_FPEN) \
588
+
cptr |= CPTR_EL2_TFP; \
589
+
if ((clr) & CPACR_ELx_ZEN) \
590
+
cptr |= CPTR_EL2_TZ; \
591
+
if ((clr) & CPACR_ELx_SMEN) \
592
+
cptr |= CPTR_EL2_TSM; \
593
+
if ((set) & CPACR_ELx_TTA) \
594
+
cptr |= CPTR_EL2_TTA; \
595
+
if ((set) & CPTR_EL2_TAM) \
596
+
cptr |= CPTR_EL2_TAM; \
597
+
if ((set) & CPTR_EL2_TCPAC) \
598
+
cptr |= CPTR_EL2_TCPAC; \
599
+
\
600
+
cptr; \
601
+
})
602
+
603
+
#define cpacr_clear_set(clr, set) \
604
+
do { \
605
+
BUILD_BUG_ON((set) & CPTR_VHE_EL2_RES0); \
606
+
BUILD_BUG_ON((clr) & CPACR_ELx_E0POE); \
607
+
__build_check_all_or_none((clr), CPACR_ELx_FPEN); \
608
+
__build_check_all_or_none((set), CPACR_ELx_FPEN); \
609
+
__build_check_all_or_none((clr), CPACR_ELx_ZEN); \
610
+
__build_check_all_or_none((set), CPACR_ELx_ZEN); \
611
+
__build_check_all_or_none((clr), CPACR_ELx_SMEN); \
612
+
__build_check_all_or_none((set), CPACR_ELx_SMEN); \
613
+
\
614
+
if (has_vhe() || has_hvhe()) \
615
+
sysreg_clear_set(cpacr_el1, clr, set); \
616
+
else \
617
+
sysreg_clear_set(cptr_el2, \
618
+
__cpacr_to_cptr_clr(clr, set), \
619
+
__cpacr_to_cptr_set(clr, set));\
620
+
} while (0)
621
+
560
622
static __always_inline void kvm_write_cptr_el2(u64 val)
561
623
{
562
624
if (has_vhe() || has_hvhe())
···
632
570
u64 val;
633
571
634
572
if (has_vhe()) {
635
-
val = (CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN |
636
-
CPACR_EL1_ZEN_EL1EN);
573
+
val = (CPACR_ELx_FPEN | CPACR_EL1_ZEN_EL1EN);
637
574
if (cpus_have_final_cap(ARM64_SME))
638
575
val |= CPACR_EL1_SMEN_EL1EN;
639
576
} else if (has_hvhe()) {
640
-
val = (CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN);
577
+
val = CPACR_ELx_FPEN;
641
578
642
579
if (!vcpu_has_sve(vcpu) || !guest_owns_fp_regs())
643
-
val |= CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN;
580
+
val |= CPACR_ELx_ZEN;
644
581
if (cpus_have_final_cap(ARM64_SME))
645
-
val |= CPACR_EL1_SMEN_EL1EN | CPACR_EL1_SMEN_EL0EN;
582
+
val |= CPACR_ELx_SMEN;
646
583
} else {
647
584
val = CPTR_NVHE_EL2_RES1;
648
585
+24
-1
arch/arm64/include/asm/kvm_host.h
+24
-1
arch/arm64/include/asm/kvm_host.h
···
76
76
DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
77
77
78
78
extern unsigned int __ro_after_init kvm_sve_max_vl;
79
+
extern unsigned int __ro_after_init kvm_host_sve_max_vl;
79
80
int __init kvm_arm_init_sve(void);
80
81
81
82
u32 __attribute_const__ kvm_target_cpu(void);
···
522
521
u64 *vncr_array;
523
522
};
524
523
524
+
struct cpu_sve_state {
525
+
__u64 zcr_el1;
526
+
527
+
/*
528
+
* Ordering is important since __sve_save_state/__sve_restore_state
529
+
* relies on it.
530
+
*/
531
+
__u32 fpsr;
532
+
__u32 fpcr;
533
+
534
+
/* Must be SVE_VQ_BYTES (128 bit) aligned. */
535
+
__u8 sve_regs[];
536
+
};
537
+
525
538
/*
526
539
* This structure is instantiated on a per-CPU basis, and contains
527
540
* data that is:
···
549
534
*/
550
535
struct kvm_host_data {
551
536
struct kvm_cpu_context host_ctxt;
552
-
struct user_fpsimd_state *fpsimd_state; /* hyp VA */
537
+
538
+
/*
539
+
* All pointers in this union are hyp VA.
540
+
* sve_state is only used in pKVM and if system_supports_sve().
541
+
*/
542
+
union {
543
+
struct user_fpsimd_state *fpsimd_state;
544
+
struct cpu_sve_state *sve_state;
545
+
};
553
546
554
547
/* Ownership of the FP regs */
555
548
enum {
+3
-1
arch/arm64/include/asm/kvm_hyp.h
+3
-1
arch/arm64/include/asm/kvm_hyp.h
···
111
111
112
112
void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
113
113
void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);
114
-
void __sve_restore_state(void *sve_pffr, u32 *fpsr);
114
+
void __sve_save_state(void *sve_pffr, u32 *fpsr, int save_ffr);
115
+
void __sve_restore_state(void *sve_pffr, u32 *fpsr, int restore_ffr);
115
116
116
117
u64 __guest_enter(struct kvm_vcpu *vcpu);
117
118
···
143
142
144
143
extern unsigned long kvm_nvhe_sym(__icache_flags);
145
144
extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits);
145
+
extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl);
146
146
147
147
#endif /* __ARM64_KVM_HYP_H__ */
+9
arch/arm64/include/asm/kvm_pkvm.h
+9
arch/arm64/include/asm/kvm_pkvm.h
···
128
128
return (2 * KVM_FFA_MBOX_NR_PAGES) + DIV_ROUND_UP(desc_max, PAGE_SIZE);
129
129
}
130
130
131
+
static inline size_t pkvm_host_sve_state_size(void)
132
+
{
133
+
if (!system_supports_sve())
134
+
return 0;
135
+
136
+
return size_add(sizeof(struct cpu_sve_state),
137
+
SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl)));
138
+
}
139
+
131
140
#endif /* __ARM64_KVM_PKVM_H__ */
+76
arch/arm64/kvm/arm.c
+76
arch/arm64/kvm/arm.c
···
1931
1931
return size ? get_order(size) : 0;
1932
1932
}
1933
1933
1934
+
static size_t pkvm_host_sve_state_order(void)
1935
+
{
1936
+
return get_order(pkvm_host_sve_state_size());
1937
+
}
1938
+
1934
1939
/* A lookup table holding the hypervisor VA for each vector slot */
1935
1940
static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
1936
1941
···
2315
2310
2316
2311
static void __init teardown_hyp_mode(void)
2317
2312
{
2313
+
bool free_sve = system_supports_sve() && is_protected_kvm_enabled();
2318
2314
int cpu;
2319
2315
2320
2316
free_hyp_pgds();
2321
2317
for_each_possible_cpu(cpu) {
2322
2318
free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
2323
2319
free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order());
2320
+
2321
+
if (free_sve) {
2322
+
struct cpu_sve_state *sve_state;
2323
+
2324
+
sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2325
+
free_pages((unsigned long) sve_state, pkvm_host_sve_state_order());
2326
+
}
2324
2327
}
2325
2328
}
2326
2329
···
2409
2396
free_hyp_pgds();
2410
2397
2411
2398
return 0;
2399
+
}
2400
+
2401
+
static int init_pkvm_host_sve_state(void)
2402
+
{
2403
+
int cpu;
2404
+
2405
+
if (!system_supports_sve())
2406
+
return 0;
2407
+
2408
+
/* Allocate pages for host sve state in protected mode. */
2409
+
for_each_possible_cpu(cpu) {
2410
+
struct page *page = alloc_pages(GFP_KERNEL, pkvm_host_sve_state_order());
2411
+
2412
+
if (!page)
2413
+
return -ENOMEM;
2414
+
2415
+
per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page);
2416
+
}
2417
+
2418
+
/*
2419
+
* Don't map the pages in hyp since these are only used in protected
2420
+
* mode, which will (re)create its own mapping when initialized.
2421
+
*/
2422
+
2423
+
return 0;
2424
+
}
2425
+
2426
+
/*
2427
+
* Finalizes the initialization of hyp mode, once everything else is initialized
2428
+
* and the initialziation process cannot fail.
2429
+
*/
2430
+
static void finalize_init_hyp_mode(void)
2431
+
{
2432
+
int cpu;
2433
+
2434
+
if (system_supports_sve() && is_protected_kvm_enabled()) {
2435
+
for_each_possible_cpu(cpu) {
2436
+
struct cpu_sve_state *sve_state;
2437
+
2438
+
sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2439
+
per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state =
2440
+
kern_hyp_va(sve_state);
2441
+
}
2442
+
} else {
2443
+
for_each_possible_cpu(cpu) {
2444
+
struct user_fpsimd_state *fpsimd_state;
2445
+
2446
+
fpsimd_state = &per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->host_ctxt.fp_regs;
2447
+
per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->fpsimd_state =
2448
+
kern_hyp_va(fpsimd_state);
2449
+
}
2450
+
}
2412
2451
}
2413
2452
2414
2453
static void pkvm_hyp_init_ptrauth(void)
···
2631
2566
goto out_err;
2632
2567
}
2633
2568
2569
+
err = init_pkvm_host_sve_state();
2570
+
if (err)
2571
+
goto out_err;
2572
+
2634
2573
err = kvm_hyp_init_protection(hyp_va_bits);
2635
2574
if (err) {
2636
2575
kvm_err("Failed to init hyp memory protection\n");
···
2798
2729
err = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2799
2730
if (err)
2800
2731
goto out_subs;
2732
+
2733
+
/*
2734
+
* This should be called after initialization is done and failure isn't
2735
+
* possible anymore.
2736
+
*/
2737
+
if (!in_hyp_mode)
2738
+
finalize_init_hyp_mode();
2801
2739
2802
2740
kvm_arm_initialised = true;
2803
2741
+11
-10
arch/arm64/kvm/emulate-nested.c
+11
-10
arch/arm64/kvm/emulate-nested.c
···
2181
2181
if (forward_traps(vcpu, HCR_NV))
2182
2182
return;
2183
2183
2184
+
spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2);
2185
+
spsr = kvm_check_illegal_exception_return(vcpu, spsr);
2186
+
2184
2187
/* Check for an ERETAx */
2185
2188
esr = kvm_vcpu_get_esr(vcpu);
2186
2189
if (esr_iss_is_eretax(esr) && !kvm_auth_eretax(vcpu, &elr)) {
2187
2190
/*
2188
-
* Oh no, ERETAx failed to authenticate. If we have
2189
-
* FPACCOMBINE, deliver an exception right away. If we
2190
-
* don't, then let the mangled ELR value trickle down the
2191
+
* Oh no, ERETAx failed to authenticate.
2192
+
*
2193
+
* If we have FPACCOMBINE and we don't have a pending
2194
+
* Illegal Execution State exception (which has priority
2195
+
* over FPAC), deliver an exception right away.
2196
+
*
2197
+
* Otherwise, let the mangled ELR value trickle down the
2191
2198
* ERET handling, and the guest will have a little surprise.
2192
2199
*/
2193
-
if (kvm_has_pauth(vcpu->kvm, FPACCOMBINE)) {
2200
+
if (kvm_has_pauth(vcpu->kvm, FPACCOMBINE) && !(spsr & PSR_IL_BIT)) {
2194
2201
esr &= ESR_ELx_ERET_ISS_ERETA;
2195
2202
esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_FPAC);
2196
2203
kvm_inject_nested_sync(vcpu, esr);
···
2208
2201
preempt_disable();
2209
2202
kvm_arch_vcpu_put(vcpu);
2210
2203
2211
-
spsr = __vcpu_sys_reg(vcpu, SPSR_EL2);
2212
-
spsr = kvm_check_illegal_exception_return(vcpu, spsr);
2213
2204
if (!esr_iss_is_eretax(esr))
2214
2205
elr = __vcpu_sys_reg(vcpu, ELR_EL2);
2215
2206
2216
2207
trace_kvm_nested_eret(vcpu, elr, spsr);
2217
2208
2218
-
/*
2219
-
* Note that the current exception level is always the virtual EL2,
2220
-
* since we set HCR_EL2.NV bit only when entering the virtual EL2.
2221
-
*/
2222
2209
*vcpu_pc(vcpu) = elr;
2223
2210
*vcpu_cpsr(vcpu) = spsr;
2224
2211
+8
-3
arch/arm64/kvm/fpsimd.c
+8
-3
arch/arm64/kvm/fpsimd.c
···
90
90
fpsimd_save_and_flush_cpu_state();
91
91
}
92
92
}
93
+
94
+
/*
95
+
* If normal guests gain SME support, maintain this behavior for pKVM
96
+
* guests, which don't support SME.
97
+
*/
98
+
WARN_ON(is_protected_kvm_enabled() && system_supports_sme() &&
99
+
read_sysreg_s(SYS_SVCR));
93
100
}
94
101
95
102
/*
···
168
161
if (has_vhe() && system_supports_sme()) {
169
162
/* Also restore EL0 state seen on entry */
170
163
if (vcpu_get_flag(vcpu, HOST_SME_ENABLED))
171
-
sysreg_clear_set(CPACR_EL1, 0,
172
-
CPACR_EL1_SMEN_EL0EN |
173
-
CPACR_EL1_SMEN_EL1EN);
164
+
sysreg_clear_set(CPACR_EL1, 0, CPACR_ELx_SMEN);
174
165
else
175
166
sysreg_clear_set(CPACR_EL1,
176
167
CPACR_EL1_SMEN_EL0EN,
+2
-1
arch/arm64/kvm/guest.c
+2
-1
arch/arm64/kvm/guest.c
···
251
251
case PSR_AA32_MODE_SVC:
252
252
case PSR_AA32_MODE_ABT:
253
253
case PSR_AA32_MODE_UND:
254
+
case PSR_AA32_MODE_SYS:
254
255
if (!vcpu_el1_is_32bit(vcpu))
255
256
return -EINVAL;
256
257
break;
···
277
276
if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) {
278
277
int i, nr_reg;
279
278
280
-
switch (*vcpu_cpsr(vcpu)) {
279
+
switch (*vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK) {
281
280
/*
282
281
* Either we are dealing with user mode, and only the
283
282
* first 15 registers (+ PC) must be narrowed to 32bit.
+16
-2
arch/arm64/kvm/hyp/aarch32.c
+16
-2
arch/arm64/kvm/hyp/aarch32.c
···
50
50
u32 cpsr_cond;
51
51
int cond;
52
52
53
-
/* Top two bits non-zero? Unconditional. */
54
-
if (kvm_vcpu_get_esr(vcpu) >> 30)
53
+
/*
54
+
* These are the exception classes that could fire with a
55
+
* conditional instruction.
56
+
*/
57
+
switch (kvm_vcpu_trap_get_class(vcpu)) {
58
+
case ESR_ELx_EC_CP15_32:
59
+
case ESR_ELx_EC_CP15_64:
60
+
case ESR_ELx_EC_CP14_MR:
61
+
case ESR_ELx_EC_CP14_LS:
62
+
case ESR_ELx_EC_FP_ASIMD:
63
+
case ESR_ELx_EC_CP10_ID:
64
+
case ESR_ELx_EC_CP14_64:
65
+
case ESR_ELx_EC_SVC32:
66
+
break;
67
+
default:
55
68
return true;
69
+
}
56
70
57
71
/* Is condition field valid? */
58
72
cond = kvm_vcpu_get_condition(vcpu);
+6
arch/arm64/kvm/hyp/fpsimd.S
+6
arch/arm64/kvm/hyp/fpsimd.S
+20
-16
arch/arm64/kvm/hyp/include/hyp/switch.h
+20
-16
arch/arm64/kvm/hyp/include/hyp/switch.h
···
316
316
{
317
317
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
318
318
__sve_restore_state(vcpu_sve_pffr(vcpu),
319
-
&vcpu->arch.ctxt.fp_regs.fpsr);
319
+
&vcpu->arch.ctxt.fp_regs.fpsr,
320
+
true);
320
321
write_sysreg_el1(__vcpu_sys_reg(vcpu, ZCR_EL1), SYS_ZCR);
321
322
}
323
+
324
+
static inline void __hyp_sve_save_host(void)
325
+
{
326
+
struct cpu_sve_state *sve_state = *host_data_ptr(sve_state);
327
+
328
+
sve_state->zcr_el1 = read_sysreg_el1(SYS_ZCR);
329
+
write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2);
330
+
__sve_save_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl),
331
+
&sve_state->fpsr,
332
+
true);
333
+
}
334
+
335
+
static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu);
322
336
323
337
/*
324
338
* We trap the first access to the FP/SIMD to save the host context and
···
344
330
{
345
331
bool sve_guest;
346
332
u8 esr_ec;
347
-
u64 reg;
348
333
349
334
if (!system_supports_fpsimd())
350
335
return false;
···
366
353
/* Valid trap. Switch the context: */
367
354
368
355
/* First disable enough traps to allow us to update the registers */
369
-
if (has_vhe() || has_hvhe()) {
370
-
reg = CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN;
371
-
if (sve_guest)
372
-
reg |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN;
373
-
374
-
sysreg_clear_set(cpacr_el1, 0, reg);
375
-
} else {
376
-
reg = CPTR_EL2_TFP;
377
-
if (sve_guest)
378
-
reg |= CPTR_EL2_TZ;
379
-
380
-
sysreg_clear_set(cptr_el2, reg, 0);
381
-
}
356
+
if (sve_guest || (is_protected_kvm_enabled() && system_supports_sve()))
357
+
cpacr_clear_set(0, CPACR_ELx_FPEN | CPACR_ELx_ZEN);
358
+
else
359
+
cpacr_clear_set(0, CPACR_ELx_FPEN);
382
360
isb();
383
361
384
362
/* Write out the host state if it's in the registers */
385
363
if (host_owns_fp_regs())
386
-
__fpsimd_save_state(*host_data_ptr(fpsimd_state));
364
+
kvm_hyp_save_fpsimd_host(vcpu);
387
365
388
366
/* Restore the guest state */
389
367
if (sve_guest)
-1
arch/arm64/kvm/hyp/include/nvhe/pkvm.h
-1
arch/arm64/kvm/hyp/include/nvhe/pkvm.h
+76
-8
arch/arm64/kvm/hyp/nvhe/hyp-main.c
+76
-8
arch/arm64/kvm/hyp/nvhe/hyp-main.c
···
23
23
24
24
void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt);
25
25
26
+
static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu)
27
+
{
28
+
__vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
29
+
/*
30
+
* On saving/restoring guest sve state, always use the maximum VL for
31
+
* the guest. The layout of the data when saving the sve state depends
32
+
* on the VL, so use a consistent (i.e., the maximum) guest VL.
33
+
*/
34
+
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
35
+
__sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr, true);
36
+
write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2);
37
+
}
38
+
39
+
static void __hyp_sve_restore_host(void)
40
+
{
41
+
struct cpu_sve_state *sve_state = *host_data_ptr(sve_state);
42
+
43
+
/*
44
+
* On saving/restoring host sve state, always use the maximum VL for
45
+
* the host. The layout of the data when saving the sve state depends
46
+
* on the VL, so use a consistent (i.e., the maximum) host VL.
47
+
*
48
+
* Setting ZCR_EL2 to ZCR_ELx_LEN_MASK sets the effective length
49
+
* supported by the system (or limited at EL3).
50
+
*/
51
+
write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2);
52
+
__sve_restore_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl),
53
+
&sve_state->fpsr,
54
+
true);
55
+
write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR);
56
+
}
57
+
58
+
static void fpsimd_sve_flush(void)
59
+
{
60
+
*host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED;
61
+
}
62
+
63
+
static void fpsimd_sve_sync(struct kvm_vcpu *vcpu)
64
+
{
65
+
if (!guest_owns_fp_regs())
66
+
return;
67
+
68
+
cpacr_clear_set(0, CPACR_ELx_FPEN | CPACR_ELx_ZEN);
69
+
isb();
70
+
71
+
if (vcpu_has_sve(vcpu))
72
+
__hyp_sve_save_guest(vcpu);
73
+
else
74
+
__fpsimd_save_state(&vcpu->arch.ctxt.fp_regs);
75
+
76
+
if (system_supports_sve())
77
+
__hyp_sve_restore_host();
78
+
else
79
+
__fpsimd_restore_state(*host_data_ptr(fpsimd_state));
80
+
81
+
*host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED;
82
+
}
83
+
26
84
static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
27
85
{
28
86
struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
29
87
88
+
fpsimd_sve_flush();
89
+
30
90
hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt;
31
91
32
92
hyp_vcpu->vcpu.arch.sve_state = kern_hyp_va(host_vcpu->arch.sve_state);
33
-
hyp_vcpu->vcpu.arch.sve_max_vl = host_vcpu->arch.sve_max_vl;
93
+
/* Limit guest vector length to the maximum supported by the host. */
94
+
hyp_vcpu->vcpu.arch.sve_max_vl = min(host_vcpu->arch.sve_max_vl, kvm_host_sve_max_vl);
34
95
35
96
hyp_vcpu->vcpu.arch.hw_mmu = host_vcpu->arch.hw_mmu;
36
97
37
98
hyp_vcpu->vcpu.arch.hcr_el2 = host_vcpu->arch.hcr_el2;
38
99
hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2;
39
-
hyp_vcpu->vcpu.arch.cptr_el2 = host_vcpu->arch.cptr_el2;
40
100
41
101
hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags;
42
102
···
114
54
struct vgic_v3_cpu_if *host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3;
115
55
unsigned int i;
116
56
57
+
fpsimd_sve_sync(&hyp_vcpu->vcpu);
58
+
117
59
host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt;
118
60
119
61
host_vcpu->arch.hcr_el2 = hyp_vcpu->vcpu.arch.hcr_el2;
120
-
host_vcpu->arch.cptr_el2 = hyp_vcpu->vcpu.arch.cptr_el2;
121
62
122
63
host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault;
123
64
···
139
78
if (unlikely(is_protected_kvm_enabled())) {
140
79
struct pkvm_hyp_vcpu *hyp_vcpu;
141
80
struct kvm *host_kvm;
81
+
82
+
/*
83
+
* KVM (and pKVM) doesn't support SME guests for now, and
84
+
* ensures that SME features aren't enabled in pstate when
85
+
* loading a vcpu. Therefore, if SME features enabled the host
86
+
* is misbehaving.
87
+
*/
88
+
if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) {
89
+
ret = -EINVAL;
90
+
goto out;
91
+
}
142
92
143
93
host_kvm = kern_hyp_va(host_vcpu->kvm);
144
94
hyp_vcpu = pkvm_load_hyp_vcpu(host_kvm->arch.pkvm.handle,
···
477
405
handle_host_smc(host_ctxt);
478
406
break;
479
407
case ESR_ELx_EC_SVE:
480
-
if (has_hvhe())
481
-
sysreg_clear_set(cpacr_el1, 0, (CPACR_EL1_ZEN_EL1EN |
482
-
CPACR_EL1_ZEN_EL0EN));
483
-
else
484
-
sysreg_clear_set(cptr_el2, CPTR_EL2_TZ, 0);
408
+
cpacr_clear_set(0, CPACR_ELx_ZEN);
485
409
isb();
486
410
sve_cond_update_zcr_vq(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2);
487
411
break;
+5
-12
arch/arm64/kvm/hyp/nvhe/pkvm.c
+5
-12
arch/arm64/kvm/hyp/nvhe/pkvm.c
···
18
18
/* Used by kvm_get_vttbr(). */
19
19
unsigned int kvm_arm_vmid_bits;
20
20
21
+
unsigned int kvm_host_sve_max_vl;
22
+
21
23
/*
22
24
* Set trap register values based on features in ID_AA64PFR0.
23
25
*/
···
65
63
/* Trap SVE */
66
64
if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), feature_ids)) {
67
65
if (has_hvhe())
68
-
cptr_clear |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN;
66
+
cptr_clear |= CPACR_ELx_ZEN;
69
67
else
70
68
cptr_set |= CPTR_EL2_TZ;
71
69
}
···
247
245
{
248
246
WARN_ON(vm_table);
249
247
vm_table = tbl;
250
-
}
251
-
252
-
void pkvm_host_fpsimd_state_init(void)
253
-
{
254
-
unsigned long i;
255
-
256
-
for (i = 0; i < hyp_nr_cpus; i++) {
257
-
struct kvm_host_data *host_data = per_cpu_ptr(&kvm_host_data, i);
258
-
259
-
host_data->fpsimd_state = &host_data->host_ctxt.fp_regs;
260
-
}
261
248
}
262
249
263
250
/*
···
576
585
577
586
if (ret)
578
587
unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu));
588
+
589
+
hyp_vcpu->vcpu.arch.cptr_el2 = kvm_get_reset_cptr_el2(&hyp_vcpu->vcpu);
579
590
580
591
return ret;
581
592
}
+24
-1
arch/arm64/kvm/hyp/nvhe/setup.c
+24
-1
arch/arm64/kvm/hyp/nvhe/setup.c
···
67
67
return 0;
68
68
}
69
69
70
+
static int pkvm_create_host_sve_mappings(void)
71
+
{
72
+
void *start, *end;
73
+
int ret, i;
74
+
75
+
if (!system_supports_sve())
76
+
return 0;
77
+
78
+
for (i = 0; i < hyp_nr_cpus; i++) {
79
+
struct kvm_host_data *host_data = per_cpu_ptr(&kvm_host_data, i);
80
+
struct cpu_sve_state *sve_state = host_data->sve_state;
81
+
82
+
start = kern_hyp_va(sve_state);
83
+
end = start + PAGE_ALIGN(pkvm_host_sve_state_size());
84
+
ret = pkvm_create_mappings(start, end, PAGE_HYP);
85
+
if (ret)
86
+
return ret;
87
+
}
88
+
89
+
return 0;
90
+
}
91
+
70
92
static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size,
71
93
unsigned long *per_cpu_base,
72
94
u32 hyp_va_bits)
···
146
124
if (ret)
147
125
return ret;
148
126
}
127
+
128
+
pkvm_create_host_sve_mappings();
149
129
150
130
/*
151
131
* Map the host sections RO in the hypervisor, but transfer the
···
324
300
goto out;
325
301
326
302
pkvm_hyp_vm_table_init(vm_table_base);
327
-
pkvm_host_fpsimd_state_init();
328
303
out:
329
304
/*
330
305
* We tail-called to here from handle___pkvm_init() and will not return,
+21
-3
arch/arm64/kvm/hyp/nvhe/switch.c
+21
-3
arch/arm64/kvm/hyp/nvhe/switch.c
···
48
48
val |= has_hvhe() ? CPACR_EL1_TTA : CPTR_EL2_TTA;
49
49
if (cpus_have_final_cap(ARM64_SME)) {
50
50
if (has_hvhe())
51
-
val &= ~(CPACR_EL1_SMEN_EL1EN | CPACR_EL1_SMEN_EL0EN);
51
+
val &= ~CPACR_ELx_SMEN;
52
52
else
53
53
val |= CPTR_EL2_TSM;
54
54
}
55
55
56
56
if (!guest_owns_fp_regs()) {
57
57
if (has_hvhe())
58
-
val &= ~(CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN |
59
-
CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN);
58
+
val &= ~(CPACR_ELx_FPEN | CPACR_ELx_ZEN);
60
59
else
61
60
val |= CPTR_EL2_TFP | CPTR_EL2_TZ;
62
61
···
179
180
*/
180
181
return (kvm_hyp_handle_sysreg(vcpu, exit_code) ||
181
182
kvm_handle_pvm_sysreg(vcpu, exit_code));
183
+
}
184
+
185
+
static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu)
186
+
{
187
+
/*
188
+
* Non-protected kvm relies on the host restoring its sve state.
189
+
* Protected kvm restores the host's sve state as not to reveal that
190
+
* fpsimd was used by a guest nor leak upper sve bits.
191
+
*/
192
+
if (unlikely(is_protected_kvm_enabled() && system_supports_sve())) {
193
+
__hyp_sve_save_host();
194
+
195
+
/* Re-enable SVE traps if not supported for the guest vcpu. */
196
+
if (!vcpu_has_sve(vcpu))
197
+
cpacr_clear_set(CPACR_ELx_ZEN, 0);
198
+
199
+
} else {
200
+
__fpsimd_save_state(*host_data_ptr(fpsimd_state));
201
+
}
182
202
}
183
203
184
204
static const exit_handler_fn hyp_exit_handlers[] = {
+8
-4
arch/arm64/kvm/hyp/vhe/switch.c
+8
-4
arch/arm64/kvm/hyp/vhe/switch.c
···
93
93
94
94
val = read_sysreg(cpacr_el1);
95
95
val |= CPACR_ELx_TTA;
96
-
val &= ~(CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN |
97
-
CPACR_EL1_SMEN_EL0EN | CPACR_EL1_SMEN_EL1EN);
96
+
val &= ~(CPACR_ELx_ZEN | CPACR_ELx_SMEN);
98
97
99
98
/*
100
99
* With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to
···
108
109
109
110
if (guest_owns_fp_regs()) {
110
111
if (vcpu_has_sve(vcpu))
111
-
val |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN;
112
+
val |= CPACR_ELx_ZEN;
112
113
} else {
113
-
val &= ~(CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN);
114
+
val &= ~CPACR_ELx_FPEN;
114
115
__activate_traps_fpsimd32(vcpu);
115
116
}
116
117
···
259
260
write_sysreg_el2(elr, SYS_ELR);
260
261
261
262
return true;
263
+
}
264
+
265
+
static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu)
266
+
{
267
+
__fpsimd_save_state(*host_data_ptr(fpsimd_state));
262
268
}
263
269
264
270
static const exit_handler_fn hyp_exit_handlers[] = {
+4
-2
arch/arm64/kvm/nested.c
+4
-2
arch/arm64/kvm/nested.c
···
58
58
break;
59
59
60
60
case SYS_ID_AA64PFR1_EL1:
61
-
/* Only support SSBS */
62
-
val &= NV_FTR(PFR1, SSBS);
61
+
/* Only support BTI, SSBS, CSV2_frac */
62
+
val &= (NV_FTR(PFR1, BT) |
63
+
NV_FTR(PFR1, SSBS) |
64
+
NV_FTR(PFR1, CSV2_frac));
63
65
break;
64
66
65
67
case SYS_ID_AA64MMFR0_EL1:
+3
arch/arm64/kvm/reset.c
+3
arch/arm64/kvm/reset.c
···
32
32
33
33
/* Maximum phys_shift supported for any VM on this host */
34
34
static u32 __ro_after_init kvm_ipa_limit;
35
+
unsigned int __ro_after_init kvm_host_sve_max_vl;
35
36
36
37
/*
37
38
* ARMv8 Reset Values
···
52
51
{
53
52
if (system_supports_sve()) {
54
53
kvm_sve_max_vl = sve_max_virtualisable_vl();
54
+
kvm_host_sve_max_vl = sve_max_vl();
55
+
kvm_nvhe_sym(kvm_host_sve_max_vl) = kvm_host_sve_max_vl;
55
56
56
57
/*
57
58
* The get_sve_reg()/set_sve_reg() ioctl interface will need
+4
-3
arch/riscv/kvm/aia_device.c
+4
-3
arch/riscv/kvm/aia_device.c
···
237
237
238
238
static u32 aia_imsic_hart_index(struct kvm_aia *aia, gpa_t addr)
239
239
{
240
-
u32 hart, group = 0;
240
+
u32 hart = 0, group = 0;
241
241
242
-
hart = (addr >> (aia->nr_guest_bits + IMSIC_MMIO_PAGE_SHIFT)) &
243
-
GENMASK_ULL(aia->nr_hart_bits - 1, 0);
242
+
if (aia->nr_hart_bits)
243
+
hart = (addr >> (aia->nr_guest_bits + IMSIC_MMIO_PAGE_SHIFT)) &
244
+
GENMASK_ULL(aia->nr_hart_bits - 1, 0);
244
245
if (aia->nr_group_bits)
245
246
group = (addr >> aia->nr_group_shift) &
246
247
GENMASK_ULL(aia->nr_group_bits - 1, 0);
+2
-2
arch/riscv/kvm/vcpu_onereg.c
+2
-2
arch/riscv/kvm/vcpu_onereg.c
···
724
724
switch (reg_subtype) {
725
725
case KVM_REG_RISCV_ISA_SINGLE:
726
726
return riscv_vcpu_set_isa_ext_single(vcpu, reg_num, reg_val);
727
-
case KVM_REG_RISCV_SBI_MULTI_EN:
727
+
case KVM_REG_RISCV_ISA_MULTI_EN:
728
728
return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, true);
729
-
case KVM_REG_RISCV_SBI_MULTI_DIS:
729
+
case KVM_REG_RISCV_ISA_MULTI_DIS:
730
730
return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, false);
731
731
default:
732
732
return -ENOENT;
+1
arch/x86/include/asm/kvm_host.h
+1
arch/x86/include/asm/kvm_host.h
···
2154
2154
2155
2155
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
2156
2156
void *insn, int insn_len);
2157
+
void kvm_mmu_print_sptes(struct kvm_vcpu *vcpu, gpa_t gpa, const char *msg);
2157
2158
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
2158
2159
void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
2159
2160
u64 addr, unsigned long roots);
+1
-1
arch/x86/include/asm/vmxfeatures.h
+1
-1
arch/x86/include/asm/vmxfeatures.h
···
77
77
#define VMX_FEATURE_ENCLS_EXITING ( 2*32+ 15) /* "" VM-Exit on ENCLS (leaf dependent) */
78
78
#define VMX_FEATURE_RDSEED_EXITING ( 2*32+ 16) /* "" VM-Exit on RDSEED */
79
79
#define VMX_FEATURE_PAGE_MOD_LOGGING ( 2*32+ 17) /* "pml" Log dirty pages into buffer */
80
-
#define VMX_FEATURE_EPT_VIOLATION_VE ( 2*32+ 18) /* "" Conditionally reflect EPT violations as #VE exceptions */
80
+
#define VMX_FEATURE_EPT_VIOLATION_VE ( 2*32+ 18) /* Conditionally reflect EPT violations as #VE exceptions */
81
81
#define VMX_FEATURE_PT_CONCEAL_VMX ( 2*32+ 19) /* "" Suppress VMX indicators in Processor Trace */
82
82
#define VMX_FEATURE_XSAVES ( 2*32+ 20) /* "" Enable XSAVES and XRSTORS in guest */
83
83
#define VMX_FEATURE_MODE_BASED_EPT_EXEC ( 2*32+ 22) /* "ept_mode_based_exec" Enable separate EPT EXEC bits for supervisor vs. user */
+7
-4
arch/x86/kvm/Kconfig
+7
-4
arch/x86/kvm/Kconfig
···
44
44
select KVM_VFIO
45
45
select HAVE_KVM_PM_NOTIFIER if PM
46
46
select KVM_GENERIC_HARDWARE_ENABLING
47
+
select KVM_WERROR if WERROR
47
48
help
48
49
Support hosting fully virtualized guest machines using hardware
49
50
virtualization extensions. You will need a fairly recent
···
67
66
# FRAME_WARN, i.e. KVM_WERROR=y with KASAN=y requires special tuning.
68
67
# Building KVM with -Werror and KASAN is still doable via enabling
69
68
# the kernel-wide WERROR=y.
70
-
depends on KVM && EXPERT && !KASAN
69
+
depends on KVM && ((EXPERT && !KASAN) || WERROR)
71
70
help
72
71
Add -Werror to the build flags for KVM.
73
72
···
98
97
99
98
config KVM_INTEL_PROVE_VE
100
99
bool "Check that guests do not receive #VE exceptions"
101
-
default KVM_PROVE_MMU || DEBUG_KERNEL
102
-
depends on KVM_INTEL
100
+
depends on KVM_INTEL && EXPERT
103
101
help
104
-
105
102
Checks that KVM's page table management code will not incorrectly
106
103
let guests receive a virtualization exception. Virtualization
107
104
exceptions will be trapped by the hypervisor rather than injected
108
105
in the guest.
106
+
107
+
Note: some CPUs appear to generate spurious EPT Violations #VEs
108
+
that trigger KVM's WARN, in particular with eptad=0 and/or nested
109
+
virtualization.
109
110
110
111
If unsure, say N.
111
112
+21
-18
arch/x86/kvm/lapic.c
+21
-18
arch/x86/kvm/lapic.c
···
59
59
#define MAX_APIC_VECTOR 256
60
60
#define APIC_VECTORS_PER_REG 32
61
61
62
-
static bool lapic_timer_advance_dynamic __read_mostly;
62
+
/*
63
+
* Enable local APIC timer advancement (tscdeadline mode only) with adaptive
64
+
* tuning. When enabled, KVM programs the host timer event to fire early, i.e.
65
+
* before the deadline expires, to account for the delay between taking the
66
+
* VM-Exit (to inject the guest event) and the subsequent VM-Enter to resume
67
+
* the guest, i.e. so that the interrupt arrives in the guest with minimal
68
+
* latency relative to the deadline programmed by the guest.
69
+
*/
70
+
static bool lapic_timer_advance __read_mostly = true;
71
+
module_param(lapic_timer_advance, bool, 0444);
72
+
63
73
#define LAPIC_TIMER_ADVANCE_ADJUST_MIN 100 /* clock cycles */
64
74
#define LAPIC_TIMER_ADVANCE_ADJUST_MAX 10000 /* clock cycles */
65
75
#define LAPIC_TIMER_ADVANCE_NS_INIT 1000
···
1864
1854
guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1865
1855
trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline);
1866
1856
1867
-
if (lapic_timer_advance_dynamic) {
1868
-
adjust_lapic_timer_advance(vcpu, guest_tsc - tsc_deadline);
1869
-
/*
1870
-
* If the timer fired early, reread the TSC to account for the
1871
-
* overhead of the above adjustment to avoid waiting longer
1872
-
* than is necessary.
1873
-
*/
1874
-
if (guest_tsc < tsc_deadline)
1875
-
guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1876
-
}
1857
+
adjust_lapic_timer_advance(vcpu, guest_tsc - tsc_deadline);
1858
+
1859
+
/*
1860
+
* If the timer fired early, reread the TSC to account for the overhead
1861
+
* of the above adjustment to avoid waiting longer than is necessary.
1862
+
*/
1863
+
if (guest_tsc < tsc_deadline)
1864
+
guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1877
1865
1878
1866
if (guest_tsc < tsc_deadline)
1879
1867
__wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
···
2820
2812
return HRTIMER_NORESTART;
2821
2813
}
2822
2814
2823
-
int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
2815
+
int kvm_create_lapic(struct kvm_vcpu *vcpu)
2824
2816
{
2825
2817
struct kvm_lapic *apic;
2826
2818
···
2853
2845
hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
2854
2846
HRTIMER_MODE_ABS_HARD);
2855
2847
apic->lapic_timer.timer.function = apic_timer_fn;
2856
-
if (timer_advance_ns == -1) {
2848
+
if (lapic_timer_advance)
2857
2849
apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
2858
-
lapic_timer_advance_dynamic = true;
2859
-
} else {
2860
-
apic->lapic_timer.timer_advance_ns = timer_advance_ns;
2861
-
lapic_timer_advance_dynamic = false;
2862
-
}
2863
2850
2864
2851
/*
2865
2852
* Stuff the APIC ENABLE bit in lieu of temporarily incrementing
+1
-1
arch/x86/kvm/lapic.h
+1
-1
arch/x86/kvm/lapic.h
+36
-12
arch/x86/kvm/mmu/mmu.c
+36
-12
arch/x86/kvm/mmu/mmu.c
···
336
336
#ifdef CONFIG_X86_64
337
337
static void __set_spte(u64 *sptep, u64 spte)
338
338
{
339
+
KVM_MMU_WARN_ON(is_ept_ve_possible(spte));
339
340
WRITE_ONCE(*sptep, spte);
340
341
}
341
342
342
343
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
343
344
{
345
+
KVM_MMU_WARN_ON(is_ept_ve_possible(spte));
344
346
WRITE_ONCE(*sptep, spte);
345
347
}
346
348
347
349
static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
348
350
{
351
+
KVM_MMU_WARN_ON(is_ept_ve_possible(spte));
349
352
return xchg(sptep, spte);
350
353
}
351
354
···
4104
4101
return leaf;
4105
4102
}
4106
4103
4104
+
static int get_sptes_lockless(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes,
4105
+
int *root_level)
4106
+
{
4107
+
int leaf;
4108
+
4109
+
walk_shadow_page_lockless_begin(vcpu);
4110
+
4111
+
if (is_tdp_mmu_active(vcpu))
4112
+
leaf = kvm_tdp_mmu_get_walk(vcpu, addr, sptes, root_level);
4113
+
else
4114
+
leaf = get_walk(vcpu, addr, sptes, root_level);
4115
+
4116
+
walk_shadow_page_lockless_end(vcpu);
4117
+
return leaf;
4118
+
}
4119
+
4107
4120
/* return true if reserved bit(s) are detected on a valid, non-MMIO SPTE. */
4108
4121
static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
4109
4122
{
···
4128
4109
int root, leaf, level;
4129
4110
bool reserved = false;
4130
4111
4131
-
walk_shadow_page_lockless_begin(vcpu);
4132
-
4133
-
if (is_tdp_mmu_active(vcpu))
4134
-
leaf = kvm_tdp_mmu_get_walk(vcpu, addr, sptes, &root);
4135
-
else
4136
-
leaf = get_walk(vcpu, addr, sptes, &root);
4137
-
4138
-
walk_shadow_page_lockless_end(vcpu);
4139
-
4112
+
leaf = get_sptes_lockless(vcpu, addr, sptes, &root);
4140
4113
if (unlikely(leaf < 0)) {
4141
4114
*sptep = 0ull;
4142
4115
return reserved;
···
4410
4399
if (!kvm_apicv_activated(vcpu->kvm))
4411
4400
return RET_PF_EMULATE;
4412
4401
}
4413
-
4414
-
fault->mmu_seq = vcpu->kvm->mmu_invalidate_seq;
4415
-
smp_rmb();
4416
4402
4417
4403
/*
4418
4404
* Check for a relevant mmu_notifier invalidation event before getting
···
5928
5920
insn_len);
5929
5921
}
5930
5922
EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
5923
+
5924
+
void kvm_mmu_print_sptes(struct kvm_vcpu *vcpu, gpa_t gpa, const char *msg)
5925
+
{
5926
+
u64 sptes[PT64_ROOT_MAX_LEVEL + 1];
5927
+
int root_level, leaf, level;
5928
+
5929
+
leaf = get_sptes_lockless(vcpu, gpa, sptes, &root_level);
5930
+
if (unlikely(leaf < 0))
5931
+
return;
5932
+
5933
+
pr_err("%s %llx", msg, gpa);
5934
+
for (level = root_level; level >= leaf; level--)
5935
+
pr_cont(", spte[%d] = 0x%llx", level, sptes[level]);
5936
+
pr_cont("\n");
5937
+
}
5938
+
EXPORT_SYMBOL_GPL(kvm_mmu_print_sptes);
5931
5939
5932
5940
static void __kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
5933
5941
u64 addr, hpa_t root_hpa)
+9
arch/x86/kvm/mmu/spte.h
+9
arch/x86/kvm/mmu/spte.h
···
3
3
#ifndef KVM_X86_MMU_SPTE_H
4
4
#define KVM_X86_MMU_SPTE_H
5
5
6
+
#include <asm/vmx.h>
7
+
6
8
#include "mmu.h"
7
9
#include "mmu_internal.h"
8
10
···
276
274
static inline bool is_shadow_present_pte(u64 pte)
277
275
{
278
276
return !!(pte & SPTE_MMU_PRESENT_MASK);
277
+
}
278
+
279
+
static inline bool is_ept_ve_possible(u64 spte)
280
+
{
281
+
return (shadow_present_mask & VMX_EPT_SUPPRESS_VE_BIT) &&
282
+
!(spte & VMX_EPT_SUPPRESS_VE_BIT) &&
283
+
(spte & VMX_EPT_RWX_MASK) != VMX_EPT_MISCONFIG_WX_VALUE;
279
284
}
280
285
281
286
/*
+2
arch/x86/kvm/mmu/tdp_iter.h
+2
arch/x86/kvm/mmu/tdp_iter.h
···
21
21
22
22
static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte)
23
23
{
24
+
KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte));
24
25
return xchg(rcu_dereference(sptep), new_spte);
25
26
}
26
27
27
28
static inline void __kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 new_spte)
28
29
{
30
+
KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte));
29
31
WRITE_ONCE(*rcu_dereference(sptep), new_spte);
30
32
}
31
33
+1
-1
arch/x86/kvm/mmu/tdp_mmu.c
+1
-1
arch/x86/kvm/mmu/tdp_mmu.c
+14
-5
arch/x86/kvm/svm/sev.c
+14
-5
arch/x86/kvm/svm/sev.c
···
779
779
*/
780
780
fpstate_set_confidential(&vcpu->arch.guest_fpu);
781
781
vcpu->arch.guest_state_protected = true;
782
+
783
+
/*
784
+
* SEV-ES guest mandates LBR Virtualization to be _always_ ON. Enable it
785
+
* only after setting guest_state_protected because KVM_SET_MSRS allows
786
+
* dynamic toggling of LBRV (for performance reason) on write access to
787
+
* MSR_IA32_DEBUGCTLMSR when guest_state_protected is not set.
788
+
*/
789
+
svm_enable_lbrv(vcpu);
782
790
return 0;
783
791
}
784
792
···
2414
2406
if (!boot_cpu_has(X86_FEATURE_SEV_ES))
2415
2407
goto out;
2416
2408
2409
+
if (!lbrv) {
2410
+
WARN_ONCE(!boot_cpu_has(X86_FEATURE_LBRV),
2411
+
"LBRV must be present for SEV-ES support");
2412
+
goto out;
2413
+
}
2414
+
2417
2415
/* Has the system been allocated ASIDs for SEV-ES? */
2418
2416
if (min_sev_asid == 1)
2419
2417
goto out;
···
3230
3216
struct kvm_vcpu *vcpu = &svm->vcpu;
3231
3217
3232
3218
svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ES_ENABLE;
3233
-
svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
3234
3219
3235
3220
/*
3236
3221
* An SEV-ES guest requires a VMSA area that is a separate from the
···
3281
3268
/* Clear intercepts on selected MSRs */
3282
3269
set_msr_interception(vcpu, svm->msrpm, MSR_EFER, 1, 1);
3283
3270
set_msr_interception(vcpu, svm->msrpm, MSR_IA32_CR_PAT, 1, 1);
3284
-
set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
3285
-
set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
3286
-
set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
3287
-
set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
3288
3271
}
3289
3272
3290
3273
void sev_init_vmcb(struct vcpu_svm *svm)
+51
-18
arch/x86/kvm/svm/svm.c
+51
-18
arch/x86/kvm/svm/svm.c
···
99
99
{ .index = MSR_IA32_SPEC_CTRL, .always = false },
100
100
{ .index = MSR_IA32_PRED_CMD, .always = false },
101
101
{ .index = MSR_IA32_FLUSH_CMD, .always = false },
102
+
{ .index = MSR_IA32_DEBUGCTLMSR, .always = false },
102
103
{ .index = MSR_IA32_LASTBRANCHFROMIP, .always = false },
103
104
{ .index = MSR_IA32_LASTBRANCHTOIP, .always = false },
104
105
{ .index = MSR_IA32_LASTINTFROMIP, .always = false },
···
216
215
module_param(vgif, int, 0444);
217
216
218
217
/* enable/disable LBR virtualization */
219
-
static int lbrv = true;
218
+
int lbrv = true;
220
219
module_param(lbrv, int, 0444);
221
220
222
221
static int tsc_scaling = true;
···
991
990
vmcb_mark_dirty(to_vmcb, VMCB_LBR);
992
991
}
993
992
994
-
static void svm_enable_lbrv(struct kvm_vcpu *vcpu)
993
+
void svm_enable_lbrv(struct kvm_vcpu *vcpu)
995
994
{
996
995
struct vcpu_svm *svm = to_svm(vcpu);
997
996
···
1001
1000
set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1002
1001
set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1003
1002
1003
+
if (sev_es_guest(vcpu->kvm))
1004
+
set_msr_interception(vcpu, svm->msrpm, MSR_IA32_DEBUGCTLMSR, 1, 1);
1005
+
1004
1006
/* Move the LBR msrs to the vmcb02 so that the guest can see them. */
1005
1007
if (is_guest_mode(vcpu))
1006
1008
svm_copy_lbrs(svm->vmcb, svm->vmcb01.ptr);
···
1012
1008
static void svm_disable_lbrv(struct kvm_vcpu *vcpu)
1013
1009
{
1014
1010
struct vcpu_svm *svm = to_svm(vcpu);
1011
+
1012
+
KVM_BUG_ON(sev_es_guest(vcpu->kvm), vcpu->kvm);
1015
1013
1016
1014
svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1017
1015
set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
···
2828
2822
return 0;
2829
2823
}
2830
2824
2825
+
static bool
2826
+
sev_es_prevent_msr_access(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2827
+
{
2828
+
return sev_es_guest(vcpu->kvm) &&
2829
+
vcpu->arch.guest_state_protected &&
2830
+
svm_msrpm_offset(msr_info->index) != MSR_INVALID &&
2831
+
!msr_write_intercepted(vcpu, msr_info->index);
2832
+
}
2833
+
2831
2834
static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2832
2835
{
2833
2836
struct vcpu_svm *svm = to_svm(vcpu);
2837
+
2838
+
if (sev_es_prevent_msr_access(vcpu, msr_info)) {
2839
+
msr_info->data = 0;
2840
+
return -EINVAL;
2841
+
}
2834
2842
2835
2843
switch (msr_info->index) {
2836
2844
case MSR_AMD64_TSC_RATIO:
···
2996
2976
2997
2977
u32 ecx = msr->index;
2998
2978
u64 data = msr->data;
2979
+
2980
+
if (sev_es_prevent_msr_access(vcpu, msr))
2981
+
return -EINVAL;
2982
+
2999
2983
switch (ecx) {
3000
2984
case MSR_AMD64_TSC_RATIO:
3001
2985
···
3870
3846
struct vcpu_svm *svm = to_svm(vcpu);
3871
3847
3872
3848
/*
3873
-
* KVM should never request an NMI window when vNMI is enabled, as KVM
3874
-
* allows at most one to-be-injected NMI and one pending NMI, i.e. if
3875
-
* two NMIs arrive simultaneously, KVM will inject one and set
3876
-
* V_NMI_PENDING for the other. WARN, but continue with the standard
3877
-
* single-step approach to try and salvage the pending NMI.
3849
+
* If NMIs are outright masked, i.e. the vCPU is already handling an
3850
+
* NMI, and KVM has not yet intercepted an IRET, then there is nothing
3851
+
* more to do at this time as KVM has already enabled IRET intercepts.
3852
+
* If KVM has already intercepted IRET, then single-step over the IRET,
3853
+
* as NMIs aren't architecturally unmasked until the IRET completes.
3854
+
*
3855
+
* If vNMI is enabled, KVM should never request an NMI window if NMIs
3856
+
* are masked, as KVM allows at most one to-be-injected NMI and one
3857
+
* pending NMI. If two NMIs arrive simultaneously, KVM will inject one
3858
+
* NMI and set V_NMI_PENDING for the other, but if and only if NMIs are
3859
+
* unmasked. KVM _will_ request an NMI window in some situations, e.g.
3860
+
* if the vCPU is in an STI shadow or if GIF=0, KVM can't immediately
3861
+
* inject the NMI. In those situations, KVM needs to single-step over
3862
+
* the STI shadow or intercept STGI.
3878
3863
*/
3879
-
WARN_ON_ONCE(is_vnmi_enabled(svm));
3864
+
if (svm_get_nmi_mask(vcpu)) {
3865
+
WARN_ON_ONCE(is_vnmi_enabled(svm));
3880
3866
3881
-
if (svm_get_nmi_mask(vcpu) && !svm->awaiting_iret_completion)
3882
-
return; /* IRET will cause a vm exit */
3867
+
if (!svm->awaiting_iret_completion)
3868
+
return; /* IRET will cause a vm exit */
3869
+
}
3883
3870
3884
3871
/*
3885
3872
* SEV-ES guests are responsible for signaling when a vCPU is ready to
···
5300
5265
5301
5266
nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS);
5302
5267
5268
+
if (lbrv) {
5269
+
if (!boot_cpu_has(X86_FEATURE_LBRV))
5270
+
lbrv = false;
5271
+
else
5272
+
pr_info("LBR virtualization supported\n");
5273
+
}
5303
5274
/*
5304
5275
* Note, SEV setup consumes npt_enabled and enable_mmio_caching (which
5305
5276
* may be modified by svm_adjust_mmio_mask()), as well as nrips.
···
5357
5316
if (!vnmi) {
5358
5317
svm_x86_ops.is_vnmi_pending = NULL;
5359
5318
svm_x86_ops.set_vnmi_pending = NULL;
5360
-
}
5361
-
5362
-
5363
-
if (lbrv) {
5364
-
if (!boot_cpu_has(X86_FEATURE_LBRV))
5365
-
lbrv = false;
5366
-
else
5367
-
pr_info("LBR virtualization supported\n");
5368
5319
}
5369
5320
5370
5321
if (!enable_pmu)
+3
-1
arch/x86/kvm/svm/svm.h
+3
-1
arch/x86/kvm/svm/svm.h
···
30
30
#define IOPM_SIZE PAGE_SIZE * 3
31
31
#define MSRPM_SIZE PAGE_SIZE * 2
32
32
33
-
#define MAX_DIRECT_ACCESS_MSRS 47
33
+
#define MAX_DIRECT_ACCESS_MSRS 48
34
34
#define MSRPM_OFFSETS 32
35
35
extern u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
36
36
extern bool npt_enabled;
···
39
39
extern bool intercept_smi;
40
40
extern bool x2avic_enabled;
41
41
extern bool vnmi;
42
+
extern int lbrv;
42
43
43
44
/*
44
45
* Clean bits in VMCB.
···
553
552
void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm);
554
553
void svm_vcpu_free_msrpm(u32 *msrpm);
555
554
void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb);
555
+
void svm_enable_lbrv(struct kvm_vcpu *vcpu);
556
556
void svm_update_lbrv(struct kvm_vcpu *vcpu);
557
557
558
558
int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer);
+5
arch/x86/kvm/vmx/nested.c
+5
arch/x86/kvm/vmx/nested.c
···
2242
2242
vmcs_write64(EPT_POINTER,
2243
2243
construct_eptp(&vmx->vcpu, 0, PT64_ROOT_4LEVEL));
2244
2244
2245
+
if (vmx->ve_info)
2246
+
vmcs_write64(VE_INFORMATION_ADDRESS, __pa(vmx->ve_info));
2247
+
2245
2248
/* All VMFUNCs are currently emulated through L0 vmexits. */
2246
2249
if (cpu_has_vmx_vmfunc())
2247
2250
vmcs_write64(VM_FUNCTION_CONTROL, 0);
···
6232
6229
return true;
6233
6230
else if (is_alignment_check(intr_info) &&
6234
6231
!vmx_guest_inject_ac(vcpu))
6232
+
return true;
6233
+
else if (is_ve_fault(intr_info))
6235
6234
return true;
6236
6235
return false;
6237
6236
case EXIT_REASON_EXTERNAL_INTERRUPT:
+9
-2
arch/x86/kvm/vmx/vmx.c
+9
-2
arch/x86/kvm/vmx/vmx.c
···
5218
5218
if (is_invalid_opcode(intr_info))
5219
5219
return handle_ud(vcpu);
5220
5220
5221
-
if (KVM_BUG_ON(is_ve_fault(intr_info), vcpu->kvm))
5222
-
return -EIO;
5221
+
if (WARN_ON_ONCE(is_ve_fault(intr_info))) {
5222
+
struct vmx_ve_information *ve_info = vmx->ve_info;
5223
+
5224
+
WARN_ONCE(ve_info->exit_reason != EXIT_REASON_EPT_VIOLATION,
5225
+
"Unexpected #VE on VM-Exit reason 0x%x", ve_info->exit_reason);
5226
+
dump_vmcs(vcpu);
5227
+
kvm_mmu_print_sptes(vcpu, ve_info->guest_physical_address, "#VE");
5228
+
return 1;
5229
+
}
5223
5230
5224
5231
error_code = 0;
5225
5232
if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
+1
-10
arch/x86/kvm/x86.c
+1
-10
arch/x86/kvm/x86.c
···
164
164
static u32 __read_mostly tsc_tolerance_ppm = 250;
165
165
module_param(tsc_tolerance_ppm, uint, 0644);
166
166
167
-
/*
168
-
* lapic timer advance (tscdeadline mode only) in nanoseconds. '-1' enables
169
-
* adaptive tuning starting from default advancement of 1000ns. '0' disables
170
-
* advancement entirely. Any other value is used as-is and disables adaptive
171
-
* tuning, i.e. allows privileged userspace to set an exact advancement time.
172
-
*/
173
-
static int __read_mostly lapic_timer_advance_ns = -1;
174
-
module_param(lapic_timer_advance_ns, int, 0644);
175
-
176
167
static bool __read_mostly vector_hashing = true;
177
168
module_param(vector_hashing, bool, 0444);
178
169
···
12160
12169
if (r < 0)
12161
12170
return r;
12162
12171
12163
-
r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
12172
+
r = kvm_create_lapic(vcpu);
12164
12173
if (r < 0)
12165
12174
goto fail_mmu_destroy;
12166
12175