+9

Documentation/virt/kvm/api.rst

reviewed

··· 6414 6414 guest_memfd range is not allowed (any number of memory regions can be bound to 6415 6415 a single guest_memfd file, but the bound ranges must not overlap). 6416 6416 6417 6417 + When the capability KVM_CAP_GUEST_MEMFD_MMAP is supported, the 'flags' field 6418 6418 + supports GUEST_MEMFD_FLAG_MMAP. Setting this flag on guest_memfd creation 6419 6419 + enables mmap() and faulting of guest_memfd memory to host userspace. 6420 6420 + 6421 6421 + When the KVM MMU performs a PFN lookup to service a guest fault and the backing 6422 6422 + guest_memfd has the GUEST_MEMFD_FLAG_MMAP set, then the fault will always be 6423 6423 + consumed from guest_memfd, regardless of whether it is a shared or a private 6424 6424 + fault. 6425 6425 + 6417 6426 See KVM_SET_USER_MEMORY_REGION2 for additional details. 6418 6427 6419 6428 4.143 KVM_PRE_FAULT_MEMORY

+2

arch/arm64/include/asm/kvm_asm.h

reviewed

··· 81 81 __KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff, 82 82 __KVM_HOST_SMCCC_FUNC___vgic_v3_save_vmcr_aprs, 83 83 __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs, 84 84 + __KVM_HOST_SMCCC_FUNC___pkvm_reserve_vm, 85 85 + __KVM_HOST_SMCCC_FUNC___pkvm_unreserve_vm, 84 86 __KVM_HOST_SMCCC_FUNC___pkvm_init_vm, 85 87 __KVM_HOST_SMCCC_FUNC___pkvm_init_vcpu, 86 88 __KVM_HOST_SMCCC_FUNC___pkvm_teardown_vm,

+28 -6

arch/arm64/include/asm/kvm_emulate.h

reviewed

··· 220 220 221 221 static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu) 222 222 { 223 223 + /* 224 224 + * DDI0487L.b Known Issue D22105 225 225 + * 226 226 + * When executing at EL2 and HCR_EL2.{E2H,TGE} = {1, 0} it is 227 227 + * IMPLEMENTATION DEFINED whether the effective value of HCR_EL2.AMO 228 228 + * is the value programmed or 1. 229 229 + * 230 230 + * Make the implementation choice of treating the effective value as 1 as 231 231 + * we cannot subsequently catch changes to TGE or AMO that would 232 232 + * otherwise lead to the SError becoming deliverable. 233 233 + */ 234 234 + if (vcpu_is_el2(vcpu) && vcpu_el2_e2h_is_set(vcpu) && !vcpu_el2_tge_is_set(vcpu)) 235 235 + return true; 236 236 + 223 237 return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_AMO; 224 238 } 225 239 ··· 525 511 if (vcpu_mode_is_32bit(vcpu)) { 526 512 *vcpu_cpsr(vcpu) |= PSR_AA32_E_BIT; 527 513 } else { 528 528 - u64 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); 514 514 + enum vcpu_sysreg r; 515 515 + u64 sctlr; 516 516 + 517 517 + r = vcpu_has_nv(vcpu) ? SCTLR_EL2 : SCTLR_EL1; 518 518 + 519 519 + sctlr = vcpu_read_sys_reg(vcpu, r); 529 520 sctlr |= SCTLR_ELx_EE; 530 530 - vcpu_write_sys_reg(vcpu, sctlr, SCTLR_EL1); 521 521 + vcpu_write_sys_reg(vcpu, sctlr, r); 531 522 } 532 523 } 533 524 534 525 static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu) 535 526 { 527 527 + enum vcpu_sysreg r; 528 528 + u64 bit; 529 529 + 536 530 if (vcpu_mode_is_32bit(vcpu)) 537 531 return !!(*vcpu_cpsr(vcpu) & PSR_AA32_E_BIT); 538 532 539 539 - if (vcpu_mode_priv(vcpu)) 540 540 - return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & SCTLR_ELx_EE); 541 541 - else 542 542 - return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & SCTLR_EL1_E0E); 533 533 + r = is_hyp_ctxt(vcpu) ? SCTLR_EL2 : SCTLR_EL1; 534 534 + bit = vcpu_mode_priv(vcpu) ? SCTLR_ELx_EE : SCTLR_EL1_E0E; 535 535 + 536 536 + return vcpu_read_sys_reg(vcpu, r) & bit; 543 537 } 544 538 545 539 static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu,

+3 -2

arch/arm64/include/asm/kvm_host.h

reviewed

··· 252 252 pkvm_handle_t handle; 253 253 struct kvm_hyp_memcache teardown_mc; 254 254 struct kvm_hyp_memcache stage2_teardown_mc; 255 255 - bool enabled; 255 255 + bool is_protected; 256 256 + bool is_created; 256 257 }; 257 258 258 259 struct kvm_mpidr_data { ··· 1443 1442 1444 1443 #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE 1445 1444 1446 1446 - #define kvm_vm_is_protected(kvm) (is_protected_kvm_enabled() && (kvm)->arch.pkvm.enabled) 1445 1445 + #define kvm_vm_is_protected(kvm) (is_protected_kvm_enabled() && (kvm)->arch.pkvm.is_protected) 1447 1446 1448 1447 #define vcpu_is_protected(vcpu) kvm_vm_is_protected((vcpu)->kvm) 1449 1448

+25 -2

arch/arm64/include/asm/kvm_nested.h

reviewed

··· 83 83 extern void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu); 84 84 extern void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu); 85 85 86 86 + extern void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu); 87 87 + 86 88 struct kvm_s2_trans { 87 89 phys_addr_t output; 88 90 unsigned long block_size; ··· 267 265 return base; 268 266 } 269 267 270 270 - static inline unsigned int ps_to_output_size(unsigned int ps) 268 268 + static inline unsigned int ps_to_output_size(unsigned int ps, bool pa52bit) 271 269 { 272 270 switch (ps) { 273 271 case 0: return 32; ··· 275 273 case 2: return 40; 276 274 case 3: return 42; 277 275 case 4: return 44; 278 278 - case 5: 276 276 + case 5: return 48; 277 277 + case 6: if (pa52bit) 278 278 + return 52; 279 279 + fallthrough; 279 280 default: 280 281 return 48; 281 282 } ··· 290 285 TR_EL2, 291 286 }; 292 287 288 288 + struct s1_walk_info; 289 289 + 290 290 + struct s1_walk_context { 291 291 + struct s1_walk_info *wi; 292 292 + u64 table_ipa; 293 293 + int level; 294 294 + }; 295 295 + 296 296 + struct s1_walk_filter { 297 297 + int (*fn)(struct s1_walk_context *, void *); 298 298 + void *priv; 299 299 + }; 300 300 + 293 301 struct s1_walk_info { 302 302 + struct s1_walk_filter *filter; 294 303 u64 baddr; 295 304 enum trans_regime regime; 296 305 unsigned int max_oa_bits; 297 306 unsigned int pgshift; 298 307 unsigned int txsz; 299 308 int sl; 309 309 + u8 sh; 300 310 bool as_el0; 301 311 bool hpd; 302 312 bool e0poe; ··· 319 299 bool pan; 320 300 bool be; 321 301 bool s2; 302 302 + bool pa52bit; 322 303 }; 323 304 324 305 struct s1_walk_result { ··· 355 334 356 335 int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 357 336 struct s1_walk_result *wr, u64 va); 337 337 + int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, 338 338 + int *level); 358 339 359 340 /* VNCR management */ 360 341 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu);

+1

arch/arm64/include/asm/kvm_pkvm.h

reviewed

··· 18 18 19 19 int pkvm_init_host_vm(struct kvm *kvm); 20 20 int pkvm_create_hyp_vm(struct kvm *kvm); 21 21 + bool pkvm_hyp_vm_is_created(struct kvm *kvm); 21 22 void pkvm_destroy_hyp_vm(struct kvm *kvm); 22 23 int pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu); 23 24

+1

arch/arm64/include/asm/traps.h

reviewed

··· 36 36 int ubsan_brk_handler(struct pt_regs *regs, unsigned long esr); 37 37 38 38 int early_brk64(unsigned long addr, unsigned long esr, struct pt_regs *regs); 39 39 + void dump_kernel_instr(unsigned long kaddr); 39 40 40 41 /* 41 42 * Move regs->pc to next instruction and do necessary setup before it

+2

arch/arm64/include/asm/vncr_mapping.h

reviewed

··· 94 94 #define VNCR_PMSICR_EL1 0x838 95 95 #define VNCR_PMSIRR_EL1 0x840 96 96 #define VNCR_PMSLATFR_EL1 0x848 97 97 + #define VNCR_PMSNEVFR_EL1 0x850 98 98 + #define VNCR_PMSDSFR_EL1 0x858 97 99 #define VNCR_TRFCR_EL1 0x880 98 100 #define VNCR_MPAM1_EL1 0x900 99 101 #define VNCR_MPAMHCR_EL2 0x930

+15

arch/arm64/kernel/cpufeature.c

reviewed

··· 2550 2550 return idr & MPAMIDR_EL1_HAS_HCR; 2551 2551 } 2552 2552 2553 2553 + static bool 2554 2554 + test_has_gicv5_legacy(const struct arm64_cpu_capabilities *entry, int scope) 2555 2555 + { 2556 2556 + if (!this_cpu_has_cap(ARM64_HAS_GICV5_CPUIF)) 2557 2557 + return false; 2558 2558 + 2559 2559 + return !!(read_sysreg_s(SYS_ICC_IDR0_EL1) & ICC_IDR0_EL1_GCIE_LEGACY); 2560 2560 + } 2561 2561 + 2553 2562 static const struct arm64_cpu_capabilities arm64_features[] = { 2554 2563 { 2555 2564 .capability = ARM64_ALWAYS_BOOT, ··· 3175 3166 .capability = ARM64_HAS_GICV5_CPUIF, 3176 3167 .matches = has_cpuid_feature, 3177 3168 ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, GCIE, IMP) 3169 3169 + }, 3170 3170 + { 3171 3171 + .desc = "GICv5 Legacy vCPU interface", 3172 3172 + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, 3173 3173 + .capability = ARM64_HAS_GICV5_LEGACY, 3174 3174 + .matches = test_has_gicv5_legacy, 3178 3175 }, 3179 3176 {}, 3180 3177 };

+3

arch/arm64/kernel/image-vars.h

reviewed

··· 105 105 KVM_NVHE_ALIAS(vgic_v2_cpuif_trap); 106 106 KVM_NVHE_ALIAS(vgic_v3_cpuif_trap); 107 107 108 108 + /* Static key indicating whether GICv3 has GICv2 compatibility */ 109 109 + KVM_NVHE_ALIAS(vgic_v3_has_v2_compat); 110 110 + 108 111 /* Static key which is set if CNTVOFF_EL2 is unusable */ 109 112 KVM_NVHE_ALIAS(broken_cntvoff_key); 110 113

+9 -6

arch/arm64/kernel/traps.c

reviewed

··· 149 149 150 150 int show_unhandled_signals = 0; 151 151 152 152 - static void dump_kernel_instr(const char *lvl, struct pt_regs *regs) 152 152 + void dump_kernel_instr(unsigned long kaddr) 153 153 { 154 154 - unsigned long addr = instruction_pointer(regs); 155 154 char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str; 156 155 int i; 157 156 158 158 - if (user_mode(regs)) 157 157 + if (!is_ttbr1_addr(kaddr)) 159 158 return; 160 159 161 160 for (i = -4; i < 1; i++) { 162 161 unsigned int val, bad; 163 162 164 164 - bad = aarch64_insn_read(&((u32 *)addr)[i], &val); 163 163 + bad = aarch64_insn_read(&((u32 *)kaddr)[i], &val); 165 164 166 165 if (!bad) 167 166 p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val); ··· 168 169 p += sprintf(p, i == 0 ? "(????????) " : "???????? "); 169 170 } 170 171 171 171 - printk("%sCode: %s\n", lvl, str); 172 172 + printk(KERN_EMERG "Code: %s\n", str); 172 173 } 173 174 174 175 #define S_SMP " SMP" ··· 177 178 { 178 179 static int die_counter; 179 180 int ret; 181 181 + unsigned long addr = instruction_pointer(regs); 180 182 181 183 pr_emerg("Internal error: %s: %016lx [#%d] " S_SMP "\n", 182 184 str, err, ++die_counter); ··· 190 190 print_modules(); 191 191 show_regs(regs); 192 192 193 193 - dump_kernel_instr(KERN_EMERG, regs); 193 193 + if (user_mode(regs)) 194 194 + return ret; 195 195 + 196 196 + dump_kernel_instr(addr); 194 197 195 198 return ret; 196 199 }

+1

arch/arm64/kvm/Kconfig

reviewed

··· 37 37 select HAVE_KVM_VCPU_RUN_PID_CHANGE 38 38 select SCHED_INFO 39 39 select GUEST_PERF_EVENTS if PERF_EVENTS 40 40 + select KVM_GUEST_MEMFD 40 41 help 41 42 Support hosting virtualized guest machines. 42 43

+13 -6

arch/arm64/kvm/arm.c

reviewed

··· 170 170 if (ret) 171 171 return ret; 172 172 173 173 - ret = pkvm_init_host_vm(kvm); 174 174 - if (ret) 175 175 - goto err_unshare_kvm; 176 176 - 177 173 if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL_ACCOUNT)) { 178 174 ret = -ENOMEM; 179 175 goto err_unshare_kvm; ··· 179 183 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type); 180 184 if (ret) 181 185 goto err_free_cpumask; 186 186 + 187 187 + if (is_protected_kvm_enabled()) { 188 188 + /* 189 189 + * If any failures occur after this is successful, make sure to 190 190 + * call __pkvm_unreserve_vm to unreserve the VM in hyp. 191 191 + */ 192 192 + ret = pkvm_init_host_vm(kvm); 193 193 + if (ret) 194 194 + goto err_free_cpumask; 195 195 + } 182 196 183 197 kvm_vgic_early_init(kvm); 184 198 ··· 2323 2317 } 2324 2318 2325 2319 if (kvm_mode == KVM_MODE_NV && 2326 2326 - !(vgic_present && kvm_vgic_global_state.type == VGIC_V3)) { 2327 2327 - kvm_err("NV support requires GICv3, giving up\n"); 2320 2320 + !(vgic_present && (kvm_vgic_global_state.type == VGIC_V3 || 2321 2321 + kvm_vgic_global_state.has_gcie_v3_compat))) { 2322 2322 + kvm_err("NV support requires GICv3 or GICv5 with legacy support, giving up\n"); 2328 2323 err = -EINVAL; 2329 2324 goto out; 2330 2325 }

+270 -106

arch/arm64/kvm/at.c

reviewed

··· 28 28 /* Return true if the IPA is out of the OA range */ 29 29 static bool check_output_size(u64 ipa, struct s1_walk_info *wi) 30 30 { 31 31 + if (wi->pa52bit) 32 32 + return wi->max_oa_bits < 52 && (ipa & GENMASK_ULL(51, wi->max_oa_bits)); 31 33 return wi->max_oa_bits < 48 && (ipa & GENMASK_ULL(47, wi->max_oa_bits)); 34 34 + } 35 35 + 36 36 + static bool has_52bit_pa(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, u64 tcr) 37 37 + { 38 38 + switch (BIT(wi->pgshift)) { 39 39 + case SZ_64K: 40 40 + default: /* IMPDEF: treat any other value as 64k */ 41 41 + if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, PARANGE, 52)) 42 42 + return false; 43 43 + return ((wi->regime == TR_EL2 ? 44 44 + FIELD_GET(TCR_EL2_PS_MASK, tcr) : 45 45 + FIELD_GET(TCR_IPS_MASK, tcr)) == 0b0110); 46 46 + case SZ_16K: 47 47 + if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN16, 52_BIT)) 48 48 + return false; 49 49 + break; 50 50 + case SZ_4K: 51 51 + if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN4, 52_BIT)) 52 52 + return false; 53 53 + break; 54 54 + } 55 55 + 56 56 + return (tcr & (wi->regime == TR_EL2 ? TCR_EL2_DS : TCR_DS)); 57 57 + } 58 58 + 59 59 + static u64 desc_to_oa(struct s1_walk_info *wi, u64 desc) 60 60 + { 61 61 + u64 addr; 62 62 + 63 63 + if (!wi->pa52bit) 64 64 + return desc & GENMASK_ULL(47, wi->pgshift); 65 65 + 66 66 + switch (BIT(wi->pgshift)) { 67 67 + case SZ_4K: 68 68 + case SZ_16K: 69 69 + addr = desc & GENMASK_ULL(49, wi->pgshift); 70 70 + addr |= FIELD_GET(KVM_PTE_ADDR_51_50_LPA2, desc) << 50; 71 71 + break; 72 72 + case SZ_64K: 73 73 + default: /* IMPDEF: treat any other value as 64k */ 74 74 + addr = desc & GENMASK_ULL(47, wi->pgshift); 75 75 + addr |= FIELD_GET(KVM_PTE_ADDR_51_48, desc) << 48; 76 76 + break; 77 77 + } 78 78 + 79 79 + return addr; 32 80 } 33 81 34 82 /* Return the translation regime that applies to an AT instruction */ ··· 98 50 } 99 51 } 100 52 53 53 + static u64 effective_tcr2(struct kvm_vcpu *vcpu, enum trans_regime regime) 54 54 + { 55 55 + if (regime == TR_EL10) { 56 56 + if (vcpu_has_nv(vcpu) && 57 57 + !(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TCR2En)) 58 58 + return 0; 59 59 + 60 60 + return vcpu_read_sys_reg(vcpu, TCR2_EL1); 61 61 + } 62 62 + 63 63 + return vcpu_read_sys_reg(vcpu, TCR2_EL2); 64 64 + } 65 65 + 101 66 static bool s1pie_enabled(struct kvm_vcpu *vcpu, enum trans_regime regime) 102 67 { 103 68 if (!kvm_has_s1pie(vcpu->kvm)) 104 69 return false; 105 70 106 106 - switch (regime) { 107 107 - case TR_EL2: 108 108 - case TR_EL20: 109 109 - return vcpu_read_sys_reg(vcpu, TCR2_EL2) & TCR2_EL2_PIE; 110 110 - case TR_EL10: 111 111 - return (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TCR2En) && 112 112 - (__vcpu_sys_reg(vcpu, TCR2_EL1) & TCR2_EL1_PIE); 113 113 - default: 114 114 - BUG(); 115 115 - } 71 71 + /* Abuse TCR2_EL1_PIE and use it for EL2 as well */ 72 72 + return effective_tcr2(vcpu, regime) & TCR2_EL1_PIE; 116 73 } 117 74 118 75 static void compute_s1poe(struct kvm_vcpu *vcpu, struct s1_walk_info *wi) ··· 129 76 return; 130 77 } 131 78 132 132 - switch (wi->regime) { 133 133 - case TR_EL2: 134 134 - case TR_EL20: 135 135 - val = vcpu_read_sys_reg(vcpu, TCR2_EL2); 136 136 - wi->poe = val & TCR2_EL2_POE; 137 137 - wi->e0poe = (wi->regime == TR_EL20) && (val & TCR2_EL2_E0POE); 138 138 - break; 139 139 - case TR_EL10: 140 140 - if (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TCR2En) { 141 141 - wi->poe = wi->e0poe = false; 142 142 - return; 143 143 - } 79 79 + val = effective_tcr2(vcpu, wi->regime); 144 80 145 145 - val = __vcpu_sys_reg(vcpu, TCR2_EL1); 146 146 - wi->poe = val & TCR2_EL1_POE; 147 147 - wi->e0poe = val & TCR2_EL1_E0POE; 148 148 - } 81 81 + /* Abuse TCR2_EL1_* for EL2 */ 82 82 + wi->poe = val & TCR2_EL1_POE; 83 83 + wi->e0poe = (wi->regime != TR_EL2) && (val & TCR2_EL1_E0POE); 149 84 } 150 85 151 86 static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, ··· 143 102 unsigned int stride, x; 144 103 bool va55, tbi, lva; 145 104 146 146 - hcr = __vcpu_sys_reg(vcpu, HCR_EL2); 147 147 - 148 105 va55 = va & BIT(55); 149 106 150 150 - if (wi->regime == TR_EL2 && va55) 151 151 - goto addrsz; 152 152 - 153 153 - wi->s2 = wi->regime == TR_EL10 && (hcr & (HCR_VM | HCR_DC)); 107 107 + if (vcpu_has_nv(vcpu)) { 108 108 + hcr = __vcpu_sys_reg(vcpu, HCR_EL2); 109 109 + wi->s2 = wi->regime == TR_EL10 && (hcr & (HCR_VM | HCR_DC)); 110 110 + } else { 111 111 + WARN_ON_ONCE(wi->regime != TR_EL10); 112 112 + wi->s2 = false; 113 113 + hcr = 0; 114 114 + } 154 115 155 116 switch (wi->regime) { 156 117 case TR_EL10: ··· 174 131 BUG(); 175 132 } 176 133 134 134 + /* Someone was silly enough to encode TG0/TG1 differently */ 135 135 + if (va55 && wi->regime != TR_EL2) { 136 136 + wi->txsz = FIELD_GET(TCR_T1SZ_MASK, tcr); 137 137 + tg = FIELD_GET(TCR_TG1_MASK, tcr); 138 138 + 139 139 + switch (tg << TCR_TG1_SHIFT) { 140 140 + case TCR_TG1_4K: 141 141 + wi->pgshift = 12; break; 142 142 + case TCR_TG1_16K: 143 143 + wi->pgshift = 14; break; 144 144 + case TCR_TG1_64K: 145 145 + default: /* IMPDEF: treat any other value as 64k */ 146 146 + wi->pgshift = 16; break; 147 147 + } 148 148 + } else { 149 149 + wi->txsz = FIELD_GET(TCR_T0SZ_MASK, tcr); 150 150 + tg = FIELD_GET(TCR_TG0_MASK, tcr); 151 151 + 152 152 + switch (tg << TCR_TG0_SHIFT) { 153 153 + case TCR_TG0_4K: 154 154 + wi->pgshift = 12; break; 155 155 + case TCR_TG0_16K: 156 156 + wi->pgshift = 14; break; 157 157 + case TCR_TG0_64K: 158 158 + default: /* IMPDEF: treat any other value as 64k */ 159 159 + wi->pgshift = 16; break; 160 160 + } 161 161 + } 162 162 + 163 163 + wi->pa52bit = has_52bit_pa(vcpu, wi, tcr); 164 164 + 165 165 + ia_bits = get_ia_size(wi); 166 166 + 167 167 + /* AArch64.S1StartLevel() */ 168 168 + stride = wi->pgshift - 3; 169 169 + wi->sl = 3 - (((ia_bits - 1) - wi->pgshift) / stride); 170 170 + 171 171 + if (wi->regime == TR_EL2 && va55) 172 172 + goto addrsz; 173 173 + 177 174 tbi = (wi->regime == TR_EL2 ? 178 175 FIELD_GET(TCR_EL2_TBI, tcr) : 179 176 (va55 ? ··· 222 139 223 140 if (!tbi && (u64)sign_extend64(va, 55) != va) 224 141 goto addrsz; 142 142 + 143 143 + wi->sh = (wi->regime == TR_EL2 ? 144 144 + FIELD_GET(TCR_EL2_SH0_MASK, tcr) : 145 145 + (va55 ? 146 146 + FIELD_GET(TCR_SH1_MASK, tcr) : 147 147 + FIELD_GET(TCR_SH0_MASK, tcr))); 225 148 226 149 va = (u64)sign_extend64(va, 55); 227 150 ··· 283 194 /* R_BVXDG */ 284 195 wi->hpd |= (wi->poe || wi->e0poe); 285 196 286 286 - /* Someone was silly enough to encode TG0/TG1 differently */ 287 287 - if (va55) { 288 288 - wi->txsz = FIELD_GET(TCR_T1SZ_MASK, tcr); 289 289 - tg = FIELD_GET(TCR_TG1_MASK, tcr); 290 290 - 291 291 - switch (tg << TCR_TG1_SHIFT) { 292 292 - case TCR_TG1_4K: 293 293 - wi->pgshift = 12; break; 294 294 - case TCR_TG1_16K: 295 295 - wi->pgshift = 14; break; 296 296 - case TCR_TG1_64K: 297 297 - default: /* IMPDEF: treat any other value as 64k */ 298 298 - wi->pgshift = 16; break; 299 299 - } 300 300 - } else { 301 301 - wi->txsz = FIELD_GET(TCR_T0SZ_MASK, tcr); 302 302 - tg = FIELD_GET(TCR_TG0_MASK, tcr); 303 303 - 304 304 - switch (tg << TCR_TG0_SHIFT) { 305 305 - case TCR_TG0_4K: 306 306 - wi->pgshift = 12; break; 307 307 - case TCR_TG0_16K: 308 308 - wi->pgshift = 14; break; 309 309 - case TCR_TG0_64K: 310 310 - default: /* IMPDEF: treat any other value as 64k */ 311 311 - wi->pgshift = 16; break; 312 312 - } 313 313 - } 314 314 - 315 197 /* R_PLCGL, R_YXNYW */ 316 198 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR2_EL1, ST, 48_47)) { 317 199 if (wi->txsz > 39) 318 318 - goto transfault_l0; 200 200 + goto transfault; 319 201 } else { 320 202 if (wi->txsz > 48 || (BIT(wi->pgshift) == SZ_64K && wi->txsz > 47)) 321 321 - goto transfault_l0; 203 203 + goto transfault; 322 204 } 323 205 324 206 /* R_GTJBY, R_SXWGM */ 325 207 switch (BIT(wi->pgshift)) { 326 208 case SZ_4K: 327 327 - lva = kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN4, 52_BIT); 328 328 - lva &= tcr & (wi->regime == TR_EL2 ? TCR_EL2_DS : TCR_DS); 329 329 - break; 330 209 case SZ_16K: 331 331 - lva = kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN16, 52_BIT); 332 332 - lva &= tcr & (wi->regime == TR_EL2 ? TCR_EL2_DS : TCR_DS); 210 210 + lva = wi->pa52bit; 333 211 break; 334 212 case SZ_64K: 335 213 lva = kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, VARange, 52); ··· 304 248 } 305 249 306 250 if ((lva && wi->txsz < 12) || (!lva && wi->txsz < 16)) 307 307 - goto transfault_l0; 308 308 - 309 309 - ia_bits = get_ia_size(wi); 251 251 + goto transfault; 310 252 311 253 /* R_YYVYV, I_THCZK */ 312 254 if ((!va55 && va > GENMASK(ia_bits - 1, 0)) || 313 255 (va55 && va < GENMASK(63, ia_bits))) 314 314 - goto transfault_l0; 256 256 + goto transfault; 315 257 316 258 /* I_ZFSYQ */ 317 259 if (wi->regime != TR_EL2 && 318 260 (tcr & (va55 ? TCR_EPD1_MASK : TCR_EPD0_MASK))) 319 319 - goto transfault_l0; 261 261 + goto transfault; 320 262 321 263 /* R_BNDVG and following statements */ 322 264 if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, E0PD, IMP) && 323 265 wi->as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0))) 324 324 - goto transfault_l0; 325 325 - 326 326 - /* AArch64.S1StartLevel() */ 327 327 - stride = wi->pgshift - 3; 328 328 - wi->sl = 3 - (((ia_bits - 1) - wi->pgshift) / stride); 266 266 + goto transfault; 329 267 330 268 ps = (wi->regime == TR_EL2 ? 331 269 FIELD_GET(TCR_EL2_PS_MASK, tcr) : FIELD_GET(TCR_IPS_MASK, tcr)); 332 270 333 333 - wi->max_oa_bits = min(get_kvm_ipa_limit(), ps_to_output_size(ps)); 271 271 + wi->max_oa_bits = min(get_kvm_ipa_limit(), ps_to_output_size(ps, wi->pa52bit)); 334 272 335 273 /* Compute minimal alignment */ 336 274 x = 3 + ia_bits - ((3 - wi->sl) * stride + wi->pgshift); 337 275 338 276 wi->baddr = ttbr & TTBRx_EL1_BADDR; 277 277 + if (wi->pa52bit) { 278 278 + /* 279 279 + * Force the alignment on 64 bytes for top-level tables 280 280 + * smaller than 8 entries, since TTBR.BADDR[5:2] are used to 281 281 + * store bits [51:48] of the first level of lookup. 282 282 + */ 283 283 + x = max(x, 6); 284 284 + 285 285 + wi->baddr |= FIELD_GET(GENMASK_ULL(5, 2), ttbr) << 48; 286 286 + } 339 287 340 288 /* R_VPBBF */ 341 289 if (check_output_size(wi->baddr, wi)) ··· 349 289 350 290 return 0; 351 291 352 352 - addrsz: /* Address Size Fault level 0 */ 292 292 + addrsz: 293 293 + /* 294 294 + * Address Size Fault level 0 to indicate it comes from TTBR. 295 295 + * yes, this is an oddity. 296 296 + */ 353 297 fail_s1_walk(wr, ESR_ELx_FSC_ADDRSZ_L(0), false); 354 298 return -EFAULT; 355 299 356 356 - transfault_l0: /* Translation Fault level 0 */ 357 357 - fail_s1_walk(wr, ESR_ELx_FSC_FAULT_L(0), false); 300 300 + transfault: 301 301 + /* Translation Fault on start level */ 302 302 + fail_s1_walk(wr, ESR_ELx_FSC_FAULT_L(wi->sl), false); 358 303 return -EFAULT; 359 304 } 360 305 ··· 404 339 ipa = kvm_s2_trans_output(&s2_trans); 405 340 } 406 341 342 342 + if (wi->filter) { 343 343 + ret = wi->filter->fn(&(struct s1_walk_context) 344 344 + { 345 345 + .wi = wi, 346 346 + .table_ipa = baddr, 347 347 + .level = level, 348 348 + }, wi->filter->priv); 349 349 + if (ret) 350 350 + return ret; 351 351 + } 352 352 + 407 353 ret = kvm_read_guest(vcpu->kvm, ipa, &desc, sizeof(desc)); 408 354 if (ret) { 409 355 fail_s1_walk(wr, ESR_ELx_FSC_SEA_TTW(level), false); ··· 445 369 wr->PXNTable |= FIELD_GET(PMD_TABLE_PXN, desc); 446 370 } 447 371 448 448 - baddr = desc & GENMASK_ULL(47, wi->pgshift); 372 372 + baddr = desc_to_oa(wi, desc); 449 373 450 374 /* Check for out-of-range OA */ 451 375 if (check_output_size(baddr, wi)) ··· 462 386 463 387 switch (BIT(wi->pgshift)) { 464 388 case SZ_4K: 465 465 - valid_block = level == 1 || level == 2; 389 389 + valid_block = level == 1 || level == 2 || (wi->pa52bit && level == 0); 466 390 break; 467 391 case SZ_16K: 468 392 case SZ_64K: 469 469 - valid_block = level == 2; 393 393 + valid_block = level == 2 || (wi->pa52bit && level == 1); 470 394 break; 471 395 } 472 396 ··· 474 398 goto transfault; 475 399 } 476 400 477 477 - if (check_output_size(desc & GENMASK(47, va_bottom), wi)) 401 401 + baddr = desc_to_oa(wi, desc); 402 402 + if (check_output_size(baddr & GENMASK(52, va_bottom), wi)) 478 403 goto addrsz; 479 404 480 405 if (!(desc & PTE_AF)) { ··· 488 411 wr->failed = false; 489 412 wr->level = level; 490 413 wr->desc = desc; 491 491 - wr->pa = desc & GENMASK(47, va_bottom); 414 414 + wr->pa = baddr & GENMASK(52, va_bottom); 492 415 wr->pa |= va & GENMASK_ULL(va_bottom - 1, 0); 493 416 494 417 wr->nG = (wi->regime != TR_EL2) && (desc & PTE_NG); ··· 717 640 #define ATTR_OSH 0b10 718 641 #define ATTR_ISH 0b11 719 642 720 720 - static u8 compute_sh(u8 attr, u64 desc) 643 643 + static u8 compute_final_sh(u8 attr, u8 sh) 721 644 { 722 722 - u8 sh; 723 723 - 724 645 /* Any form of device, as well as NC has SH[1:0]=0b10 */ 725 646 if (MEMATTR_IS_DEVICE(attr) || attr == MEMATTR(NC, NC)) 726 647 return ATTR_OSH; 727 648 728 728 - sh = FIELD_GET(PTE_SHARED, desc); 729 649 if (sh == ATTR_RSV) /* Reserved, mapped to NSH */ 730 650 sh = ATTR_NSH; 731 651 732 652 return sh; 653 653 + } 654 654 + 655 655 + static u8 compute_s1_sh(struct s1_walk_info *wi, struct s1_walk_result *wr, 656 656 + u8 attr) 657 657 + { 658 658 + u8 sh; 659 659 + 660 660 + /* 661 661 + * non-52bit and LPA have their basic shareability described in the 662 662 + * descriptor. LPA2 gets it from the corresponding field in TCR, 663 663 + * conveniently recorded in the walk info. 664 664 + */ 665 665 + if (!wi->pa52bit || BIT(wi->pgshift) == SZ_64K) 666 666 + sh = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S1_SH, wr->desc); 667 667 + else 668 668 + sh = wi->sh; 669 669 + 670 670 + return compute_final_sh(attr, sh); 733 671 } 734 672 735 673 static u8 combine_sh(u8 s1_sh, u8 s2_sh) ··· 760 668 static u64 compute_par_s12(struct kvm_vcpu *vcpu, u64 s1_par, 761 669 struct kvm_s2_trans *tr) 762 670 { 763 763 - u8 s1_parattr, s2_memattr, final_attr; 671 671 + u8 s1_parattr, s2_memattr, final_attr, s2_sh; 764 672 u64 par; 765 673 766 674 /* If S2 has failed to translate, report the damage */ ··· 833 741 !MEMATTR_IS_DEVICE(final_attr)) 834 742 final_attr = MEMATTR(NC, NC); 835 743 744 744 + s2_sh = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S2_SH, tr->desc); 745 745 + 836 746 par = FIELD_PREP(SYS_PAR_EL1_ATTR, final_attr); 837 747 par |= tr->output & GENMASK(47, 12); 838 748 par |= FIELD_PREP(SYS_PAR_EL1_SH, 839 749 combine_sh(FIELD_GET(SYS_PAR_EL1_SH, s1_par), 840 840 - compute_sh(final_attr, tr->desc))); 750 750 + compute_final_sh(final_attr, s2_sh))); 841 751 842 752 return par; 843 753 } 844 754 845 845 - static u64 compute_par_s1(struct kvm_vcpu *vcpu, struct s1_walk_result *wr, 846 846 - enum trans_regime regime) 755 755 + static u64 compute_par_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 756 756 + struct s1_walk_result *wr) 847 757 { 848 758 u64 par; 849 759 ··· 858 764 } else if (wr->level == S1_MMU_DISABLED) { 859 765 /* MMU off or HCR_EL2.DC == 1 */ 860 766 par = SYS_PAR_EL1_NSE; 861 861 - par |= wr->pa & GENMASK_ULL(47, 12); 767 767 + par |= wr->pa & SYS_PAR_EL1_PA; 862 768 863 863 - if (regime == TR_EL10 && 769 769 + if (wi->regime == TR_EL10 && vcpu_has_nv(vcpu) && 864 770 (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_DC)) { 865 771 par |= FIELD_PREP(SYS_PAR_EL1_ATTR, 866 772 MEMATTR(WbRaWa, WbRaWa)); ··· 875 781 876 782 par = SYS_PAR_EL1_NSE; 877 783 878 878 - mair = (regime == TR_EL10 ? 784 784 + mair = (wi->regime == TR_EL10 ? 879 785 vcpu_read_sys_reg(vcpu, MAIR_EL1) : 880 786 vcpu_read_sys_reg(vcpu, MAIR_EL2)); 881 787 882 788 mair >>= FIELD_GET(PTE_ATTRINDX_MASK, wr->desc) * 8; 883 789 mair &= 0xff; 884 790 885 885 - sctlr = (regime == TR_EL10 ? 791 791 + sctlr = (wi->regime == TR_EL10 ? 886 792 vcpu_read_sys_reg(vcpu, SCTLR_EL1) : 887 793 vcpu_read_sys_reg(vcpu, SCTLR_EL2)); 888 794 ··· 891 797 mair = MEMATTR(NC, NC); 892 798 893 799 par |= FIELD_PREP(SYS_PAR_EL1_ATTR, mair); 894 894 - par |= wr->pa & GENMASK_ULL(47, 12); 800 800 + par |= wr->pa & SYS_PAR_EL1_PA; 895 801 896 896 - sh = compute_sh(mair, wr->desc); 802 802 + sh = compute_s1_sh(wi, wr, mair); 897 803 par |= FIELD_PREP(SYS_PAR_EL1_SH, sh); 898 804 } 899 805 ··· 967 873 wxn = (vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_WXN); 968 874 break; 969 875 case TR_EL10: 970 970 - wxn = (__vcpu_sys_reg(vcpu, SCTLR_EL1) & SCTLR_ELx_WXN); 876 876 + wxn = (vcpu_read_sys_reg(vcpu, SCTLR_EL1) & SCTLR_ELx_WXN); 971 877 break; 972 878 } 973 879 ··· 1280 1186 fail_s1_walk(&wr, ESR_ELx_FSC_PERM_L(wr.level), false); 1281 1187 1282 1188 compute_par: 1283 1283 - return compute_par_s1(vcpu, &wr, wi.regime); 1189 1189 + return compute_par_s1(vcpu, &wi, &wr); 1284 1190 } 1285 1191 1286 1192 /* ··· 1296 1202 { 1297 1203 struct mmu_config config; 1298 1204 struct kvm_s2_mmu *mmu; 1299 1299 - bool fail; 1205 1205 + bool fail, mmu_cs; 1300 1206 u64 par; 1301 1207 1302 1208 par = SYS_PAR_EL1_F; ··· 1312 1218 * If HCR_EL2.{E2H,TGE} == {1,1}, the MMU context is already 1313 1219 * the right one (as we trapped from vEL2). If not, save the 1314 1220 * full MMU context. 1221 1221 + * 1222 1222 + * We are also guaranteed to be in the correct context if 1223 1223 + * we're not in a nested VM. 1315 1224 */ 1316 1316 - if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) 1225 1225 + mmu_cs = (vcpu_has_nv(vcpu) && 1226 1226 + !(vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu))); 1227 1227 + if (!mmu_cs) 1317 1228 goto skip_mmu_switch; 1318 1229 1319 1230 /* ··· 1386 1287 1387 1288 write_sysreg_hcr(HCR_HOST_VHE_FLAGS); 1388 1289 1389 1389 - if (!(vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu))) 1290 1290 + if (mmu_cs) 1390 1291 __mmu_config_restore(&config); 1391 1292 1392 1293 return par; ··· 1568 1469 } 1569 1470 1570 1471 return 0; 1472 1472 + } 1473 1473 + 1474 1474 + struct desc_match { 1475 1475 + u64 ipa; 1476 1476 + int level; 1477 1477 + }; 1478 1478 + 1479 1479 + static int match_s1_desc(struct s1_walk_context *ctxt, void *priv) 1480 1480 + { 1481 1481 + struct desc_match *dm = priv; 1482 1482 + u64 ipa = dm->ipa; 1483 1483 + 1484 1484 + /* Use S1 granule alignment */ 1485 1485 + ipa &= GENMASK(51, ctxt->wi->pgshift); 1486 1486 + 1487 1487 + /* Not the IPA we're looking for? Continue. */ 1488 1488 + if (ipa != ctxt->table_ipa) 1489 1489 + return 0; 1490 1490 + 1491 1491 + /* Note the level and interrupt the walk */ 1492 1492 + dm->level = ctxt->level; 1493 1493 + return -EINTR; 1494 1494 + } 1495 1495 + 1496 1496 + int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, int *level) 1497 1497 + { 1498 1498 + struct desc_match dm = { 1499 1499 + .ipa = ipa, 1500 1500 + }; 1501 1501 + struct s1_walk_info wi = { 1502 1502 + .filter = &(struct s1_walk_filter){ 1503 1503 + .fn = match_s1_desc, 1504 1504 + .priv = &dm, 1505 1505 + }, 1506 1506 + .regime = TR_EL10, 1507 1507 + .as_el0 = false, 1508 1508 + .pan = false, 1509 1509 + }; 1510 1510 + struct s1_walk_result wr = {}; 1511 1511 + int ret; 1512 1512 + 1513 1513 + ret = setup_s1_walk(vcpu, &wi, &wr, va); 1514 1514 + if (ret) 1515 1515 + return ret; 1516 1516 + 1517 1517 + /* We really expect the S1 MMU to be on here... */ 1518 1518 + if (WARN_ON_ONCE(wr.level == S1_MMU_DISABLED)) { 1519 1519 + *level = 0; 1520 1520 + return 0; 1521 1521 + } 1522 1522 + 1523 1523 + /* Walk the guest's PT, looking for a match along the way */ 1524 1524 + ret = walk_s1(vcpu, &wi, &wr, va); 1525 1525 + switch (ret) { 1526 1526 + case -EINTR: 1527 1527 + /* We interrupted the walk on a match, return the level */ 1528 1528 + *level = dm.level; 1529 1529 + return 0; 1530 1530 + case 0: 1531 1531 + /* The walk completed, we failed to find the entry */ 1532 1532 + return -ENOENT; 1533 1533 + default: 1534 1534 + /* Any other error... */ 1535 1535 + return ret; 1536 1536 + } 1571 1537 }

+227 -131

arch/arm64/kvm/config.c

reviewed

··· 7 7 #include <linux/kvm_host.h> 8 8 #include <asm/sysreg.h> 9 9 10 10 + /* 11 11 + * Describes the dependencies between a set of bits (or the negation 12 12 + * of a set of RES0 bits) and a feature. The flags indicate how the 13 13 + * data is interpreted. 14 14 + */ 10 15 struct reg_bits_to_feat_map { 11 11 - u64 bits; 16 16 + union { 17 17 + u64 bits; 18 18 + u64 *res0p; 19 19 + }; 12 20 13 21 #define NEVER_FGU BIT(0) /* Can trap, but never UNDEF */ 14 22 #define CALL_FUNC BIT(1) /* Needs to evaluate tons of crap */ 15 23 #define FIXED_VALUE BIT(2) /* RAZ/WI or RAO/WI in KVM */ 24 24 + #define RES0_POINTER BIT(3) /* Pointer to RES0 value instead of bits */ 25 25 + 16 26 unsigned long flags; 17 27 18 28 union { ··· 38 28 }; 39 29 }; 40 30 41 41 - #define __NEEDS_FEAT_3(m, f, id, fld, lim) \ 31 31 + /* 32 32 + * Describes the dependencies for a given register: 33 33 + * 34 34 + * @feat_map describes the dependency for the whole register. If the 35 35 + * features the register depends on are not present, the whole 36 36 + * register is effectively RES0. 37 37 + * 38 38 + * @bit_feat_map describes the dependencies for a set of bits in that 39 39 + * register. If the features these bits depend on are not present, the 40 40 + * bits are effectively RES0. 41 41 + */ 42 42 + struct reg_feat_map_desc { 43 43 + const char *name; 44 44 + const struct reg_bits_to_feat_map feat_map; 45 45 + const struct reg_bits_to_feat_map *bit_feat_map; 46 46 + const unsigned int bit_feat_map_sz; 47 47 + }; 48 48 + 49 49 + #define __NEEDS_FEAT_3(m, f, w, id, fld, lim) \ 42 50 { \ 43 43 - .bits = (m), \ 51 51 + .w = (m), \ 44 52 .flags = (f), \ 45 53 .regidx = IDREG_IDX(SYS_ ## id), \ 46 54 .shift = id ##_## fld ## _SHIFT, \ ··· 67 39 .lo_lim = id ##_## fld ##_## lim \ 68 40 } 69 41 70 70 - #define __NEEDS_FEAT_2(m, f, fun, dummy) \ 42 42 + #define __NEEDS_FEAT_2(m, f, w, fun, dummy) \ 71 43 { \ 72 72 - .bits = (m), \ 44 44 + .w = (m), \ 73 45 .flags = (f) | CALL_FUNC, \ 74 46 .fval = (fun), \ 75 47 } 76 48 77 77 - #define __NEEDS_FEAT_1(m, f, fun) \ 49 49 + #define __NEEDS_FEAT_1(m, f, w, fun) \ 78 50 { \ 79 79 - .bits = (m), \ 51 51 + .w = (m), \ 80 52 .flags = (f) | CALL_FUNC, \ 81 53 .match = (fun), \ 82 54 } 83 55 56 56 + #define __NEEDS_FEAT_FLAG(m, f, w, ...) \ 57 57 + CONCATENATE(__NEEDS_FEAT_, COUNT_ARGS(__VA_ARGS__))(m, f, w, __VA_ARGS__) 58 58 + 84 59 #define NEEDS_FEAT_FLAG(m, f, ...) \ 85 85 - CONCATENATE(__NEEDS_FEAT_, COUNT_ARGS(__VA_ARGS__))(m, f, __VA_ARGS__) 60 60 + __NEEDS_FEAT_FLAG(m, f, bits, __VA_ARGS__) 86 61 87 62 #define NEEDS_FEAT_FIXED(m, ...) \ 88 88 - NEEDS_FEAT_FLAG(m, FIXED_VALUE, __VA_ARGS__, 0) 63 63 + __NEEDS_FEAT_FLAG(m, FIXED_VALUE, bits, __VA_ARGS__, 0) 89 64 65 65 + #define NEEDS_FEAT_RES0(p, ...) \ 66 66 + __NEEDS_FEAT_FLAG(p, RES0_POINTER, res0p, __VA_ARGS__) 67 67 + 68 68 + /* 69 69 + * Declare the dependency between a set of bits and a set of features, 70 70 + * generating a struct reg_bit_to_feat_map. 71 71 + */ 90 72 #define NEEDS_FEAT(m, ...) NEEDS_FEAT_FLAG(m, 0, __VA_ARGS__) 73 73 + 74 74 + /* 75 75 + * Declare the dependency between a non-FGT register, a set of 76 76 + * feature, and the set of individual bits it contains. This generates 77 77 + * a struct reg_feat_map_desc. 78 78 + */ 79 79 + #define DECLARE_FEAT_MAP(n, r, m, f) \ 80 80 + struct reg_feat_map_desc n = { \ 81 81 + .name = #r, \ 82 82 + .feat_map = NEEDS_FEAT(~r##_RES0, f), \ 83 83 + .bit_feat_map = m, \ 84 84 + .bit_feat_map_sz = ARRAY_SIZE(m), \ 85 85 + } 86 86 + 87 87 + /* 88 88 + * Specialised version of the above for FGT registers that have their 89 89 + * RES0 masks described as struct fgt_masks. 90 90 + */ 91 91 + #define DECLARE_FEAT_MAP_FGT(n, msk, m, f) \ 92 92 + struct reg_feat_map_desc n = { \ 93 93 + .name = #msk, \ 94 94 + .feat_map = NEEDS_FEAT_RES0(&msk.res0, f),\ 95 95 + .bit_feat_map = m, \ 96 96 + .bit_feat_map_sz = ARRAY_SIZE(m), \ 97 97 + } 91 98 92 99 #define FEAT_SPE ID_AA64DFR0_EL1, PMSVer, IMP 93 100 #define FEAT_SPE_FnE ID_AA64DFR0_EL1, PMSVer, V1P2 ··· 136 73 #define FEAT_AA32EL0 ID_AA64PFR0_EL1, EL0, AARCH32 137 74 #define FEAT_AA32EL1 ID_AA64PFR0_EL1, EL1, AARCH32 138 75 #define FEAT_AA64EL1 ID_AA64PFR0_EL1, EL1, IMP 76 76 + #define FEAT_AA64EL2 ID_AA64PFR0_EL1, EL2, IMP 139 77 #define FEAT_AA64EL3 ID_AA64PFR0_EL1, EL3, IMP 140 78 #define FEAT_AIE ID_AA64MMFR3_EL1, AIE, IMP 141 79 #define FEAT_S2POE ID_AA64MMFR3_EL1, S2POE, IMP ··· 195 131 #define FEAT_SPMU ID_AA64DFR1_EL1, SPMU, IMP 196 132 #define FEAT_SPE_nVM ID_AA64DFR2_EL1, SPE_nVM, IMP 197 133 #define FEAT_STEP2 ID_AA64DFR2_EL1, STEP, IMP 198 198 - #define FEAT_SYSREG128 ID_AA64ISAR2_EL1, SYSREG_128, IMP 199 134 #define FEAT_CPA2 ID_AA64ISAR3_EL1, CPA, CPA2 200 135 #define FEAT_ASID2 ID_AA64MMFR4_EL1, ASID2, IMP 201 136 #define FEAT_MEC ID_AA64MMFR3_EL1, MEC, IMP ··· 206 143 #define FEAT_LSMAOC ID_AA64MMFR2_EL1, LSM, IMP 207 144 #define FEAT_MixedEnd ID_AA64MMFR0_EL1, BIGEND, IMP 208 145 #define FEAT_MixedEndEL0 ID_AA64MMFR0_EL1, BIGENDEL0, IMP 209 209 - #define FEAT_MTE2 ID_AA64PFR1_EL1, MTE, MTE2 210 146 #define FEAT_MTE_ASYNC ID_AA64PFR1_EL1, MTE_frac, ASYNC 211 147 #define FEAT_MTE_STORE_ONLY ID_AA64PFR2_EL1, MTESTOREONLY, IMP 212 148 #define FEAT_PAN ID_AA64MMFR1_EL1, PAN, IMP ··· 213 151 #define FEAT_SSBS ID_AA64PFR1_EL1, SSBS, IMP 214 152 #define FEAT_TIDCP1 ID_AA64MMFR1_EL1, TIDCP1, IMP 215 153 #define FEAT_FGT ID_AA64MMFR0_EL1, FGT, IMP 154 154 + #define FEAT_FGT2 ID_AA64MMFR0_EL1, FGT, FGT2 216 155 #define FEAT_MTPMU ID_AA64DFR0_EL1, MTPMU, IMP 156 156 + #define FEAT_HCX ID_AA64MMFR1_EL1, HCX, IMP 217 157 218 158 static bool not_feat_aa64el3(struct kvm *kvm) 219 159 { ··· 461 397 NEVER_FGU, FEAT_AA64EL1), 462 398 }; 463 399 400 400 + 401 401 + static const DECLARE_FEAT_MAP_FGT(hfgrtr_desc, hfgrtr_masks, 402 402 + hfgrtr_feat_map, FEAT_FGT); 403 403 + 464 404 static const struct reg_bits_to_feat_map hfgwtr_feat_map[] = { 465 405 NEEDS_FEAT(HFGWTR_EL2_nAMAIR2_EL1 | 466 406 HFGWTR_EL2_nMAIR2_EL1, ··· 528 460 HFGWTR_EL2_AFSR0_EL1, 529 461 NEVER_FGU, FEAT_AA64EL1), 530 462 }; 463 463 + 464 464 + static const DECLARE_FEAT_MAP_FGT(hfgwtr_desc, hfgwtr_masks, 465 465 + hfgwtr_feat_map, FEAT_FGT); 531 466 532 467 static const struct reg_bits_to_feat_map hdfgrtr_feat_map[] = { 533 468 NEEDS_FEAT(HDFGRTR_EL2_PMBIDR_EL1 | ··· 599 528 NEVER_FGU, FEAT_AA64EL1) 600 529 }; 601 530 531 531 + static const DECLARE_FEAT_MAP_FGT(hdfgrtr_desc, hdfgrtr_masks, 532 532 + hdfgrtr_feat_map, FEAT_FGT); 533 533 + 602 534 static const struct reg_bits_to_feat_map hdfgwtr_feat_map[] = { 603 535 NEEDS_FEAT(HDFGWTR_EL2_PMSLATFR_EL1 | 604 536 HDFGWTR_EL2_PMSIRR_EL1 | ··· 662 588 NEEDS_FEAT(HDFGWTR_EL2_TRFCR_EL1, FEAT_TRF), 663 589 }; 664 590 591 591 + static const DECLARE_FEAT_MAP_FGT(hdfgwtr_desc, hdfgwtr_masks, 592 592 + hdfgwtr_feat_map, FEAT_FGT); 665 593 666 594 static const struct reg_bits_to_feat_map hfgitr_feat_map[] = { 667 595 NEEDS_FEAT(HFGITR_EL2_PSBCSYNC, FEAT_SPEv1p5), ··· 738 662 NEVER_FGU, FEAT_AA64EL1), 739 663 }; 740 664 665 665 + static const DECLARE_FEAT_MAP_FGT(hfgitr_desc, hfgitr_masks, 666 666 + hfgitr_feat_map, FEAT_FGT); 667 667 + 741 668 static const struct reg_bits_to_feat_map hafgrtr_feat_map[] = { 742 669 NEEDS_FEAT(HAFGRTR_EL2_AMEVTYPER115_EL0 | 743 670 HAFGRTR_EL2_AMEVTYPER114_EL0 | ··· 783 704 FEAT_AMUv1), 784 705 }; 785 706 707 707 + static const DECLARE_FEAT_MAP_FGT(hafgrtr_desc, hafgrtr_masks, 708 708 + hafgrtr_feat_map, FEAT_FGT); 709 709 + 786 710 static const struct reg_bits_to_feat_map hfgitr2_feat_map[] = { 787 711 NEEDS_FEAT(HFGITR2_EL2_nDCCIVAPS, FEAT_PoPS), 788 712 NEEDS_FEAT(HFGITR2_EL2_TSBCSYNC, FEAT_TRBEv1p1) 789 713 }; 714 714 + 715 715 + static const DECLARE_FEAT_MAP_FGT(hfgitr2_desc, hfgitr2_masks, 716 716 + hfgitr2_feat_map, FEAT_FGT2); 790 717 791 718 static const struct reg_bits_to_feat_map hfgrtr2_feat_map[] = { 792 719 NEEDS_FEAT(HFGRTR2_EL2_nPFAR_EL1, FEAT_PFAR), ··· 813 728 NEEDS_FEAT(HFGRTR2_EL2_nRCWSMASK_EL1, FEAT_THE), 814 729 }; 815 730 731 731 + static const DECLARE_FEAT_MAP_FGT(hfgrtr2_desc, hfgrtr2_masks, 732 732 + hfgrtr2_feat_map, FEAT_FGT2); 733 733 + 816 734 static const struct reg_bits_to_feat_map hfgwtr2_feat_map[] = { 817 735 NEEDS_FEAT(HFGWTR2_EL2_nPFAR_EL1, FEAT_PFAR), 818 736 NEEDS_FEAT(HFGWTR2_EL2_nACTLRALIAS_EL1 | ··· 833 745 FEAT_SRMASK), 834 746 NEEDS_FEAT(HFGWTR2_EL2_nRCWSMASK_EL1, FEAT_THE), 835 747 }; 748 748 + 749 749 + static const DECLARE_FEAT_MAP_FGT(hfgwtr2_desc, hfgwtr2_masks, 750 750 + hfgwtr2_feat_map, FEAT_FGT2); 836 751 837 752 static const struct reg_bits_to_feat_map hdfgrtr2_feat_map[] = { 838 753 NEEDS_FEAT(HDFGRTR2_EL2_nMDSELR_EL1, FEAT_Debugv8p9), ··· 867 776 NEEDS_FEAT(HDFGRTR2_EL2_nTRBMPAM_EL1, feat_trbe_mpam), 868 777 }; 869 778 779 779 + static const DECLARE_FEAT_MAP_FGT(hdfgrtr2_desc, hdfgrtr2_masks, 780 780 + hdfgrtr2_feat_map, FEAT_FGT2); 781 781 + 870 782 static const struct reg_bits_to_feat_map hdfgwtr2_feat_map[] = { 871 783 NEEDS_FEAT(HDFGWTR2_EL2_nMDSELR_EL1, FEAT_Debugv8p9), 872 784 NEEDS_FEAT(HDFGWTR2_EL2_nPMECR_EL1, feat_ebep_pmuv3_ss), ··· 897 803 NEEDS_FEAT(HDFGWTR2_EL2_nMDSTEPOP_EL1, FEAT_STEP2), 898 804 NEEDS_FEAT(HDFGWTR2_EL2_nTRBMPAM_EL1, feat_trbe_mpam), 899 805 }; 806 806 + 807 807 + static const DECLARE_FEAT_MAP_FGT(hdfgwtr2_desc, hdfgwtr2_masks, 808 808 + hdfgwtr2_feat_map, FEAT_FGT2); 809 809 + 900 810 901 811 static const struct reg_bits_to_feat_map hcrx_feat_map[] = { 902 812 NEEDS_FEAT(HCRX_EL2_PACMEn, feat_pauth_lr), ··· 930 832 NEEDS_FEAT(HCRX_EL2_EnALS, FEAT_LS64), 931 833 NEEDS_FEAT(HCRX_EL2_EnAS0, FEAT_LS64_ACCDATA), 932 834 }; 835 835 + 836 836 + 837 837 + static const DECLARE_FEAT_MAP(hcrx_desc, __HCRX_EL2, 838 838 + hcrx_feat_map, FEAT_HCX); 933 839 934 840 static const struct reg_bits_to_feat_map hcr_feat_map[] = { 935 841 NEEDS_FEAT(HCR_EL2_TID0, FEAT_AA32EL0), ··· 1006 904 NEEDS_FEAT_FIXED(HCR_EL2_E2H, compute_hcr_e2h), 1007 905 }; 1008 906 907 907 + static const DECLARE_FEAT_MAP(hcr_desc, HCR_EL2, 908 908 + hcr_feat_map, FEAT_AA64EL2); 909 909 + 1009 910 static const struct reg_bits_to_feat_map sctlr2_feat_map[] = { 1010 911 NEEDS_FEAT(SCTLR2_EL1_NMEA | 1011 912 SCTLR2_EL1_EASE, ··· 1025 920 SCTLR2_EL1_CPTM0, 1026 921 FEAT_CPA2), 1027 922 }; 923 923 + 924 924 + static const DECLARE_FEAT_MAP(sctlr2_desc, SCTLR2_EL1, 925 925 + sctlr2_feat_map, FEAT_SCTLR2); 1028 926 1029 927 static const struct reg_bits_to_feat_map tcr2_el2_feat_map[] = { 1030 928 NEEDS_FEAT(TCR2_EL2_FNG1 | ··· 1050 942 FEAT_S1POE), 1051 943 NEEDS_FEAT(TCR2_EL2_PIE, FEAT_S1PIE), 1052 944 }; 945 945 + 946 946 + static const DECLARE_FEAT_MAP(tcr2_el2_desc, TCR2_EL2, 947 947 + tcr2_el2_feat_map, FEAT_TCR2); 1053 948 1054 949 static const struct reg_bits_to_feat_map sctlr_el1_feat_map[] = { 1055 950 NEEDS_FEAT(SCTLR_EL1_CP15BEN | ··· 1128 1017 FEAT_AA64EL1), 1129 1018 }; 1130 1019 1020 1020 + static const DECLARE_FEAT_MAP(sctlr_el1_desc, SCTLR_EL1, 1021 1021 + sctlr_el1_feat_map, FEAT_AA64EL1); 1022 1022 + 1131 1023 static const struct reg_bits_to_feat_map mdcr_el2_feat_map[] = { 1132 1024 NEEDS_FEAT(MDCR_EL2_EBWE, FEAT_Debugv8p9), 1133 1025 NEEDS_FEAT(MDCR_EL2_TDOSA, FEAT_DoubleLock), ··· 1162 1048 FEAT_AA64EL1), 1163 1049 }; 1164 1050 1051 1051 + static const DECLARE_FEAT_MAP(mdcr_el2_desc, MDCR_EL2, 1052 1052 + mdcr_el2_feat_map, FEAT_AA64EL2); 1053 1053 + 1165 1054 static void __init check_feat_map(const struct reg_bits_to_feat_map *map, 1166 1055 int map_size, u64 res0, const char *str) 1167 1056 { ··· 1178 1061 str, mask ^ ~res0); 1179 1062 } 1180 1063 1064 1064 + static u64 reg_feat_map_bits(const struct reg_bits_to_feat_map *map) 1065 1065 + { 1066 1066 + return map->flags & RES0_POINTER ? ~(*map->res0p) : map->bits; 1067 1067 + } 1068 1068 + 1069 1069 + static void __init check_reg_desc(const struct reg_feat_map_desc *r) 1070 1070 + { 1071 1071 + check_feat_map(r->bit_feat_map, r->bit_feat_map_sz, 1072 1072 + ~reg_feat_map_bits(&r->feat_map), r->name); 1073 1073 + } 1074 1074 + 1181 1075 void __init check_feature_map(void) 1182 1076 { 1183 1183 - check_feat_map(hfgrtr_feat_map, ARRAY_SIZE(hfgrtr_feat_map), 1184 1184 - hfgrtr_masks.res0, hfgrtr_masks.str); 1185 1185 - check_feat_map(hfgwtr_feat_map, ARRAY_SIZE(hfgwtr_feat_map), 1186 1186 - hfgwtr_masks.res0, hfgwtr_masks.str); 1187 1187 - check_feat_map(hfgitr_feat_map, ARRAY_SIZE(hfgitr_feat_map), 1188 1188 - hfgitr_masks.res0, hfgitr_masks.str); 1189 1189 - check_feat_map(hdfgrtr_feat_map, ARRAY_SIZE(hdfgrtr_feat_map), 1190 1190 - hdfgrtr_masks.res0, hdfgrtr_masks.str); 1191 1191 - check_feat_map(hdfgwtr_feat_map, ARRAY_SIZE(hdfgwtr_feat_map), 1192 1192 - hdfgwtr_masks.res0, hdfgwtr_masks.str); 1193 1193 - check_feat_map(hafgrtr_feat_map, ARRAY_SIZE(hafgrtr_feat_map), 1194 1194 - hafgrtr_masks.res0, hafgrtr_masks.str); 1195 1195 - check_feat_map(hcrx_feat_map, ARRAY_SIZE(hcrx_feat_map), 1196 1196 - __HCRX_EL2_RES0, "HCRX_EL2"); 1197 1197 - check_feat_map(hcr_feat_map, ARRAY_SIZE(hcr_feat_map), 1198 1198 - HCR_EL2_RES0, "HCR_EL2"); 1199 1199 - check_feat_map(sctlr2_feat_map, ARRAY_SIZE(sctlr2_feat_map), 1200 1200 - SCTLR2_EL1_RES0, "SCTLR2_EL1"); 1201 1201 - check_feat_map(tcr2_el2_feat_map, ARRAY_SIZE(tcr2_el2_feat_map), 1202 1202 - TCR2_EL2_RES0, "TCR2_EL2"); 1203 1203 - check_feat_map(sctlr_el1_feat_map, ARRAY_SIZE(sctlr_el1_feat_map), 1204 1204 - SCTLR_EL1_RES0, "SCTLR_EL1"); 1205 1205 - check_feat_map(mdcr_el2_feat_map, ARRAY_SIZE(mdcr_el2_feat_map), 1206 1206 - MDCR_EL2_RES0, "MDCR_EL2"); 1077 1077 + check_reg_desc(&hfgrtr_desc); 1078 1078 + check_reg_desc(&hfgwtr_desc); 1079 1079 + check_reg_desc(&hfgitr_desc); 1080 1080 + check_reg_desc(&hdfgrtr_desc); 1081 1081 + check_reg_desc(&hdfgwtr_desc); 1082 1082 + check_reg_desc(&hafgrtr_desc); 1083 1083 + check_reg_desc(&hfgrtr2_desc); 1084 1084 + check_reg_desc(&hfgwtr2_desc); 1085 1085 + check_reg_desc(&hfgitr2_desc); 1086 1086 + check_reg_desc(&hdfgrtr2_desc); 1087 1087 + check_reg_desc(&hdfgwtr2_desc); 1088 1088 + check_reg_desc(&hcrx_desc); 1089 1089 + check_reg_desc(&hcr_desc); 1090 1090 + check_reg_desc(&sctlr2_desc); 1091 1091 + check_reg_desc(&tcr2_el2_desc); 1092 1092 + check_reg_desc(&sctlr_el1_desc); 1093 1093 + check_reg_desc(&mdcr_el2_desc); 1207 1094 } 1208 1095 1209 1096 static bool idreg_feat_match(struct kvm *kvm, const struct reg_bits_to_feat_map *map) ··· 1250 1129 match = idreg_feat_match(kvm, &map[i]); 1251 1130 1252 1131 if (!match || (map[i].flags & FIXED_VALUE)) 1253 1253 - val |= map[i].bits; 1132 1132 + val |= reg_feat_map_bits(&map[i]); 1254 1133 } 1255 1134 1256 1135 return val; ··· 1266 1145 require, exclude | FIXED_VALUE); 1267 1146 } 1268 1147 1269 1269 - static u64 compute_fixed_bits(struct kvm *kvm, 1270 1270 - const struct reg_bits_to_feat_map *map, 1271 1271 - int map_size, 1272 1272 - u64 *fixed_bits, 1273 1273 - unsigned long require, 1274 1274 - unsigned long exclude) 1148 1148 + static u64 compute_reg_res0_bits(struct kvm *kvm, 1149 1149 + const struct reg_feat_map_desc *r, 1150 1150 + unsigned long require, unsigned long exclude) 1151 1151 + 1275 1152 { 1276 1276 - return __compute_fixed_bits(kvm, map, map_size, fixed_bits, 1277 1277 - require | FIXED_VALUE, exclude); 1153 1153 + u64 res0; 1154 1154 + 1155 1155 + res0 = compute_res0_bits(kvm, r->bit_feat_map, r->bit_feat_map_sz, 1156 1156 + require, exclude); 1157 1157 + 1158 1158 + /* 1159 1159 + * If computing FGUs, don't take RES0 or register existence 1160 1160 + * into account -- we're not computing bits for the register 1161 1161 + * itself. 1162 1162 + */ 1163 1163 + if (!(exclude & NEVER_FGU)) { 1164 1164 + res0 |= compute_res0_bits(kvm, &r->feat_map, 1, require, exclude); 1165 1165 + res0 |= ~reg_feat_map_bits(&r->feat_map); 1166 1166 + } 1167 1167 + 1168 1168 + return res0; 1169 1169 + } 1170 1170 + 1171 1171 + static u64 compute_reg_fixed_bits(struct kvm *kvm, 1172 1172 + const struct reg_feat_map_desc *r, 1173 1173 + u64 *fixed_bits, unsigned long require, 1174 1174 + unsigned long exclude) 1175 1175 + { 1176 1176 + return __compute_fixed_bits(kvm, r->bit_feat_map, r->bit_feat_map_sz, 1177 1177 + fixed_bits, require | FIXED_VALUE, exclude); 1278 1178 } 1279 1179 1280 1180 void compute_fgu(struct kvm *kvm, enum fgt_group_id fgt) ··· 1304 1162 1305 1163 switch (fgt) { 1306 1164 case HFGRTR_GROUP: 1307 1307 - val |= compute_res0_bits(kvm, hfgrtr_feat_map, 1308 1308 - ARRAY_SIZE(hfgrtr_feat_map), 1309 1309 - 0, NEVER_FGU); 1310 1310 - val |= compute_res0_bits(kvm, hfgwtr_feat_map, 1311 1311 - ARRAY_SIZE(hfgwtr_feat_map), 1312 1312 - 0, NEVER_FGU); 1165 1165 + val |= compute_reg_res0_bits(kvm, &hfgrtr_desc, 1166 1166 + 0, NEVER_FGU); 1167 1167 + val |= compute_reg_res0_bits(kvm, &hfgwtr_desc, 1168 1168 + 0, NEVER_FGU); 1313 1169 break; 1314 1170 case HFGITR_GROUP: 1315 1315 - val |= compute_res0_bits(kvm, hfgitr_feat_map, 1316 1316 - ARRAY_SIZE(hfgitr_feat_map), 1317 1317 - 0, NEVER_FGU); 1171 1171 + val |= compute_reg_res0_bits(kvm, &hfgitr_desc, 1172 1172 + 0, NEVER_FGU); 1318 1173 break; 1319 1174 case HDFGRTR_GROUP: 1320 1320 - val |= compute_res0_bits(kvm, hdfgrtr_feat_map, 1321 1321 - ARRAY_SIZE(hdfgrtr_feat_map), 1322 1322 - 0, NEVER_FGU); 1323 1323 - val |= compute_res0_bits(kvm, hdfgwtr_feat_map, 1324 1324 - ARRAY_SIZE(hdfgwtr_feat_map), 1325 1325 - 0, NEVER_FGU); 1175 1175 + val |= compute_reg_res0_bits(kvm, &hdfgrtr_desc, 1176 1176 + 0, NEVER_FGU); 1177 1177 + val |= compute_reg_res0_bits(kvm, &hdfgwtr_desc, 1178 1178 + 0, NEVER_FGU); 1326 1179 break; 1327 1180 case HAFGRTR_GROUP: 1328 1328 - val |= compute_res0_bits(kvm, hafgrtr_feat_map, 1329 1329 - ARRAY_SIZE(hafgrtr_feat_map), 1330 1330 - 0, NEVER_FGU); 1181 1181 + val |= compute_reg_res0_bits(kvm, &hafgrtr_desc, 1182 1182 + 0, NEVER_FGU); 1331 1183 break; 1332 1184 case HFGRTR2_GROUP: 1333 1333 - val |= compute_res0_bits(kvm, hfgrtr2_feat_map, 1334 1334 - ARRAY_SIZE(hfgrtr2_feat_map), 1335 1335 - 0, NEVER_FGU); 1336 1336 - val |= compute_res0_bits(kvm, hfgwtr2_feat_map, 1337 1337 - ARRAY_SIZE(hfgwtr2_feat_map), 1338 1338 - 0, NEVER_FGU); 1185 1185 + val |= compute_reg_res0_bits(kvm, &hfgrtr2_desc, 1186 1186 + 0, NEVER_FGU); 1187 1187 + val |= compute_reg_res0_bits(kvm, &hfgwtr2_desc, 1188 1188 + 0, NEVER_FGU); 1339 1189 break; 1340 1190 case HFGITR2_GROUP: 1341 1341 - val |= compute_res0_bits(kvm, hfgitr2_feat_map, 1342 1342 - ARRAY_SIZE(hfgitr2_feat_map), 1343 1343 - 0, NEVER_FGU); 1191 1191 + val |= compute_reg_res0_bits(kvm, &hfgitr2_desc, 1192 1192 + 0, NEVER_FGU); 1344 1193 break; 1345 1194 case HDFGRTR2_GROUP: 1346 1346 - val |= compute_res0_bits(kvm, hdfgrtr2_feat_map, 1347 1347 - ARRAY_SIZE(hdfgrtr2_feat_map), 1348 1348 - 0, NEVER_FGU); 1349 1349 - val |= compute_res0_bits(kvm, hdfgwtr2_feat_map, 1350 1350 - ARRAY_SIZE(hdfgwtr2_feat_map), 1351 1351 - 0, NEVER_FGU); 1195 1195 + val |= compute_reg_res0_bits(kvm, &hdfgrtr2_desc, 1196 1196 + 0, NEVER_FGU); 1197 1197 + val |= compute_reg_res0_bits(kvm, &hdfgwtr2_desc, 1198 1198 + 0, NEVER_FGU); 1352 1199 break; 1353 1200 default: 1354 1201 BUG(); ··· 1352 1221 1353 1222 switch (reg) { 1354 1223 case HFGRTR_EL2: 1355 1355 - *res0 = compute_res0_bits(kvm, hfgrtr_feat_map, 1356 1356 - ARRAY_SIZE(hfgrtr_feat_map), 0, 0); 1357 1357 - *res0 |= hfgrtr_masks.res0; 1224 1224 + *res0 = compute_reg_res0_bits(kvm, &hfgrtr_desc, 0, 0); 1358 1225 *res1 = HFGRTR_EL2_RES1; 1359 1226 break; 1360 1227 case HFGWTR_EL2: 1361 1361 - *res0 = compute_res0_bits(kvm, hfgwtr_feat_map, 1362 1362 - ARRAY_SIZE(hfgwtr_feat_map), 0, 0); 1363 1363 - *res0 |= hfgwtr_masks.res0; 1228 1228 + *res0 = compute_reg_res0_bits(kvm, &hfgwtr_desc, 0, 0); 1364 1229 *res1 = HFGWTR_EL2_RES1; 1365 1230 break; 1366 1231 case HFGITR_EL2: 1367 1367 - *res0 = compute_res0_bits(kvm, hfgitr_feat_map, 1368 1368 - ARRAY_SIZE(hfgitr_feat_map), 0, 0); 1369 1369 - *res0 |= hfgitr_masks.res0; 1232 1232 + *res0 = compute_reg_res0_bits(kvm, &hfgitr_desc, 0, 0); 1370 1233 *res1 = HFGITR_EL2_RES1; 1371 1234 break; 1372 1235 case HDFGRTR_EL2: 1373 1373 - *res0 = compute_res0_bits(kvm, hdfgrtr_feat_map, 1374 1374 - ARRAY_SIZE(hdfgrtr_feat_map), 0, 0); 1375 1375 - *res0 |= hdfgrtr_masks.res0; 1236 1236 + *res0 = compute_reg_res0_bits(kvm, &hdfgrtr_desc, 0, 0); 1376 1237 *res1 = HDFGRTR_EL2_RES1; 1377 1238 break; 1378 1239 case HDFGWTR_EL2: 1379 1379 - *res0 = compute_res0_bits(kvm, hdfgwtr_feat_map, 1380 1380 - ARRAY_SIZE(hdfgwtr_feat_map), 0, 0); 1381 1381 - *res0 |= hdfgwtr_masks.res0; 1240 1240 + *res0 = compute_reg_res0_bits(kvm, &hdfgwtr_desc, 0, 0); 1382 1241 *res1 = HDFGWTR_EL2_RES1; 1383 1242 break; 1384 1243 case HAFGRTR_EL2: 1385 1385 - *res0 = compute_res0_bits(kvm, hafgrtr_feat_map, 1386 1386 - ARRAY_SIZE(hafgrtr_feat_map), 0, 0); 1387 1387 - *res0 |= hafgrtr_masks.res0; 1244 1244 + *res0 = compute_reg_res0_bits(kvm, &hafgrtr_desc, 0, 0); 1388 1245 *res1 = HAFGRTR_EL2_RES1; 1389 1246 break; 1390 1247 case HFGRTR2_EL2: 1391 1391 - *res0 = compute_res0_bits(kvm, hfgrtr2_feat_map, 1392 1392 - ARRAY_SIZE(hfgrtr2_feat_map), 0, 0); 1393 1393 - *res0 |= hfgrtr2_masks.res0; 1248 1248 + *res0 = compute_reg_res0_bits(kvm, &hfgrtr2_desc, 0, 0); 1394 1249 *res1 = HFGRTR2_EL2_RES1; 1395 1250 break; 1396 1251 case HFGWTR2_EL2: 1397 1397 - *res0 = compute_res0_bits(kvm, hfgwtr2_feat_map, 1398 1398 - ARRAY_SIZE(hfgwtr2_feat_map), 0, 0); 1399 1399 - *res0 |= hfgwtr2_masks.res0; 1252 1252 + *res0 = compute_reg_res0_bits(kvm, &hfgwtr2_desc, 0, 0); 1400 1253 *res1 = HFGWTR2_EL2_RES1; 1401 1254 break; 1402 1255 case HFGITR2_EL2: 1403 1403 - *res0 = compute_res0_bits(kvm, hfgitr2_feat_map, 1404 1404 - ARRAY_SIZE(hfgitr2_feat_map), 0, 0); 1405 1405 - *res0 |= hfgitr2_masks.res0; 1256 1256 + *res0 = compute_reg_res0_bits(kvm, &hfgitr2_desc, 0, 0); 1406 1257 *res1 = HFGITR2_EL2_RES1; 1407 1258 break; 1408 1259 case HDFGRTR2_EL2: 1409 1409 - *res0 = compute_res0_bits(kvm, hdfgrtr2_feat_map, 1410 1410 - ARRAY_SIZE(hdfgrtr2_feat_map), 0, 0); 1411 1411 - *res0 |= hdfgrtr2_masks.res0; 1260 1260 + *res0 = compute_reg_res0_bits(kvm, &hdfgrtr2_desc, 0, 0); 1412 1261 *res1 = HDFGRTR2_EL2_RES1; 1413 1262 break; 1414 1263 case HDFGWTR2_EL2: 1415 1415 - *res0 = compute_res0_bits(kvm, hdfgwtr2_feat_map, 1416 1416 - ARRAY_SIZE(hdfgwtr2_feat_map), 0, 0); 1417 1417 - *res0 |= hdfgwtr2_masks.res0; 1264 1264 + *res0 = compute_reg_res0_bits(kvm, &hdfgwtr2_desc, 0, 0); 1418 1265 *res1 = HDFGWTR2_EL2_RES1; 1419 1266 break; 1420 1267 case HCRX_EL2: 1421 1421 - *res0 = compute_res0_bits(kvm, hcrx_feat_map, 1422 1422 - ARRAY_SIZE(hcrx_feat_map), 0, 0); 1423 1423 - *res0 |= __HCRX_EL2_RES0; 1268 1268 + *res0 = compute_reg_res0_bits(kvm, &hcrx_desc, 0, 0); 1424 1269 *res1 = __HCRX_EL2_RES1; 1425 1270 break; 1426 1271 case HCR_EL2: 1427 1427 - mask = compute_fixed_bits(kvm, hcr_feat_map, 1428 1428 - ARRAY_SIZE(hcr_feat_map), &fixed, 1429 1429 - 0, 0); 1430 1430 - *res0 = compute_res0_bits(kvm, hcr_feat_map, 1431 1431 - ARRAY_SIZE(hcr_feat_map), 0, 0); 1432 1432 - *res0 |= HCR_EL2_RES0 | (mask & ~fixed); 1272 1272 + mask = compute_reg_fixed_bits(kvm, &hcr_desc, &fixed, 0, 0); 1273 1273 + *res0 = compute_reg_res0_bits(kvm, &hcr_desc, 0, 0); 1274 1274 + *res0 |= (mask & ~fixed); 1433 1275 *res1 = HCR_EL2_RES1 | (mask & fixed); 1434 1276 break; 1435 1277 case SCTLR2_EL1: 1436 1278 case SCTLR2_EL2: 1437 1437 - *res0 = compute_res0_bits(kvm, sctlr2_feat_map, 1438 1438 - ARRAY_SIZE(sctlr2_feat_map), 0, 0); 1439 1439 - *res0 |= SCTLR2_EL1_RES0; 1279 1279 + *res0 = compute_reg_res0_bits(kvm, &sctlr2_desc, 0, 0); 1440 1280 *res1 = SCTLR2_EL1_RES1; 1441 1281 break; 1442 1282 case TCR2_EL2: 1443 1443 - *res0 = compute_res0_bits(kvm, tcr2_el2_feat_map, 1444 1444 - ARRAY_SIZE(tcr2_el2_feat_map), 0, 0); 1445 1445 - *res0 |= TCR2_EL2_RES0; 1283 1283 + *res0 = compute_reg_res0_bits(kvm, &tcr2_el2_desc, 0, 0); 1446 1284 *res1 = TCR2_EL2_RES1; 1447 1285 break; 1448 1286 case SCTLR_EL1: 1449 1449 - *res0 = compute_res0_bits(kvm, sctlr_el1_feat_map, 1450 1450 - ARRAY_SIZE(sctlr_el1_feat_map), 0, 0); 1451 1451 - *res0 |= SCTLR_EL1_RES0; 1287 1287 + *res0 = compute_reg_res0_bits(kvm, &sctlr_el1_desc, 0, 0); 1452 1288 *res1 = SCTLR_EL1_RES1; 1453 1289 break; 1454 1290 case MDCR_EL2: 1455 1455 - *res0 = compute_res0_bits(kvm, mdcr_el2_feat_map, 1456 1456 - ARRAY_SIZE(mdcr_el2_feat_map), 0, 0); 1457 1457 - *res0 |= MDCR_EL2_RES0; 1291 1291 + *res0 = compute_reg_res0_bits(kvm, &mdcr_el2_desc, 0, 0); 1458 1292 *res1 = MDCR_EL2_RES1; 1459 1293 break; 1460 1294 default:

+14 -11

arch/arm64/kvm/debug.c

reviewed

··· 56 56 if (!kvm_guest_owns_debug_regs(vcpu)) 57 57 vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA; 58 58 59 59 + if (vcpu_has_nv(vcpu)) 60 60 + kvm_nested_setup_mdcr_el2(vcpu); 61 61 + 59 62 /* Write MDCR_EL2 directly if we're already at EL2 */ 60 63 if (has_vhe()) 61 64 write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2); ··· 246 243 preempt_enable(); 247 244 } 248 245 246 246 + static bool skip_trbe_access(bool skip_condition) 247 247 + { 248 248 + return (WARN_ON_ONCE(preemptible()) || skip_condition || 249 249 + is_protected_kvm_enabled() || !is_kvm_arm_initialised()); 250 250 + } 251 251 + 249 252 void kvm_enable_trbe(void) 250 253 { 251 251 - if (has_vhe() || is_protected_kvm_enabled() || 252 252 - WARN_ON_ONCE(preemptible())) 253 253 - return; 254 254 - 255 255 - host_data_set_flag(TRBE_ENABLED); 254 254 + if (!skip_trbe_access(has_vhe())) 255 255 + host_data_set_flag(TRBE_ENABLED); 256 256 } 257 257 EXPORT_SYMBOL_GPL(kvm_enable_trbe); 258 258 259 259 void kvm_disable_trbe(void) 260 260 { 261 261 - if (has_vhe() || is_protected_kvm_enabled() || 262 262 - WARN_ON_ONCE(preemptible())) 263 263 - return; 264 264 - 265 265 - host_data_clear_flag(TRBE_ENABLED); 261 261 + if (!skip_trbe_access(has_vhe())) 262 262 + host_data_clear_flag(TRBE_ENABLED); 266 263 } 267 264 EXPORT_SYMBOL_GPL(kvm_disable_trbe); 268 265 269 266 void kvm_tracing_set_el1_configuration(u64 trfcr_while_in_guest) 270 267 { 271 271 - if (is_protected_kvm_enabled() || WARN_ON_ONCE(preemptible())) 268 268 + if (skip_trbe_access(false)) 272 269 return; 273 270 274 271 if (has_vhe()) {

+1

arch/arm64/kvm/emulate-nested.c

reviewed

··· 1185 1185 SR_TRAP(SYS_PMSIRR_EL1, CGT_MDCR_TPMS), 1186 1186 SR_TRAP(SYS_PMSLATFR_EL1, CGT_MDCR_TPMS), 1187 1187 SR_TRAP(SYS_PMSNEVFR_EL1, CGT_MDCR_TPMS), 1188 1188 + SR_TRAP(SYS_PMSDSFR_EL1, CGT_MDCR_TPMS), 1188 1189 SR_TRAP(SYS_TRFCR_EL1, CGT_MDCR_TTRF), 1189 1190 SR_TRAP(SYS_TRBBASER_EL1, CGT_MDCR_E2TB), 1190 1191 SR_TRAP(SYS_TRBLIMITR_EL1, CGT_MDCR_E2TB),

+3

arch/arm64/kvm/handle_exit.c

reviewed

··· 559 559 /* Dump the nVHE hypervisor backtrace */ 560 560 kvm_nvhe_dump_backtrace(hyp_offset); 561 561 562 562 + /* Dump the faulting instruction */ 563 563 + dump_kernel_instr(panic_addr + kaslr_offset()); 564 564 + 562 565 /* 563 566 * Hyp has panicked and we're going to handle that by panicking the 564 567 * kernel. The kernel offset will be revealed in the panic so we're

+3 -1

arch/arm64/kvm/hyp/include/nvhe/pkvm.h

reviewed

··· 29 29 }; 30 30 31 31 /* 32 32 - * Holds the relevant data for running a protected vm. 32 32 + * Holds the relevant data for running a vm in protected mode. 33 33 */ 34 34 struct pkvm_hyp_vm { 35 35 struct kvm kvm; ··· 67 67 68 68 void pkvm_hyp_vm_table_init(void *tbl); 69 69 70 70 + int __pkvm_reserve_vm(void); 71 71 + void __pkvm_unreserve_vm(pkvm_handle_t handle); 70 72 int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, 71 73 unsigned long pgd_hva); 72 74 int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,

+2 -1

arch/arm64/kvm/hyp/include/nvhe/trap_handler.h

reviewed

··· 12 12 #include <asm/kvm_host.h> 13 13 14 14 #define cpu_reg(ctxt, r) (ctxt)->regs.regs[r] 15 15 - #define DECLARE_REG(type, name, ctxt, reg) \ 15 15 + #define DECLARE_REG(type, name, ctxt, reg) \ 16 16 + __always_unused int ___check_reg_ ## reg; \ 16 17 type name = (type)cpu_reg(ctxt, (reg)) 17 18 18 19 #endif /* __ARM64_KVM_NVHE_TRAP_HANDLER_H__ */

+1

arch/arm64/kvm/hyp/nvhe/Makefile

reviewed

··· 27 27 cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o 28 28 hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ 29 29 ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o 30 30 + hyp-obj-y += ../../../kernel/smccc-call.o 30 31 hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o 31 32 hyp-obj-y += $(lib-objs) 32 33

+144 -73

arch/arm64/kvm/hyp/nvhe/ffa.c

reviewed

··· 71 71 static bool has_version_negotiated; 72 72 static hyp_spinlock_t version_lock; 73 73 74 74 - static void ffa_to_smccc_error(struct arm_smccc_res *res, u64 ffa_errno) 74 74 + static void ffa_to_smccc_error(struct arm_smccc_1_2_regs *res, u64 ffa_errno) 75 75 { 76 76 - *res = (struct arm_smccc_res) { 76 76 + *res = (struct arm_smccc_1_2_regs) { 77 77 .a0 = FFA_ERROR, 78 78 .a2 = ffa_errno, 79 79 }; 80 80 } 81 81 82 82 - static void ffa_to_smccc_res_prop(struct arm_smccc_res *res, int ret, u64 prop) 82 82 + static void ffa_to_smccc_res_prop(struct arm_smccc_1_2_regs *res, int ret, u64 prop) 83 83 { 84 84 if (ret == FFA_RET_SUCCESS) { 85 85 - *res = (struct arm_smccc_res) { .a0 = FFA_SUCCESS, 86 86 - .a2 = prop }; 85 85 + *res = (struct arm_smccc_1_2_regs) { .a0 = FFA_SUCCESS, 86 86 + .a2 = prop }; 87 87 } else { 88 88 ffa_to_smccc_error(res, ret); 89 89 } 90 90 } 91 91 92 92 - static void ffa_to_smccc_res(struct arm_smccc_res *res, int ret) 92 92 + static void ffa_to_smccc_res(struct arm_smccc_1_2_regs *res, int ret) 93 93 { 94 94 ffa_to_smccc_res_prop(res, ret, 0); 95 95 } 96 96 97 97 static void ffa_set_retval(struct kvm_cpu_context *ctxt, 98 98 - struct arm_smccc_res *res) 98 98 + struct arm_smccc_1_2_regs *res) 99 99 { 100 100 cpu_reg(ctxt, 0) = res->a0; 101 101 cpu_reg(ctxt, 1) = res->a1; 102 102 cpu_reg(ctxt, 2) = res->a2; 103 103 cpu_reg(ctxt, 3) = res->a3; 104 104 + cpu_reg(ctxt, 4) = res->a4; 105 105 + cpu_reg(ctxt, 5) = res->a5; 106 106 + cpu_reg(ctxt, 6) = res->a6; 107 107 + cpu_reg(ctxt, 7) = res->a7; 108 108 + 109 109 + /* 110 110 + * DEN0028C 2.6: SMC32/HVC32 call from aarch64 must preserve x8-x30. 111 111 + * 112 112 + * In FF-A 1.2, we cannot rely on the function ID sent by the caller to 113 113 + * detect 32-bit calls because the CPU cycle management interfaces (e.g. 114 114 + * FFA_MSG_WAIT, FFA_RUN) are 32-bit only but can have 64-bit responses. 115 115 + * 116 116 + * FFA-1.3 introduces 64-bit variants of the CPU cycle management 117 117 + * interfaces. Moreover, FF-A 1.3 clarifies that SMC32 direct requests 118 118 + * complete with SMC32 direct reponses which *should* allow us use the 119 119 + * function ID sent by the caller to determine whether to return x8-x17. 120 120 + * 121 121 + * Note that we also cannot rely on function IDs in the response. 122 122 + * 123 123 + * Given the above, assume SMC64 and send back x0-x17 unconditionally 124 124 + * as the passthrough code (__kvm_hyp_host_forward_smc) does the same. 125 125 + */ 126 126 + cpu_reg(ctxt, 8) = res->a8; 127 127 + cpu_reg(ctxt, 9) = res->a9; 128 128 + cpu_reg(ctxt, 10) = res->a10; 129 129 + cpu_reg(ctxt, 11) = res->a11; 130 130 + cpu_reg(ctxt, 12) = res->a12; 131 131 + cpu_reg(ctxt, 13) = res->a13; 132 132 + cpu_reg(ctxt, 14) = res->a14; 133 133 + cpu_reg(ctxt, 15) = res->a15; 134 134 + cpu_reg(ctxt, 16) = res->a16; 135 135 + cpu_reg(ctxt, 17) = res->a17; 104 136 } 105 137 106 138 static bool is_ffa_call(u64 func_id) ··· 145 113 146 114 static int ffa_map_hyp_buffers(u64 ffa_page_count) 147 115 { 148 148 - struct arm_smccc_res res; 116 116 + struct arm_smccc_1_2_regs res; 149 117 150 150 - arm_smccc_1_1_smc(FFA_FN64_RXTX_MAP, 151 151 - hyp_virt_to_phys(hyp_buffers.tx), 152 152 - hyp_virt_to_phys(hyp_buffers.rx), 153 153 - ffa_page_count, 154 154 - 0, 0, 0, 0, 155 155 - &res); 118 118 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 119 119 + .a0 = FFA_FN64_RXTX_MAP, 120 120 + .a1 = hyp_virt_to_phys(hyp_buffers.tx), 121 121 + .a2 = hyp_virt_to_phys(hyp_buffers.rx), 122 122 + .a3 = ffa_page_count, 123 123 + }, &res); 156 124 157 125 return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2; 158 126 } 159 127 160 128 static int ffa_unmap_hyp_buffers(void) 161 129 { 162 162 - struct arm_smccc_res res; 130 130 + struct arm_smccc_1_2_regs res; 163 131 164 164 - arm_smccc_1_1_smc(FFA_RXTX_UNMAP, 165 165 - HOST_FFA_ID, 166 166 - 0, 0, 0, 0, 0, 0, 167 167 - &res); 132 132 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 133 133 + .a0 = FFA_RXTX_UNMAP, 134 134 + .a1 = HOST_FFA_ID, 135 135 + }, &res); 168 136 169 137 return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2; 170 138 } 171 139 172 172 - static void ffa_mem_frag_tx(struct arm_smccc_res *res, u32 handle_lo, 140 140 + static void ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, u32 handle_lo, 173 141 u32 handle_hi, u32 fraglen, u32 endpoint_id) 174 142 { 175 175 - arm_smccc_1_1_smc(FFA_MEM_FRAG_TX, 176 176 - handle_lo, handle_hi, fraglen, endpoint_id, 177 177 - 0, 0, 0, 178 178 - res); 143 143 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 144 144 + .a0 = FFA_MEM_FRAG_TX, 145 145 + .a1 = handle_lo, 146 146 + .a2 = handle_hi, 147 147 + .a3 = fraglen, 148 148 + .a4 = endpoint_id, 149 149 + }, res); 179 150 } 180 151 181 181 - static void ffa_mem_frag_rx(struct arm_smccc_res *res, u32 handle_lo, 152 152 + static void ffa_mem_frag_rx(struct arm_smccc_1_2_regs *res, u32 handle_lo, 182 153 u32 handle_hi, u32 fragoff) 183 154 { 184 184 - arm_smccc_1_1_smc(FFA_MEM_FRAG_RX, 185 185 - handle_lo, handle_hi, fragoff, HOST_FFA_ID, 186 186 - 0, 0, 0, 187 187 - res); 155 155 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 156 156 + .a0 = FFA_MEM_FRAG_RX, 157 157 + .a1 = handle_lo, 158 158 + .a2 = handle_hi, 159 159 + .a3 = fragoff, 160 160 + .a4 = HOST_FFA_ID, 161 161 + }, res); 188 162 } 189 163 190 190 - static void ffa_mem_xfer(struct arm_smccc_res *res, u64 func_id, u32 len, 164 164 + static void ffa_mem_xfer(struct arm_smccc_1_2_regs *res, u64 func_id, u32 len, 191 165 u32 fraglen) 192 166 { 193 193 - arm_smccc_1_1_smc(func_id, len, fraglen, 194 194 - 0, 0, 0, 0, 0, 195 195 - res); 167 167 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 168 168 + .a0 = func_id, 169 169 + .a1 = len, 170 170 + .a2 = fraglen, 171 171 + }, res); 196 172 } 197 173 198 198 - static void ffa_mem_reclaim(struct arm_smccc_res *res, u32 handle_lo, 174 174 + static void ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, u32 handle_lo, 199 175 u32 handle_hi, u32 flags) 200 176 { 201 201 - arm_smccc_1_1_smc(FFA_MEM_RECLAIM, 202 202 - handle_lo, handle_hi, flags, 203 203 - 0, 0, 0, 0, 204 204 - res); 177 177 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 178 178 + .a0 = FFA_MEM_RECLAIM, 179 179 + .a1 = handle_lo, 180 180 + .a2 = handle_hi, 181 181 + .a3 = flags, 182 182 + }, res); 205 183 } 206 184 207 207 - static void ffa_retrieve_req(struct arm_smccc_res *res, u32 len) 185 185 + static void ffa_retrieve_req(struct arm_smccc_1_2_regs *res, u32 len) 208 186 { 209 209 - arm_smccc_1_1_smc(FFA_FN64_MEM_RETRIEVE_REQ, 210 210 - len, len, 211 211 - 0, 0, 0, 0, 0, 212 212 - res); 187 187 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 188 188 + .a0 = FFA_FN64_MEM_RETRIEVE_REQ, 189 189 + .a1 = len, 190 190 + .a2 = len, 191 191 + }, res); 213 192 } 214 193 215 215 - static void ffa_rx_release(struct arm_smccc_res *res) 194 194 + static void ffa_rx_release(struct arm_smccc_1_2_regs *res) 216 195 { 217 217 - arm_smccc_1_1_smc(FFA_RX_RELEASE, 218 218 - 0, 0, 219 219 - 0, 0, 0, 0, 0, 220 220 - res); 196 196 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 197 197 + .a0 = FFA_RX_RELEASE, 198 198 + }, res); 221 199 } 222 200 223 223 - static void do_ffa_rxtx_map(struct arm_smccc_res *res, 201 201 + static void do_ffa_rxtx_map(struct arm_smccc_1_2_regs *res, 224 202 struct kvm_cpu_context *ctxt) 225 203 { 226 204 DECLARE_REG(phys_addr_t, tx, ctxt, 1); ··· 309 267 goto out_unlock; 310 268 } 311 269 312 312 - static void do_ffa_rxtx_unmap(struct arm_smccc_res *res, 270 270 + static void do_ffa_rxtx_unmap(struct arm_smccc_1_2_regs *res, 313 271 struct kvm_cpu_context *ctxt) 314 272 { 315 273 DECLARE_REG(u32, id, ctxt, 1); ··· 410 368 return ret; 411 369 } 412 370 413 413 - static void do_ffa_mem_frag_tx(struct arm_smccc_res *res, 371 371 + static void do_ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, 414 372 struct kvm_cpu_context *ctxt) 415 373 { 416 374 DECLARE_REG(u32, handle_lo, ctxt, 1); ··· 469 427 } 470 428 471 429 static void __do_ffa_mem_xfer(const u64 func_id, 472 472 - struct arm_smccc_res *res, 430 430 + struct arm_smccc_1_2_regs *res, 473 431 struct kvm_cpu_context *ctxt) 474 432 { 475 433 DECLARE_REG(u32, len, ctxt, 1); ··· 563 521 __do_ffa_mem_xfer((fid), (res), (ctxt)); \ 564 522 } while (0); 565 523 566 566 - static void do_ffa_mem_reclaim(struct arm_smccc_res *res, 524 524 + static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, 567 525 struct kvm_cpu_context *ctxt) 568 526 { 569 527 DECLARE_REG(u32, handle_lo, ctxt, 1); ··· 670 628 case FFA_RXTX_MAP: 671 629 case FFA_MEM_DONATE: 672 630 case FFA_MEM_RETRIEVE_REQ: 631 631 + /* Optional notification interfaces added in FF-A 1.1 */ 632 632 + case FFA_NOTIFICATION_BITMAP_CREATE: 633 633 + case FFA_NOTIFICATION_BITMAP_DESTROY: 634 634 + case FFA_NOTIFICATION_BIND: 635 635 + case FFA_NOTIFICATION_UNBIND: 636 636 + case FFA_NOTIFICATION_SET: 637 637 + case FFA_NOTIFICATION_GET: 638 638 + case FFA_NOTIFICATION_INFO_GET: 639 639 + /* Optional interfaces added in FF-A 1.2 */ 640 640 + case FFA_MSG_SEND_DIRECT_REQ2: /* Optional per 7.5.1 */ 641 641 + case FFA_MSG_SEND_DIRECT_RESP2: /* Optional per 7.5.1 */ 642 642 + case FFA_CONSOLE_LOG: /* Optional per 13.1: not in Table 13.1 */ 643 643 + case FFA_PARTITION_INFO_GET_REGS: /* Optional for virtual instances per 13.1 */ 673 644 return false; 674 645 } 675 646 676 647 return true; 677 648 } 678 649 679 679 - static bool do_ffa_features(struct arm_smccc_res *res, 650 650 + static bool do_ffa_features(struct arm_smccc_1_2_regs *res, 680 651 struct kvm_cpu_context *ctxt) 681 652 { 682 653 DECLARE_REG(u32, id, ctxt, 1); ··· 721 666 static int hyp_ffa_post_init(void) 722 667 { 723 668 size_t min_rxtx_sz; 724 724 - struct arm_smccc_res res; 669 669 + struct arm_smccc_1_2_regs res; 725 670 726 726 - arm_smccc_1_1_smc(FFA_ID_GET, 0, 0, 0, 0, 0, 0, 0, &res); 671 671 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){ 672 672 + .a0 = FFA_ID_GET, 673 673 + }, &res); 727 674 if (res.a0 != FFA_SUCCESS) 728 675 return -EOPNOTSUPP; 729 676 730 677 if (res.a2 != HOST_FFA_ID) 731 678 return -EINVAL; 732 679 733 733 - arm_smccc_1_1_smc(FFA_FEATURES, FFA_FN64_RXTX_MAP, 734 734 - 0, 0, 0, 0, 0, 0, &res); 680 680 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){ 681 681 + .a0 = FFA_FEATURES, 682 682 + .a1 = FFA_FN64_RXTX_MAP, 683 683 + }, &res); 735 684 if (res.a0 != FFA_SUCCESS) 736 685 return -EOPNOTSUPP; 737 686 738 738 - switch (res.a2) { 687 687 + switch (res.a2 & FFA_FEAT_RXTX_MIN_SZ_MASK) { 739 688 case FFA_FEAT_RXTX_MIN_SZ_4K: 740 689 min_rxtx_sz = SZ_4K; 741 690 break; ··· 759 700 return 0; 760 701 } 761 702 762 762 - static void do_ffa_version(struct arm_smccc_res *res, 703 703 + static void do_ffa_version(struct arm_smccc_1_2_regs *res, 763 704 struct kvm_cpu_context *ctxt) 764 705 { 765 706 DECLARE_REG(u32, ffa_req_version, ctxt, 1); ··· 771 712 772 713 hyp_spin_lock(&version_lock); 773 714 if (has_version_negotiated) { 774 774 - res->a0 = hyp_ffa_version; 715 715 + if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version)) 716 716 + res->a0 = FFA_RET_NOT_SUPPORTED; 717 717 + else 718 718 + res->a0 = hyp_ffa_version; 775 719 goto unlock; 776 720 } 777 721 ··· 783 721 * first if TEE supports it. 784 722 */ 785 723 if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version)) { 786 786 - arm_smccc_1_1_smc(FFA_VERSION, ffa_req_version, 0, 787 787 - 0, 0, 0, 0, 0, 788 788 - res); 724 724 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 725 725 + .a0 = FFA_VERSION, 726 726 + .a1 = ffa_req_version, 727 727 + }, res); 789 728 if (res->a0 == FFA_RET_NOT_SUPPORTED) 790 729 goto unlock; 791 730 ··· 803 740 hyp_spin_unlock(&version_lock); 804 741 } 805 742 806 806 - static void do_ffa_part_get(struct arm_smccc_res *res, 743 743 + static void do_ffa_part_get(struct arm_smccc_1_2_regs *res, 807 744 struct kvm_cpu_context *ctxt) 808 745 { 809 746 DECLARE_REG(u32, uuid0, ctxt, 1); ··· 819 756 goto out_unlock; 820 757 } 821 758 822 822 - arm_smccc_1_1_smc(FFA_PARTITION_INFO_GET, uuid0, uuid1, 823 823 - uuid2, uuid3, flags, 0, 0, 824 824 - res); 759 759 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 760 760 + .a0 = FFA_PARTITION_INFO_GET, 761 761 + .a1 = uuid0, 762 762 + .a2 = uuid1, 763 763 + .a3 = uuid2, 764 764 + .a4 = uuid3, 765 765 + .a5 = flags, 766 766 + }, res); 825 767 826 768 if (res->a0 != FFA_SUCCESS) 827 769 goto out_unlock; ··· 859 791 860 792 bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) 861 793 { 862 862 - struct arm_smccc_res res; 794 794 + struct arm_smccc_1_2_regs res; 863 795 864 796 /* 865 797 * There's no way we can tell what a non-standard SMC call might ··· 928 860 929 861 int hyp_ffa_init(void *pages) 930 862 { 931 931 - struct arm_smccc_res res; 863 863 + struct arm_smccc_1_2_regs res; 932 864 void *tx, *rx; 933 865 934 866 if (kvm_host_psci_config.smccc_version < ARM_SMCCC_VERSION_1_2) 935 867 return 0; 936 868 937 937 - arm_smccc_1_1_smc(FFA_VERSION, FFA_VERSION_1_1, 0, 0, 0, 0, 0, 0, &res); 869 869 + arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 870 870 + .a0 = FFA_VERSION, 871 871 + .a1 = FFA_VERSION_1_2, 872 872 + }, &res); 938 873 if (res.a0 == FFA_RET_NOT_SUPPORTED) 939 874 return 0; 940 875 ··· 957 886 if (FFA_MAJOR_VERSION(res.a0) != 1) 958 887 return -EOPNOTSUPP; 959 888 960 960 - if (FFA_MINOR_VERSION(res.a0) < FFA_MINOR_VERSION(FFA_VERSION_1_1)) 889 889 + if (FFA_MINOR_VERSION(res.a0) < FFA_MINOR_VERSION(FFA_VERSION_1_2)) 961 890 hyp_ffa_version = res.a0; 962 891 else 963 963 - hyp_ffa_version = FFA_VERSION_1_1; 892 892 + hyp_ffa_version = FFA_VERSION_1_2; 964 893 965 894 tx = pages; 966 895 pages += KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE;

+14

arch/arm64/kvm/hyp/nvhe/hyp-main.c

reviewed

··· 546 546 cpu_reg(host_ctxt, 1) = __pkvm_prot_finalize(); 547 547 } 548 548 549 549 + static void handle___pkvm_reserve_vm(struct kvm_cpu_context *host_ctxt) 550 550 + { 551 551 + cpu_reg(host_ctxt, 1) = __pkvm_reserve_vm(); 552 552 + } 553 553 + 554 554 + static void handle___pkvm_unreserve_vm(struct kvm_cpu_context *host_ctxt) 555 555 + { 556 556 + DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); 557 557 + 558 558 + __pkvm_unreserve_vm(handle); 559 559 + } 560 560 + 549 561 static void handle___pkvm_init_vm(struct kvm_cpu_context *host_ctxt) 550 562 { 551 563 DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1); ··· 618 606 HANDLE_FUNC(__kvm_timer_set_cntvoff), 619 607 HANDLE_FUNC(__vgic_v3_save_vmcr_aprs), 620 608 HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs), 609 609 + HANDLE_FUNC(__pkvm_reserve_vm), 610 610 + HANDLE_FUNC(__pkvm_unreserve_vm), 621 611 HANDLE_FUNC(__pkvm_init_vm), 622 612 HANDLE_FUNC(__pkvm_init_vcpu), 623 613 HANDLE_FUNC(__pkvm_teardown_vm),

+6 -3

arch/arm64/kvm/hyp/nvhe/mem_protect.c

reviewed

··· 1010 1010 return ret; 1011 1011 if (!kvm_pte_valid(pte)) 1012 1012 return -ENOENT; 1013 1013 - if (kvm_granule_size(level) != size) 1013 1013 + if (size && kvm_granule_size(level) != size) 1014 1014 return -E2BIG; 1015 1015 + 1016 1016 + if (!size) 1017 1017 + size = kvm_granule_size(level); 1015 1018 1016 1019 state = guest_get_page_state(pte, ipa); 1017 1020 if (state != PKVM_PAGE_SHARED_BORROWED) ··· 1103 1100 if (prot & ~KVM_PGTABLE_PROT_RWX) 1104 1101 return -EINVAL; 1105 1102 1106 1106 - assert_host_shared_guest(vm, ipa, PAGE_SIZE); 1103 1103 + assert_host_shared_guest(vm, ipa, 0); 1107 1104 guest_lock_component(vm); 1108 1105 ret = kvm_pgtable_stage2_relax_perms(&vm->pgt, ipa, prot, 0); 1109 1106 guest_unlock_component(vm); ··· 1159 1156 if (pkvm_hyp_vm_is_protected(vm)) 1160 1157 return -EPERM; 1161 1158 1162 1162 - assert_host_shared_guest(vm, ipa, PAGE_SIZE); 1159 1159 + assert_host_shared_guest(vm, ipa, 0); 1163 1160 guest_lock_component(vm); 1164 1161 kvm_pgtable_stage2_mkyoung(&vm->pgt, ipa, 0); 1165 1162 guest_unlock_component(vm);

+134 -43

arch/arm64/kvm/hyp/nvhe/pkvm.c

reviewed

··· 23 23 unsigned int kvm_host_sve_max_vl; 24 24 25 25 /* 26 26 - * The currently loaded hyp vCPU for each physical CPU. Used only when 27 27 - * protected KVM is enabled, but for both protected and non-protected VMs. 26 26 + * The currently loaded hyp vCPU for each physical CPU. Used in protected mode 27 27 + * for both protected and non-protected VMs. 28 28 */ 29 29 static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu); 30 30 ··· 135 135 { 136 136 struct kvm *kvm = vcpu->kvm; 137 137 138 138 - /* Protected KVM does not support AArch32 guests. */ 138 138 + /* No AArch32 support for protected guests. */ 139 139 if (kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL0, AARCH32) || 140 140 kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL1, AARCH32)) 141 141 return -EINVAL; ··· 192 192 */ 193 193 #define HANDLE_OFFSET 0x1000 194 194 195 195 + /* 196 196 + * Marks a reserved but not yet used entry in the VM table. 197 197 + */ 198 198 + #define RESERVED_ENTRY ((void *)0xa110ca7ed) 199 199 + 195 200 static unsigned int vm_handle_to_idx(pkvm_handle_t handle) 196 201 { 197 202 return handle - HANDLE_OFFSET; ··· 215 210 DEFINE_HYP_SPINLOCK(vm_table_lock); 216 211 217 212 /* 218 218 - * The table of VM entries for protected VMs in hyp. 219 219 - * Allocated at hyp initialization and setup. 213 213 + * A table that tracks all VMs in protected mode. 214 214 + * Allocated during hyp initialization and setup. 220 215 */ 221 216 static struct pkvm_hyp_vm **vm_table; 222 217 ··· 234 229 unsigned int idx = vm_handle_to_idx(handle); 235 230 236 231 if (unlikely(idx >= KVM_MAX_PVMS)) 232 232 + return NULL; 233 233 + 234 234 + /* A reserved entry doesn't represent an initialized VM. */ 235 235 + if (unlikely(vm_table[idx] == RESERVED_ENTRY)) 237 236 return NULL; 238 237 239 238 return vm_table[idx]; ··· 410 401 } 411 402 412 403 static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, 413 413 - unsigned int nr_vcpus) 404 404 + unsigned int nr_vcpus, pkvm_handle_t handle) 414 405 { 406 406 + struct kvm_s2_mmu *mmu = &hyp_vm->kvm.arch.mmu; 407 407 + int idx = vm_handle_to_idx(handle); 408 408 + 409 409 + hyp_vm->kvm.arch.pkvm.handle = handle; 410 410 + 415 411 hyp_vm->host_kvm = host_kvm; 416 412 hyp_vm->kvm.created_vcpus = nr_vcpus; 417 417 - hyp_vm->kvm.arch.mmu.vtcr = host_mmu.arch.mmu.vtcr; 418 418 - hyp_vm->kvm.arch.pkvm.enabled = READ_ONCE(host_kvm->arch.pkvm.enabled); 413 413 + hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected); 414 414 + hyp_vm->kvm.arch.pkvm.is_created = true; 419 415 hyp_vm->kvm.arch.flags = 0; 420 416 pkvm_init_features_from_host(hyp_vm, host_kvm); 417 417 + 418 418 + /* VMID 0 is reserved for the host */ 419 419 + atomic64_set(&mmu->vmid.id, idx + 1); 420 420 + 421 421 + mmu->vtcr = host_mmu.arch.mmu.vtcr; 422 422 + mmu->arch = &hyp_vm->kvm.arch; 423 423 + mmu->pgt = &hyp_vm->pgt; 421 424 } 422 425 423 426 static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *host_vcpu) ··· 501 480 return ret; 502 481 } 503 482 504 504 - static int find_free_vm_table_entry(struct kvm *host_kvm) 483 483 + static int find_free_vm_table_entry(void) 505 484 { 506 485 int i; 507 486 ··· 514 493 } 515 494 516 495 /* 517 517 - * Allocate a VM table entry and insert a pointer to the new vm. 496 496 + * Reserve a VM table entry. 518 497 * 519 519 - * Return a unique handle to the protected VM on success, 498 498 + * Return a unique handle to the VM on success, 520 499 * negative error code on failure. 521 500 */ 522 522 - static pkvm_handle_t insert_vm_table_entry(struct kvm *host_kvm, 523 523 - struct pkvm_hyp_vm *hyp_vm) 501 501 + static int allocate_vm_table_entry(void) 524 502 { 525 525 - struct kvm_s2_mmu *mmu = &hyp_vm->kvm.arch.mmu; 526 503 int idx; 527 504 528 505 hyp_assert_lock_held(&vm_table_lock); ··· 533 514 if (unlikely(!vm_table)) 534 515 return -EINVAL; 535 516 536 536 - idx = find_free_vm_table_entry(host_kvm); 537 537 - if (idx < 0) 517 517 + idx = find_free_vm_table_entry(); 518 518 + if (unlikely(idx < 0)) 538 519 return idx; 539 520 540 540 - hyp_vm->kvm.arch.pkvm.handle = idx_to_vm_handle(idx); 521 521 + vm_table[idx] = RESERVED_ENTRY; 541 522 542 542 - /* VMID 0 is reserved for the host */ 543 543 - atomic64_set(&mmu->vmid.id, idx + 1); 523 523 + return idx; 524 524 + } 544 525 545 545 - mmu->arch = &hyp_vm->kvm.arch; 546 546 - mmu->pgt = &hyp_vm->pgt; 526 526 + static int __insert_vm_table_entry(pkvm_handle_t handle, 527 527 + struct pkvm_hyp_vm *hyp_vm) 528 528 + { 529 529 + unsigned int idx; 530 530 + 531 531 + hyp_assert_lock_held(&vm_table_lock); 532 532 + 533 533 + /* 534 534 + * Initializing protected state might have failed, yet a malicious 535 535 + * host could trigger this function. Thus, ensure that 'vm_table' 536 536 + * exists. 537 537 + */ 538 538 + if (unlikely(!vm_table)) 539 539 + return -EINVAL; 540 540 + 541 541 + idx = vm_handle_to_idx(handle); 542 542 + if (unlikely(idx >= KVM_MAX_PVMS)) 543 543 + return -EINVAL; 544 544 + 545 545 + if (unlikely(vm_table[idx] != RESERVED_ENTRY)) 546 546 + return -EINVAL; 547 547 548 548 vm_table[idx] = hyp_vm; 549 549 - return hyp_vm->kvm.arch.pkvm.handle; 549 549 + 550 550 + return 0; 551 551 + } 552 552 + 553 553 + /* 554 554 + * Insert a pointer to the initialized VM into the VM table. 555 555 + * 556 556 + * Return 0 on success, or negative error code on failure. 557 557 + */ 558 558 + static int insert_vm_table_entry(pkvm_handle_t handle, 559 559 + struct pkvm_hyp_vm *hyp_vm) 560 560 + { 561 561 + int ret; 562 562 + 563 563 + hyp_spin_lock(&vm_table_lock); 564 564 + ret = __insert_vm_table_entry(handle, hyp_vm); 565 565 + hyp_spin_unlock(&vm_table_lock); 566 566 + 567 567 + return ret; 550 568 } 551 569 552 570 /* ··· 650 594 } 651 595 652 596 /* 653 653 - * Initialize the hypervisor copy of the protected VM state using the 654 654 - * memory donated by the host. 597 597 + * Reserves an entry in the hypervisor for a new VM in protected mode. 655 598 * 656 656 - * Unmaps the donated memory from the host at stage 2. 599 599 + * Return a unique handle to the VM on success, negative error code on failure. 600 600 + */ 601 601 + int __pkvm_reserve_vm(void) 602 602 + { 603 603 + int ret; 604 604 + 605 605 + hyp_spin_lock(&vm_table_lock); 606 606 + ret = allocate_vm_table_entry(); 607 607 + hyp_spin_unlock(&vm_table_lock); 608 608 + 609 609 + if (ret < 0) 610 610 + return ret; 611 611 + 612 612 + return idx_to_vm_handle(ret); 613 613 + } 614 614 + 615 615 + /* 616 616 + * Removes a reserved entry, but only if is hasn't been used yet. 617 617 + * Otherwise, the VM needs to be destroyed. 618 618 + */ 619 619 + void __pkvm_unreserve_vm(pkvm_handle_t handle) 620 620 + { 621 621 + unsigned int idx = vm_handle_to_idx(handle); 622 622 + 623 623 + if (unlikely(!vm_table)) 624 624 + return; 625 625 + 626 626 + hyp_spin_lock(&vm_table_lock); 627 627 + if (likely(idx < KVM_MAX_PVMS && vm_table[idx] == RESERVED_ENTRY)) 628 628 + remove_vm_table_entry(handle); 629 629 + hyp_spin_unlock(&vm_table_lock); 630 630 + } 631 631 + 632 632 + /* 633 633 + * Initialize the hypervisor copy of the VM state using host-donated memory. 634 634 + * 635 635 + * Unmap the donated memory from the host at stage 2. 657 636 * 658 637 * host_kvm: A pointer to the host's struct kvm. 659 638 * vm_hva: The host va of the area being donated for the VM state. ··· 697 606 * the VM. Must be page aligned. Its size is implied by the VM's 698 607 * VTCR. 699 608 * 700 700 - * Return a unique handle to the protected VM on success, 701 701 - * negative error code on failure. 609 609 + * Return 0 success, negative error code on failure. 702 610 */ 703 611 int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, 704 612 unsigned long pgd_hva) ··· 705 615 struct pkvm_hyp_vm *hyp_vm = NULL; 706 616 size_t vm_size, pgd_size; 707 617 unsigned int nr_vcpus; 618 618 + pkvm_handle_t handle; 708 619 void *pgd = NULL; 709 620 int ret; 710 621 ··· 715 624 716 625 nr_vcpus = READ_ONCE(host_kvm->created_vcpus); 717 626 if (nr_vcpus < 1) { 627 627 + ret = -EINVAL; 628 628 + goto err_unpin_kvm; 629 629 + } 630 630 + 631 631 + handle = READ_ONCE(host_kvm->arch.pkvm.handle); 632 632 + if (unlikely(handle < HANDLE_OFFSET)) { 718 633 ret = -EINVAL; 719 634 goto err_unpin_kvm; 720 635 } ··· 738 641 if (!pgd) 739 642 goto err_remove_mappings; 740 643 741 741 - init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus); 742 742 - 743 743 - hyp_spin_lock(&vm_table_lock); 744 744 - ret = insert_vm_table_entry(host_kvm, hyp_vm); 745 745 - if (ret < 0) 746 746 - goto err_unlock; 644 644 + init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus, handle); 747 645 748 646 ret = kvm_guest_prepare_stage2(hyp_vm, pgd); 749 647 if (ret) 750 750 - goto err_remove_vm_table_entry; 751 751 - hyp_spin_unlock(&vm_table_lock); 648 648 + goto err_remove_mappings; 752 649 753 753 - return hyp_vm->kvm.arch.pkvm.handle; 650 650 + /* Must be called last since this publishes the VM. */ 651 651 + ret = insert_vm_table_entry(handle, hyp_vm); 652 652 + if (ret) 653 653 + goto err_remove_mappings; 754 654 755 755 - err_remove_vm_table_entry: 756 756 - remove_vm_table_entry(hyp_vm->kvm.arch.pkvm.handle); 757 757 - err_unlock: 758 758 - hyp_spin_unlock(&vm_table_lock); 655 655 + return 0; 656 656 + 759 657 err_remove_mappings: 760 658 unmap_donated_memory(hyp_vm, vm_size); 761 659 unmap_donated_memory(pgd, pgd_size); ··· 760 668 } 761 669 762 670 /* 763 763 - * Initialize the hypervisor copy of the protected vCPU state using the 764 764 - * memory donated by the host. 671 671 + * Initialize the hypervisor copy of the vCPU state using host-donated memory. 765 672 * 766 766 - * handle: The handle for the protected vm. 673 673 + * handle: The hypervisor handle for the vm. 767 674 * host_vcpu: A pointer to the corresponding host vcpu. 768 675 * vcpu_hva: The host va of the area being donated for the vcpu state. 769 676 * Must be page aligned. The size of the area must be equal to

+10 -2

arch/arm64/kvm/hyp/nvhe/setup.c

reviewed

··· 192 192 enum pkvm_page_state state; 193 193 struct hyp_page *page; 194 194 phys_addr_t phys; 195 195 + enum kvm_pgtable_prot prot; 195 196 196 197 if (!kvm_pte_valid(ctx->old)) 197 198 return 0; ··· 211 210 * configured in the hypervisor stage-1, and make sure to propagate them 212 211 * to the hyp_vmemmap state. 213 212 */ 214 214 - state = pkvm_getstate(kvm_pgtable_hyp_pte_prot(ctx->old)); 213 213 + prot = kvm_pgtable_hyp_pte_prot(ctx->old); 214 214 + state = pkvm_getstate(prot); 215 215 switch (state) { 216 216 case PKVM_PAGE_OWNED: 217 217 set_hyp_state(page, PKVM_PAGE_OWNED); 218 218 - return host_stage2_set_owner_locked(phys, PAGE_SIZE, PKVM_ID_HYP); 218 218 + /* hyp text is RO in the host stage-2 to be inspected on panic. */ 219 219 + if (prot == PAGE_HYP_EXEC) { 220 220 + set_host_state(page, PKVM_NOPAGE); 221 221 + return host_stage2_idmap_locked(phys, PAGE_SIZE, KVM_PGTABLE_PROT_R); 222 222 + } else { 223 223 + return host_stage2_set_owner_locked(phys, PAGE_SIZE, PKVM_ID_HYP); 224 224 + } 219 225 case PKVM_PAGE_SHARED_OWNED: 220 226 set_hyp_state(page, PKVM_PAGE_SHARED_OWNED); 221 227 set_host_state(page, PKVM_PAGE_SHARED_BORROWED);

+9 -16

arch/arm64/kvm/hyp/vgic-v3-sr.c

reviewed

··· 295 295 } 296 296 } 297 297 298 298 - /* 299 299 - * GICv5 BET0 FEAT_GCIE_LEGACY doesn't include ICC_SRE_EL2. This is due 300 300 - * to be relaxed in a future spec release, at which point this in 301 301 - * condition can be dropped. 302 302 - */ 303 303 - if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) { 298 298 + /* Only disable SRE if the host implements the GICv2 interface */ 299 299 + if (static_branch_unlikely(&vgic_v3_has_v2_compat)) { 304 300 /* 305 301 * Prevent the guest from touching the ICC_SRE_EL1 system 306 302 * register. Note that this may not have any effect, as ··· 325 329 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2); 326 330 } 327 331 328 328 - /* 329 329 - * Can be dropped in the future when GICv5 spec is relaxed. See comment 330 330 - * above. 331 331 - */ 332 332 - if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) { 332 332 + /* Only restore SRE if the host implements the GICv2 interface */ 333 333 + if (static_branch_unlikely(&vgic_v3_has_v2_compat)) { 333 334 val = read_gicreg(ICC_SRE_EL2); 334 335 write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2); 335 335 - } 336 336 337 337 - if (!cpu_if->vgic_sre) { 338 338 - /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */ 339 339 - isb(); 340 340 - write_gicreg(1, ICC_SRE_EL1); 337 337 + if (!cpu_if->vgic_sre) { 338 338 + /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */ 339 339 + isb(); 340 340 + write_gicreg(1, ICC_SRE_EL1); 341 341 + } 341 342 } 342 343 343 344 /*

+7

arch/arm64/kvm/hyp/vhe/switch.c

reviewed

··· 95 95 /* Force NV2 in case the guest is forgetful... */ 96 96 guest_hcr |= HCR_NV2; 97 97 } 98 98 + 99 99 + /* 100 100 + * Exclude the guest's TWED configuration if it hasn't set TWE 101 101 + * to avoid potentially delaying traps for the host. 102 102 + */ 103 103 + if (!(guest_hcr & HCR_TWE)) 104 104 + guest_hcr &= ~(HCR_EL2_TWEDEn | HCR_EL2_TWEDEL); 98 105 } 99 106 100 107 BUG_ON(host_data_test_flag(VCPU_IN_HYP_CONTEXT) &&

+25 -2

arch/arm64/kvm/inject_fault.c

reviewed

··· 106 106 { 107 107 unsigned long cpsr = *vcpu_cpsr(vcpu); 108 108 bool is_aarch32 = vcpu_mode_is_32bit(vcpu); 109 109 - u64 esr = 0; 109 109 + u64 esr = 0, fsc; 110 110 + int level; 111 111 + 112 112 + /* 113 113 + * If injecting an abort from a failed S1PTW, rewalk the S1 PTs to 114 114 + * find the failing level. If we can't find it, assume the error was 115 115 + * transient and restart without changing the state. 116 116 + */ 117 117 + if (kvm_vcpu_abt_iss1tw(vcpu)) { 118 118 + u64 hpfar = kvm_vcpu_get_fault_ipa(vcpu); 119 119 + int ret; 120 120 + 121 121 + if (hpfar == INVALID_GPA) 122 122 + return; 123 123 + 124 124 + ret = __kvm_find_s1_desc_level(vcpu, addr, hpfar, &level); 125 125 + if (ret) 126 126 + return; 127 127 + 128 128 + WARN_ON_ONCE(level < -1 || level > 3); 129 129 + fsc = ESR_ELx_FSC_SEA_TTW(level); 130 130 + } else { 131 131 + fsc = ESR_ELx_FSC_EXTABT; 132 132 + } 110 133 111 134 /* This delight is brought to you by FEAT_DoubleFault2. */ 112 135 if (effective_sctlr2_ease(vcpu)) ··· 156 133 if (!is_iabt) 157 134 esr |= ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT; 158 135 159 159 - esr |= ESR_ELx_FSC_EXTABT; 136 136 + esr |= fsc; 160 137 161 138 vcpu_write_sys_reg(vcpu, addr, exception_far_elx(vcpu)); 162 139 vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));

+157 -55

arch/arm64/kvm/mmu.c

reviewed

··· 1431 1431 * able to see the page's tags and therefore they must be initialised first. If 1432 1432 * PG_mte_tagged is set, tags have already been initialised. 1433 1433 * 1434 1434 - * The race in the test/set of the PG_mte_tagged flag is handled by: 1435 1435 - * - preventing VM_SHARED mappings in a memslot with MTE preventing two VMs 1436 1436 - * racing to santise the same page 1437 1437 - * - mmap_lock protects between a VM faulting a page in and the VMM performing 1438 1438 - * an mprotect() to add VM_MTE 1434 1434 + * Must be called with kvm->mmu_lock held to ensure the memory remains mapped 1435 1435 + * while the tags are zeroed. 1439 1436 */ 1440 1437 static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn, 1441 1438 unsigned long size) ··· 1479 1482 } 1480 1483 } 1481 1484 1485 1485 + static int prepare_mmu_memcache(struct kvm_vcpu *vcpu, bool topup_memcache, 1486 1486 + void **memcache) 1487 1487 + { 1488 1488 + int min_pages; 1489 1489 + 1490 1490 + if (!is_protected_kvm_enabled()) 1491 1491 + *memcache = &vcpu->arch.mmu_page_cache; 1492 1492 + else 1493 1493 + *memcache = &vcpu->arch.pkvm_memcache; 1494 1494 + 1495 1495 + if (!topup_memcache) 1496 1496 + return 0; 1497 1497 + 1498 1498 + min_pages = kvm_mmu_cache_min_pages(vcpu->arch.hw_mmu); 1499 1499 + 1500 1500 + if (!is_protected_kvm_enabled()) 1501 1501 + return kvm_mmu_topup_memory_cache(*memcache, min_pages); 1502 1502 + 1503 1503 + return topup_hyp_memcache(*memcache, min_pages); 1504 1504 + } 1505 1505 + 1506 1506 + /* 1507 1507 + * Potentially reduce shadow S2 permissions to match the guest's own S2. For 1508 1508 + * exec faults, we'd only reach this point if the guest actually allowed it (see 1509 1509 + * kvm_s2_handle_perm_fault). 1510 1510 + * 1511 1511 + * Also encode the level of the original translation in the SW bits of the leaf 1512 1512 + * entry as a proxy for the span of that translation. This will be retrieved on 1513 1513 + * TLB invalidation from the guest and used to limit the invalidation scope if a 1514 1514 + * TTL hint or a range isn't provided. 1515 1515 + */ 1516 1516 + static void adjust_nested_fault_perms(struct kvm_s2_trans *nested, 1517 1517 + enum kvm_pgtable_prot *prot, 1518 1518 + bool *writable) 1519 1519 + { 1520 1520 + *writable &= kvm_s2_trans_writable(nested); 1521 1521 + if (!kvm_s2_trans_readable(nested)) 1522 1522 + *prot &= ~KVM_PGTABLE_PROT_R; 1523 1523 + 1524 1524 + *prot |= kvm_encode_nested_level(nested); 1525 1525 + } 1526 1526 + 1527 1527 + #define KVM_PGTABLE_WALK_MEMABORT_FLAGS (KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED) 1528 1528 + 1529 1529 + static int gmem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, 1530 1530 + struct kvm_s2_trans *nested, 1531 1531 + struct kvm_memory_slot *memslot, bool is_perm) 1532 1532 + { 1533 1533 + bool write_fault, exec_fault, writable; 1534 1534 + enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_MEMABORT_FLAGS; 1535 1535 + enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; 1536 1536 + struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt; 1537 1537 + unsigned long mmu_seq; 1538 1538 + struct page *page; 1539 1539 + struct kvm *kvm = vcpu->kvm; 1540 1540 + void *memcache; 1541 1541 + kvm_pfn_t pfn; 1542 1542 + gfn_t gfn; 1543 1543 + int ret; 1544 1544 + 1545 1545 + ret = prepare_mmu_memcache(vcpu, true, &memcache); 1546 1546 + if (ret) 1547 1547 + return ret; 1548 1548 + 1549 1549 + if (nested) 1550 1550 + gfn = kvm_s2_trans_output(nested) >> PAGE_SHIFT; 1551 1551 + else 1552 1552 + gfn = fault_ipa >> PAGE_SHIFT; 1553 1553 + 1554 1554 + write_fault = kvm_is_write_fault(vcpu); 1555 1555 + exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu); 1556 1556 + 1557 1557 + VM_WARN_ON_ONCE(write_fault && exec_fault); 1558 1558 + 1559 1559 + mmu_seq = kvm->mmu_invalidate_seq; 1560 1560 + /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */ 1561 1561 + smp_rmb(); 1562 1562 + 1563 1563 + ret = kvm_gmem_get_pfn(kvm, memslot, gfn, &pfn, &page, NULL); 1564 1564 + if (ret) { 1565 1565 + kvm_prepare_memory_fault_exit(vcpu, fault_ipa, PAGE_SIZE, 1566 1566 + write_fault, exec_fault, false); 1567 1567 + return ret; 1568 1568 + } 1569 1569 + 1570 1570 + writable = !(memslot->flags & KVM_MEM_READONLY); 1571 1571 + 1572 1572 + if (nested) 1573 1573 + adjust_nested_fault_perms(nested, &prot, &writable); 1574 1574 + 1575 1575 + if (writable) 1576 1576 + prot |= KVM_PGTABLE_PROT_W; 1577 1577 + 1578 1578 + if (exec_fault || 1579 1579 + (cpus_have_final_cap(ARM64_HAS_CACHE_DIC) && 1580 1580 + (!nested || kvm_s2_trans_executable(nested)))) 1581 1581 + prot |= KVM_PGTABLE_PROT_X; 1582 1582 + 1583 1583 + kvm_fault_lock(kvm); 1584 1584 + if (mmu_invalidate_retry(kvm, mmu_seq)) { 1585 1585 + ret = -EAGAIN; 1586 1586 + goto out_unlock; 1587 1587 + } 1588 1588 + 1589 1589 + ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, fault_ipa, PAGE_SIZE, 1590 1590 + __pfn_to_phys(pfn), prot, 1591 1591 + memcache, flags); 1592 1592 + 1593 1593 + out_unlock: 1594 1594 + kvm_release_faultin_page(kvm, page, !!ret, writable); 1595 1595 + kvm_fault_unlock(kvm); 1596 1596 + 1597 1597 + if (writable && !ret) 1598 1598 + mark_page_dirty_in_slot(kvm, memslot, gfn); 1599 1599 + 1600 1600 + return ret != -EAGAIN ? ret : 0; 1601 1601 + } 1602 1602 + 1482 1603 static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, 1483 1604 struct kvm_s2_trans *nested, 1484 1605 struct kvm_memory_slot *memslot, unsigned long hva, 1485 1606 bool fault_is_perm) 1486 1607 { 1487 1608 int ret = 0; 1488 1488 - bool write_fault, writable, force_pte = false; 1609 1609 + bool topup_memcache; 1610 1610 + bool write_fault, writable; 1489 1611 bool exec_fault, mte_allowed, is_vma_cacheable; 1490 1612 bool s2_force_noncacheable = false, vfio_allow_any_uc = false; 1491 1613 unsigned long mmu_seq; ··· 1616 1500 gfn_t gfn; 1617 1501 kvm_pfn_t pfn; 1618 1502 bool logging_active = memslot_is_logging(memslot); 1503 1503 + bool force_pte = logging_active; 1619 1504 long vma_pagesize, fault_granule; 1620 1505 enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; 1621 1506 struct kvm_pgtable *pgt; 1622 1507 struct page *page; 1623 1508 vm_flags_t vm_flags; 1624 1624 - enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED; 1509 1509 + enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_MEMABORT_FLAGS; 1625 1510 1626 1511 if (fault_is_perm) 1627 1512 fault_granule = kvm_vcpu_trap_get_perm_fault_granule(vcpu); 1628 1513 write_fault = kvm_is_write_fault(vcpu); 1629 1514 exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu); 1630 1630 - VM_BUG_ON(write_fault && exec_fault); 1631 1631 - 1632 1632 - if (!is_protected_kvm_enabled()) 1633 1633 - memcache = &vcpu->arch.mmu_page_cache; 1634 1634 - else 1635 1635 - memcache = &vcpu->arch.pkvm_memcache; 1515 1515 + VM_WARN_ON_ONCE(write_fault && exec_fault); 1636 1516 1637 1517 /* 1638 1518 * Permission faults just need to update the existing leaf entry, ··· 1636 1524 * only exception to this is when dirty logging is enabled at runtime 1637 1525 * and a write fault needs to collapse a block entry into a table. 1638 1526 */ 1639 1639 - if (!fault_is_perm || (logging_active && write_fault)) { 1640 1640 - int min_pages = kvm_mmu_cache_min_pages(vcpu->arch.hw_mmu); 1641 1641 - 1642 1642 - if (!is_protected_kvm_enabled()) 1643 1643 - ret = kvm_mmu_topup_memory_cache(memcache, min_pages); 1644 1644 - else 1645 1645 - ret = topup_hyp_memcache(memcache, min_pages); 1646 1646 - 1647 1647 - if (ret) 1648 1648 - return ret; 1649 1649 - } 1527 1527 + topup_memcache = !fault_is_perm || (logging_active && write_fault); 1528 1528 + ret = prepare_mmu_memcache(vcpu, topup_memcache, &memcache); 1529 1529 + if (ret) 1530 1530 + return ret; 1650 1531 1651 1532 /* 1652 1533 * Let's check if we will get back a huge page backed by hugetlbfs, or ··· 1653 1548 return -EFAULT; 1654 1549 } 1655 1550 1656 1656 - /* 1657 1657 - * logging_active is guaranteed to never be true for VM_PFNMAP 1658 1658 - * memslots. 1659 1659 - */ 1660 1660 - if (logging_active) { 1661 1661 - force_pte = true; 1551 1551 + if (force_pte) 1662 1552 vma_shift = PAGE_SHIFT; 1663 1663 - } else { 1553 1553 + else 1664 1554 vma_shift = get_vma_page_shift(vma, hva); 1665 1665 - } 1666 1555 1667 1556 switch (vma_shift) { 1668 1557 #ifndef __PAGETABLE_PMD_FOLDED ··· 1708 1609 max_map_size = PAGE_SIZE; 1709 1610 1710 1611 force_pte = (max_map_size == PAGE_SIZE); 1711 1711 - vma_pagesize = min(vma_pagesize, (long)max_map_size); 1612 1612 + vma_pagesize = min_t(long, vma_pagesize, max_map_size); 1712 1613 } 1713 1614 1714 1615 /* ··· 1741 1642 * Rely on mmap_read_unlock() for an implicit smp_rmb(), which pairs 1742 1643 * with the smp_wmb() in kvm_mmu_invalidate_end(). 1743 1644 */ 1744 1744 - mmu_seq = vcpu->kvm->mmu_invalidate_seq; 1645 1645 + mmu_seq = kvm->mmu_invalidate_seq; 1745 1646 mmap_read_unlock(current->mm); 1746 1647 1747 1648 pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0, ··· 1772 1673 * cache maintenance. 1773 1674 */ 1774 1675 if (!kvm_supports_cacheable_pfnmap()) 1775 1775 - return -EFAULT; 1676 1676 + ret = -EFAULT; 1776 1677 } else { 1777 1678 /* 1778 1679 * If the page was identified as device early by looking at ··· 1795 1696 } 1796 1697 1797 1698 if (exec_fault && s2_force_noncacheable) 1798 1798 - return -ENOEXEC; 1699 1699 + ret = -ENOEXEC; 1799 1700 1800 1800 - /* 1801 1801 - * Potentially reduce shadow S2 permissions to match the guest's own 1802 1802 - * S2. For exec faults, we'd only reach this point if the guest 1803 1803 - * actually allowed it (see kvm_s2_handle_perm_fault). 1804 1804 - * 1805 1805 - * Also encode the level of the original translation in the SW bits 1806 1806 - * of the leaf entry as a proxy for the span of that translation. 1807 1807 - * This will be retrieved on TLB invalidation from the guest and 1808 1808 - * used to limit the invalidation scope if a TTL hint or a range 1809 1809 - * isn't provided. 1810 1810 - */ 1811 1811 - if (nested) { 1812 1812 - writable &= kvm_s2_trans_writable(nested); 1813 1813 - if (!kvm_s2_trans_readable(nested)) 1814 1814 - prot &= ~KVM_PGTABLE_PROT_R; 1815 1815 - 1816 1816 - prot |= kvm_encode_nested_level(nested); 1701 1701 + if (ret) { 1702 1702 + kvm_release_page_unused(page); 1703 1703 + return ret; 1817 1704 } 1705 1705 + 1706 1706 + if (nested) 1707 1707 + adjust_nested_fault_perms(nested, &prot, &writable); 1818 1708 1819 1709 kvm_fault_lock(kvm); 1820 1710 pgt = vcpu->arch.hw_mmu->pgt; ··· 2073 1985 goto out_unlock; 2074 1986 } 2075 1987 2076 2076 - ret = user_mem_abort(vcpu, fault_ipa, nested, memslot, hva, 2077 2077 - esr_fsc_is_permission_fault(esr)); 1988 1988 + VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) && 1989 1989 + !write_fault && !kvm_vcpu_trap_is_exec_fault(vcpu)); 1990 1990 + 1991 1991 + if (kvm_slot_has_gmem(memslot)) 1992 1992 + ret = gmem_abort(vcpu, fault_ipa, nested, memslot, 1993 1993 + esr_fsc_is_permission_fault(esr)); 1994 1994 + else 1995 1995 + ret = user_mem_abort(vcpu, fault_ipa, nested, memslot, hva, 1996 1996 + esr_fsc_is_permission_fault(esr)); 2078 1997 if (ret == 0) 2079 1998 ret = 1; 2080 1999 out: ··· 2312 2217 */ 2313 2218 if ((new->base_gfn + new->npages) > (kvm_phys_size(&kvm->arch.mmu) >> PAGE_SHIFT)) 2314 2219 return -EFAULT; 2220 2220 + 2221 2221 + /* 2222 2222 + * Only support guest_memfd backed memslots with mappable memory, since 2223 2223 + * there aren't any CoCo VMs that support only private memory on arm64. 2224 2224 + */ 2225 2225 + if (kvm_slot_has_gmem(new) && !kvm_memslot_is_gmem_only(new)) 2226 2226 + return -EINVAL; 2315 2227 2316 2228 hva = new->userspace_addr; 2317 2229 reg_end = hva + (new->npages << PAGE_SHIFT);

+96 -23

arch/arm64/kvm/nested.c

reviewed

··· 349 349 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); 350 350 /* Global limit for now, should eventually be per-VM */ 351 351 wi->max_oa_bits = min(get_kvm_ipa_limit(), 352 352 - ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr))); 352 352 + ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr), false)); 353 353 } 354 354 355 355 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, ··· 1172 1172 return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48); 1173 1173 } 1174 1174 1175 1175 - static int kvm_translate_vncr(struct kvm_vcpu *vcpu) 1175 1175 + static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) 1176 1176 { 1177 1177 + struct kvm_memory_slot *memslot; 1177 1178 bool write_fault, writable; 1178 1179 unsigned long mmu_seq; 1179 1180 struct vncr_tlb *vt; ··· 1217 1216 smp_rmb(); 1218 1217 1219 1218 gfn = vt->wr.pa >> PAGE_SHIFT; 1220 1220 - pfn = kvm_faultin_pfn(vcpu, gfn, write_fault, &writable, &page); 1221 1221 - if (is_error_noslot_pfn(pfn) || (write_fault && !writable)) 1219 1219 + memslot = gfn_to_memslot(vcpu->kvm, gfn); 1220 1220 + if (!memslot) 1222 1221 return -EFAULT; 1222 1222 + 1223 1223 + *is_gmem = kvm_slot_has_gmem(memslot); 1224 1224 + if (!*is_gmem) { 1225 1225 + pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0, 1226 1226 + &writable, &page); 1227 1227 + if (is_error_noslot_pfn(pfn) || (write_fault && !writable)) 1228 1228 + return -EFAULT; 1229 1229 + } else { 1230 1230 + ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); 1231 1231 + if (ret) { 1232 1232 + kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE, 1233 1233 + write_fault, false, false); 1234 1234 + return ret; 1235 1235 + } 1236 1236 + } 1223 1237 1224 1238 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { 1225 1239 if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) ··· 1311 1295 if (esr_fsc_is_permission_fault(esr)) { 1312 1296 inject_vncr_perm(vcpu); 1313 1297 } else if (esr_fsc_is_translation_fault(esr)) { 1314 1314 - bool valid; 1298 1298 + bool valid, is_gmem = false; 1315 1299 int ret; 1316 1300 1317 1301 scoped_guard(read_lock, &vcpu->kvm->mmu_lock) 1318 1302 valid = kvm_vncr_tlb_lookup(vcpu); 1319 1303 1320 1304 if (!valid) 1321 1321 - ret = kvm_translate_vncr(vcpu); 1305 1305 + ret = kvm_translate_vncr(vcpu, &is_gmem); 1322 1306 else 1323 1307 ret = -EPERM; 1324 1308 1325 1309 switch (ret) { 1326 1310 case -EAGAIN: 1327 1327 - case -ENOMEM: 1328 1311 /* Let's try again... */ 1329 1312 break; 1313 1313 + case -ENOMEM: 1314 1314 + /* 1315 1315 + * For guest_memfd, this indicates that it failed to 1316 1316 + * create a folio to back the memory. Inform userspace. 1317 1317 + */ 1318 1318 + if (is_gmem) 1319 1319 + return 0; 1320 1320 + /* Otherwise, let's try again... */ 1321 1321 + break; 1330 1322 case -EFAULT: 1323 1323 + case -EIO: 1324 1324 + case -EHWPOISON: 1325 1325 + if (is_gmem) 1326 1326 + return 0; 1327 1327 + fallthrough; 1331 1328 case -EINVAL: 1332 1329 case -ENOENT: 1333 1330 case -EACCES: ··· 1491 1462 1492 1463 case SYS_ID_AA64PFR1_EL1: 1493 1464 /* Only support BTI, SSBS, CSV2_frac */ 1494 1494 - val &= (ID_AA64PFR1_EL1_BT | 1495 1495 - ID_AA64PFR1_EL1_SSBS | 1496 1496 - ID_AA64PFR1_EL1_CSV2_frac); 1465 1465 + val &= ~(ID_AA64PFR1_EL1_PFAR | 1466 1466 + ID_AA64PFR1_EL1_MTEX | 1467 1467 + ID_AA64PFR1_EL1_THE | 1468 1468 + ID_AA64PFR1_EL1_GCS | 1469 1469 + ID_AA64PFR1_EL1_MTE_frac | 1470 1470 + ID_AA64PFR1_EL1_NMI | 1471 1471 + ID_AA64PFR1_EL1_SME | 1472 1472 + ID_AA64PFR1_EL1_RES0 | 1473 1473 + ID_AA64PFR1_EL1_MPAM_frac | 1474 1474 + ID_AA64PFR1_EL1_MTE); 1497 1475 break; 1498 1476 1499 1477 case SYS_ID_AA64MMFR0_EL1: ··· 1553 1517 break; 1554 1518 1555 1519 case SYS_ID_AA64MMFR1_EL1: 1556 1556 - val &= (ID_AA64MMFR1_EL1_HCX | 1557 1557 - ID_AA64MMFR1_EL1_PAN | 1558 1558 - ID_AA64MMFR1_EL1_LO | 1559 1559 - ID_AA64MMFR1_EL1_HPDS | 1560 1560 - ID_AA64MMFR1_EL1_VH | 1561 1561 - ID_AA64MMFR1_EL1_VMIDBits); 1520 1520 + val &= ~(ID_AA64MMFR1_EL1_CMOW | 1521 1521 + ID_AA64MMFR1_EL1_nTLBPA | 1522 1522 + ID_AA64MMFR1_EL1_ETS | 1523 1523 + ID_AA64MMFR1_EL1_XNX | 1524 1524 + ID_AA64MMFR1_EL1_HAFDBS); 1562 1525 /* FEAT_E2H0 implies no VHE */ 1563 1526 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) 1564 1527 val &= ~ID_AA64MMFR1_EL1_VH; ··· 1599 1564 1600 1565 case SYS_ID_AA64DFR0_EL1: 1601 1566 /* Only limited support for PMU, Debug, BPs, WPs, and HPMN0 */ 1602 1602 - val &= (ID_AA64DFR0_EL1_PMUVer | 1603 1603 - ID_AA64DFR0_EL1_WRPs | 1604 1604 - ID_AA64DFR0_EL1_BRPs | 1605 1605 - ID_AA64DFR0_EL1_DebugVer| 1606 1606 - ID_AA64DFR0_EL1_HPMN0); 1567 1567 + val &= ~(ID_AA64DFR0_EL1_ExtTrcBuff | 1568 1568 + ID_AA64DFR0_EL1_BRBE | 1569 1569 + ID_AA64DFR0_EL1_MTPMU | 1570 1570 + ID_AA64DFR0_EL1_TraceBuffer | 1571 1571 + ID_AA64DFR0_EL1_TraceFilt | 1572 1572 + ID_AA64DFR0_EL1_PMSVer | 1573 1573 + ID_AA64DFR0_EL1_CTX_CMPs | 1574 1574 + ID_AA64DFR0_EL1_SEBEP | 1575 1575 + ID_AA64DFR0_EL1_PMSS | 1576 1576 + ID_AA64DFR0_EL1_TraceVer); 1607 1577 1608 1608 - /* Cap Debug to ARMv8.1 */ 1609 1609 - val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, VHE); 1578 1578 + /* 1579 1579 + * FEAT_Debugv8p9 requires support for extended breakpoints / 1580 1580 + * watchpoints. 1581 1581 + */ 1582 1582 + val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8); 1610 1583 break; 1611 1584 } 1612 1585 ··· 1838 1795 */ 1839 1796 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING))) 1840 1797 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu)); 1798 1798 + } 1799 1799 + 1800 1800 + /* 1801 1801 + * KVM unconditionally sets most of these traps anyway but use an allowlist 1802 1802 + * to document the guest hypervisor traps that may take precedence and guard 1803 1803 + * against future changes to the non-nested trap configuration. 1804 1804 + */ 1805 1805 + #define NV_MDCR_GUEST_INCLUDE (MDCR_EL2_TDE | \ 1806 1806 + MDCR_EL2_TDA | \ 1807 1807 + MDCR_EL2_TDRA | \ 1808 1808 + MDCR_EL2_TTRF | \ 1809 1809 + MDCR_EL2_TPMS | \ 1810 1810 + MDCR_EL2_TPM | \ 1811 1811 + MDCR_EL2_TPMCR | \ 1812 1812 + MDCR_EL2_TDCC | \ 1813 1813 + MDCR_EL2_TDOSA) 1814 1814 + 1815 1815 + void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu) 1816 1816 + { 1817 1817 + u64 guest_mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2); 1818 1818 + 1819 1819 + /* 1820 1820 + * In yet another example where FEAT_NV2 is fscking broken, accesses 1821 1821 + * to MDSCR_EL1 are redirected to the VNCR despite having an effect 1822 1822 + * at EL2. Use a big hammer to apply sanity. 1823 1823 + */ 1824 1824 + if (is_hyp_ctxt(vcpu)) 1825 1825 + vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA; 1826 1826 + else 1827 1827 + vcpu->arch.mdcr_el2 |= (guest_mdcr & NV_MDCR_GUEST_INCLUDE); 1841 1828 }

+52 -24

arch/arm64/kvm/pkvm.c

reviewed

··· 85 85 hyp_mem_base); 86 86 } 87 87 88 88 - static void __pkvm_destroy_hyp_vm(struct kvm *host_kvm) 88 88 + static void __pkvm_destroy_hyp_vm(struct kvm *kvm) 89 89 { 90 90 - if (host_kvm->arch.pkvm.handle) { 90 90 + if (pkvm_hyp_vm_is_created(kvm)) { 91 91 WARN_ON(kvm_call_hyp_nvhe(__pkvm_teardown_vm, 92 92 - host_kvm->arch.pkvm.handle)); 92 92 + kvm->arch.pkvm.handle)); 93 93 + } else if (kvm->arch.pkvm.handle) { 94 94 + /* 95 95 + * The VM could have been reserved but hyp initialization has 96 96 + * failed. Make sure to unreserve it. 97 97 + */ 98 98 + kvm_call_hyp_nvhe(__pkvm_unreserve_vm, kvm->arch.pkvm.handle); 93 99 } 94 100 95 95 - host_kvm->arch.pkvm.handle = 0; 96 96 - free_hyp_memcache(&host_kvm->arch.pkvm.teardown_mc); 97 97 - free_hyp_memcache(&host_kvm->arch.pkvm.stage2_teardown_mc); 101 101 + kvm->arch.pkvm.handle = 0; 102 102 + kvm->arch.pkvm.is_created = false; 103 103 + free_hyp_memcache(&kvm->arch.pkvm.teardown_mc); 104 104 + free_hyp_memcache(&kvm->arch.pkvm.stage2_teardown_mc); 98 105 } 99 106 100 107 static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) ··· 136 129 * 137 130 * Return 0 on success, negative error code on failure. 138 131 */ 139 139 - static int __pkvm_create_hyp_vm(struct kvm *host_kvm) 132 132 + static int __pkvm_create_hyp_vm(struct kvm *kvm) 140 133 { 141 134 size_t pgd_sz, hyp_vm_sz; 142 135 void *pgd, *hyp_vm; 143 136 int ret; 144 137 145 145 - if (host_kvm->created_vcpus < 1) 138 138 + if (kvm->created_vcpus < 1) 146 139 return -EINVAL; 147 140 148 148 - pgd_sz = kvm_pgtable_stage2_pgd_size(host_kvm->arch.mmu.vtcr); 141 141 + pgd_sz = kvm_pgtable_stage2_pgd_size(kvm->arch.mmu.vtcr); 149 142 150 143 /* 151 144 * The PGD pages will be reclaimed using a hyp_memcache which implies ··· 159 152 /* Allocate memory to donate to hyp for vm and vcpu pointers. */ 160 153 hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE, 161 154 size_mul(sizeof(void *), 162 162 - host_kvm->created_vcpus))); 155 155 + kvm->created_vcpus))); 163 156 hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT); 164 157 if (!hyp_vm) { 165 158 ret = -ENOMEM; ··· 167 160 } 168 161 169 162 /* Donate the VM memory to hyp and let hyp initialize it. */ 170 170 - ret = kvm_call_hyp_nvhe(__pkvm_init_vm, host_kvm, hyp_vm, pgd); 171 171 - if (ret < 0) 163 163 + ret = kvm_call_hyp_nvhe(__pkvm_init_vm, kvm, hyp_vm, pgd); 164 164 + if (ret) 172 165 goto free_vm; 173 166 174 174 - host_kvm->arch.pkvm.handle = ret; 175 175 - host_kvm->arch.pkvm.stage2_teardown_mc.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2; 167 167 + kvm->arch.pkvm.is_created = true; 168 168 + kvm->arch.pkvm.stage2_teardown_mc.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2; 176 169 kvm_account_pgtable_pages(pgd, pgd_sz / PAGE_SIZE); 177 170 178 171 return 0; ··· 183 176 return ret; 184 177 } 185 178 186 186 - int pkvm_create_hyp_vm(struct kvm *host_kvm) 179 179 + bool pkvm_hyp_vm_is_created(struct kvm *kvm) 180 180 + { 181 181 + return READ_ONCE(kvm->arch.pkvm.is_created); 182 182 + } 183 183 + 184 184 + int pkvm_create_hyp_vm(struct kvm *kvm) 187 185 { 188 186 int ret = 0; 189 187 190 190 - mutex_lock(&host_kvm->arch.config_lock); 191 191 - if (!host_kvm->arch.pkvm.handle) 192 192 - ret = __pkvm_create_hyp_vm(host_kvm); 193 193 - mutex_unlock(&host_kvm->arch.config_lock); 188 188 + mutex_lock(&kvm->arch.config_lock); 189 189 + if (!pkvm_hyp_vm_is_created(kvm)) 190 190 + ret = __pkvm_create_hyp_vm(kvm); 191 191 + mutex_unlock(&kvm->arch.config_lock); 194 192 195 193 return ret; 196 194 } ··· 212 200 return ret; 213 201 } 214 202 215 215 - void pkvm_destroy_hyp_vm(struct kvm *host_kvm) 203 203 + void pkvm_destroy_hyp_vm(struct kvm *kvm) 216 204 { 217 217 - mutex_lock(&host_kvm->arch.config_lock); 218 218 - __pkvm_destroy_hyp_vm(host_kvm); 219 219 - mutex_unlock(&host_kvm->arch.config_lock); 205 205 + mutex_lock(&kvm->arch.config_lock); 206 206 + __pkvm_destroy_hyp_vm(kvm); 207 207 + mutex_unlock(&kvm->arch.config_lock); 220 208 } 221 209 222 222 - int pkvm_init_host_vm(struct kvm *host_kvm) 210 210 + int pkvm_init_host_vm(struct kvm *kvm) 223 211 { 212 212 + int ret; 213 213 + 214 214 + if (pkvm_hyp_vm_is_created(kvm)) 215 215 + return -EINVAL; 216 216 + 217 217 + /* VM is already reserved, no need to proceed. */ 218 218 + if (kvm->arch.pkvm.handle) 219 219 + return 0; 220 220 + 221 221 + /* Reserve the VM in hyp and obtain a hyp handle for the VM. */ 222 222 + ret = kvm_call_hyp_nvhe(__pkvm_reserve_vm); 223 223 + if (ret < 0) 224 224 + return ret; 225 225 + 226 226 + kvm->arch.pkvm.handle = ret; 227 227 + 224 228 return 0; 225 229 } 226 230

+10 -10

arch/arm64/kvm/ptdump.c

reviewed

··· 32 32 .set = " ", 33 33 .clear = "F", 34 34 }, { 35 35 - .mask = KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R | PTE_VALID, 36 36 - .val = KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R | PTE_VALID, 35 35 + .mask = KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R, 36 36 + .val = KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R, 37 37 .set = "R", 38 38 .clear = " ", 39 39 }, { 40 40 - .mask = KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W | PTE_VALID, 41 41 - .val = KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W | PTE_VALID, 40 40 + .mask = KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W, 41 41 + .val = KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W, 42 42 .set = "W", 43 43 .clear = " ", 44 44 }, { 45 45 - .mask = KVM_PTE_LEAF_ATTR_HI_S2_XN | PTE_VALID, 46 46 - .val = PTE_VALID, 47 47 - .set = " ", 48 48 - .clear = "X", 45 45 + .mask = KVM_PTE_LEAF_ATTR_HI_S2_XN, 46 46 + .val = KVM_PTE_LEAF_ATTR_HI_S2_XN, 47 47 + .set = "NX", 48 48 + .clear = "x ", 49 49 }, { 50 50 - .mask = KVM_PTE_LEAF_ATTR_LO_S2_AF | PTE_VALID, 51 51 - .val = KVM_PTE_LEAF_ATTR_LO_S2_AF | PTE_VALID, 50 50 + .mask = KVM_PTE_LEAF_ATTR_LO_S2_AF, 51 51 + .val = KVM_PTE_LEAF_ATTR_LO_S2_AF, 52 52 .set = "AF", 53 53 .clear = " ", 54 54 }, {

+47 -8

arch/arm64/kvm/sys_regs.c

reviewed

··· 1757 1757 val &= ~ID_AA64ISAR2_EL1_WFxT; 1758 1758 break; 1759 1759 case SYS_ID_AA64ISAR3_EL1: 1760 1760 - val &= ID_AA64ISAR3_EL1_FPRCVT | ID_AA64ISAR3_EL1_FAMINMAX; 1760 1760 + val &= ID_AA64ISAR3_EL1_FPRCVT | ID_AA64ISAR3_EL1_LSFE | 1761 1761 + ID_AA64ISAR3_EL1_FAMINMAX; 1761 1762 break; 1762 1763 case SYS_ID_AA64MMFR2_EL1: 1763 1764 val &= ~ID_AA64MMFR2_EL1_CCIDX_MASK; ··· 1998 1997 return val; 1999 1998 } 2000 1999 2000 2000 + /* 2001 2001 + * Older versions of KVM erroneously claim support for FEAT_DoubleLock with 2002 2002 + * NV-enabled VMs on unsupporting hardware. Silently ignore the incorrect 2003 2003 + * value if it is consistent with the bug. 2004 2004 + */ 2005 2005 + static bool ignore_feat_doublelock(struct kvm_vcpu *vcpu, u64 val) 2006 2006 + { 2007 2007 + u8 host, user; 2008 2008 + 2009 2009 + if (!vcpu_has_nv(vcpu)) 2010 2010 + return false; 2011 2011 + 2012 2012 + host = SYS_FIELD_GET(ID_AA64DFR0_EL1, DoubleLock, 2013 2013 + read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1)); 2014 2014 + user = SYS_FIELD_GET(ID_AA64DFR0_EL1, DoubleLock, val); 2015 2015 + 2016 2016 + return host == ID_AA64DFR0_EL1_DoubleLock_NI && 2017 2017 + user == ID_AA64DFR0_EL1_DoubleLock_IMP; 2018 2018 + } 2019 2019 + 2001 2020 static int set_id_aa64dfr0_el1(struct kvm_vcpu *vcpu, 2002 2021 const struct sys_reg_desc *rd, 2003 2022 u64 val) ··· 2048 2027 */ 2049 2028 if (debugver < ID_AA64DFR0_EL1_DebugVer_IMP) 2050 2029 return -EINVAL; 2030 2030 + 2031 2031 + if (ignore_feat_doublelock(vcpu, val)) { 2032 2032 + val &= ~ID_AA64DFR0_EL1_DoubleLock; 2033 2033 + val |= SYS_FIELD_PREP_ENUM(ID_AA64DFR0_EL1, DoubleLock, NI); 2034 2034 + } 2051 2035 2052 2036 return set_id_reg(vcpu, rd, val); 2053 2037 } ··· 2174 2148 return set_id_reg(vcpu, rd, user_val); 2175 2149 } 2176 2150 2151 2151 + /* 2152 2152 + * Allow userspace to de-feature a stage-2 translation granule but prevent it 2153 2153 + * from claiming the impossible. 2154 2154 + */ 2155 2155 + #define tgran2_val_allowed(tg, safe, user) \ 2156 2156 + ({ \ 2157 2157 + u8 __s = SYS_FIELD_GET(ID_AA64MMFR0_EL1, tg, safe); \ 2158 2158 + u8 __u = SYS_FIELD_GET(ID_AA64MMFR0_EL1, tg, user); \ 2159 2159 + \ 2160 2160 + __s == __u || __u == ID_AA64MMFR0_EL1_##tg##_NI; \ 2161 2161 + }) 2162 2162 + 2177 2163 static int set_id_aa64mmfr0_el1(struct kvm_vcpu *vcpu, 2178 2164 const struct sys_reg_desc *rd, u64 user_val) 2179 2165 { 2180 2166 u64 sanitized_val = kvm_read_sanitised_id_reg(vcpu, rd); 2181 2181 - u64 tgran2_mask = ID_AA64MMFR0_EL1_TGRAN4_2_MASK | 2182 2182 - ID_AA64MMFR0_EL1_TGRAN16_2_MASK | 2183 2183 - ID_AA64MMFR0_EL1_TGRAN64_2_MASK; 2184 2167 2185 2185 - if (vcpu_has_nv(vcpu) && 2186 2186 - ((sanitized_val & tgran2_mask) != (user_val & tgran2_mask))) 2168 2168 + if (!vcpu_has_nv(vcpu)) 2169 2169 + return set_id_reg(vcpu, rd, user_val); 2170 2170 + 2171 2171 + if (!tgran2_val_allowed(TGRAN4_2, sanitized_val, user_val) || 2172 2172 + !tgran2_val_allowed(TGRAN16_2, sanitized_val, user_val) || 2173 2173 + !tgran2_val_allowed(TGRAN64_2, sanitized_val, user_val)) 2187 2174 return -EINVAL; 2188 2175 2189 2176 return set_id_reg(vcpu, rd, user_val); ··· 3180 3141 ID_AA64ISAR2_EL1_APA3 | 3181 3142 ID_AA64ISAR2_EL1_GPA3)), 3182 3143 ID_WRITABLE(ID_AA64ISAR3_EL1, (ID_AA64ISAR3_EL1_FPRCVT | 3144 3144 + ID_AA64ISAR3_EL1_LSFE | 3183 3145 ID_AA64ISAR3_EL1_FAMINMAX)), 3184 3146 ID_UNALLOCATED(6,4), 3185 3147 ID_UNALLOCATED(6,5), ··· 3192 3152 ~(ID_AA64MMFR0_EL1_RES0 | 3193 3153 ID_AA64MMFR0_EL1_ASIDBITS)), 3194 3154 ID_WRITABLE(ID_AA64MMFR1_EL1, ~(ID_AA64MMFR1_EL1_RES0 | 3195 3195 - ID_AA64MMFR1_EL1_HCX | 3196 3196 - ID_AA64MMFR1_EL1_TWED | 3197 3155 ID_AA64MMFR1_EL1_XNX | 3198 3156 ID_AA64MMFR1_EL1_VH | 3199 3157 ID_AA64MMFR1_EL1_VMIDBits)), ··· 3276 3238 { SYS_DESC(SYS_PMBLIMITR_EL1), undef_access }, 3277 3239 { SYS_DESC(SYS_PMBPTR_EL1), undef_access }, 3278 3240 { SYS_DESC(SYS_PMBSR_EL1), undef_access }, 3241 3241 + { SYS_DESC(SYS_PMSDSFR_EL1), undef_access }, 3279 3242 /* PMBIDR_EL1 is not trapped */ 3280 3243 3281 3244 { PMU_SYS_REG(PMINTENSET_EL1),

+3 -11

arch/arm64/kvm/vgic/vgic-init.c

reviewed

··· 554 554 * Also map the virtual CPU interface into the VM. 555 555 * v2 calls vgic_init() if not already done. 556 556 * v3 and derivatives return an error if the VGIC is not initialized. 557 557 - * vgic_ready() returns true if this function has succeeded. 558 557 */ 559 558 int kvm_vgic_map_resources(struct kvm *kvm) 560 559 { ··· 562 563 gpa_t dist_base; 563 564 int ret = 0; 564 565 565 565 - if (likely(vgic_ready(kvm))) 566 566 + if (likely(smp_load_acquire(&dist->ready))) 566 567 return 0; 567 568 568 569 mutex_lock(&kvm->slots_lock); 569 570 mutex_lock(&kvm->arch.config_lock); 570 570 - if (vgic_ready(kvm)) 571 571 + if (dist->ready) 571 572 goto out; 572 573 573 574 if (!irqchip_in_kernel(kvm)) ··· 593 594 goto out_slots; 594 595 } 595 596 596 596 - /* 597 597 - * kvm_io_bus_register_dev() guarantees all readers see the new MMIO 598 598 - * registration before returning through synchronize_srcu(), which also 599 599 - * implies a full memory barrier. As such, marking the distributor as 600 600 - * 'ready' here is guaranteed to be ordered after all vCPUs having seen 601 601 - * a completely configured distributor. 602 602 - */ 603 603 - dist->ready = true; 597 597 + smp_store_release(&dist->ready, true); 604 598 goto out_slots; 605 599 out: 606 600 mutex_unlock(&kvm->arch.config_lock);

+8

arch/arm64/kvm/vgic/vgic-v3.c

reviewed

··· 588 588 } 589 589 590 590 DEFINE_STATIC_KEY_FALSE(vgic_v3_cpuif_trap); 591 591 + DEFINE_STATIC_KEY_FALSE(vgic_v3_has_v2_compat); 591 592 592 593 static int __init early_group0_trap_cfg(char *buf) 593 594 { ··· 697 696 698 697 if (kvm_vgic_global_state.vcpu_base == 0) 699 698 kvm_info("disabling GICv2 emulation\n"); 699 699 + 700 700 + /* 701 701 + * Flip the static branch if the HW supports v2, even if we're 702 702 + * not using it (such as in protected mode). 703 703 + */ 704 704 + if (has_v2) 705 705 + static_branch_enable(&vgic_v3_has_v2_compat); 700 706 701 707 if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_30115)) { 702 708 group0_trap = true;

+1 -1

arch/arm64/kvm/vgic/vgic-v5.c

reviewed

··· 15 15 u64 ich_vtr_el2; 16 16 int ret; 17 17 18 18 - if (!info->has_gcie_v3_compat) 18 18 + if (!cpus_have_final_cap(ARM64_HAS_GICV5_LEGACY)) 19 19 return -ENODEV; 20 20 21 21 kvm_vgic_global_state.type = VGIC_V5;

+1

arch/arm64/tools/cpucaps

reviewed

··· 37 37 HAS_GENERIC_AUTH_IMP_DEF 38 38 HAS_GICV3_CPUIF 39 39 HAS_GICV5_CPUIF 40 40 + HAS_GICV5_LEGACY 40 41 HAS_GIC_PRIO_MASKING 41 42 HAS_GIC_PRIO_RELAXED_SYNC 42 43 HAS_HCR_NV1

+14 -1

arch/loongarch/include/asm/kvm_pch_pic.h

reviewed

··· 34 34 #define PCH_PIC_INT_ISR_END 0x3af 35 35 #define PCH_PIC_POLARITY_START 0x3e0 36 36 #define PCH_PIC_POLARITY_END 0x3e7 37 37 - #define PCH_PIC_INT_ID_VAL 0x7000000UL 37 37 + #define PCH_PIC_INT_ID_VAL 0x7UL 38 38 #define PCH_PIC_INT_ID_VER 0x1UL 39 39 + 40 40 + union pch_pic_id { 41 41 + struct { 42 42 + uint8_t reserved_0[3]; 43 43 + uint8_t id; 44 44 + uint8_t version; 45 45 + uint8_t reserved_1; 46 46 + uint8_t irq_num; 47 47 + uint8_t reserved_2; 48 48 + } desc; 49 49 + uint64_t data; 50 50 + }; 39 51 40 52 struct loongarch_pch_pic { 41 53 spinlock_t lock; 42 54 struct kvm *kvm; 43 55 struct kvm_io_device device; 56 56 + union pch_pic_id id; 44 57 uint64_t mask; /* 1:disable irq, 0:enable irq */ 45 58 uint64_t htmsi_en; /* 1:msi */ 46 59 uint64_t edge; /* 1:edge triggered, 0:level triggered */

+1

arch/loongarch/include/uapi/asm/kvm.h

reviewed

··· 103 103 #define KVM_LOONGARCH_VM_FEAT_PMU 5 104 104 #define KVM_LOONGARCH_VM_FEAT_PV_IPI 6 105 105 #define KVM_LOONGARCH_VM_FEAT_PV_STEALTIME 7 106 106 + #define KVM_LOONGARCH_VM_FEAT_PTW 8 106 107 107 108 /* Device Control API on vcpu fd */ 108 109 #define KVM_LOONGARCH_VCPU_CPUCFG 0

+10 -9

arch/loongarch/kvm/exit.c

reviewed

··· 218 218 } 219 219 trace_kvm_iocsr(KVM_TRACE_IOCSR_WRITE, run->iocsr_io.len, addr, val); 220 220 } else { 221 221 + vcpu->arch.io_gpr = rd; /* Set register id for iocsr read completion */ 221 222 idx = srcu_read_lock(&vcpu->kvm->srcu); 222 222 - ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, run->iocsr_io.len, val); 223 223 + ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, 224 224 + run->iocsr_io.len, run->iocsr_io.data); 223 225 srcu_read_unlock(&vcpu->kvm->srcu, idx); 224 224 - if (ret == 0) 226 226 + if (ret == 0) { 227 227 + kvm_complete_iocsr_read(vcpu, run); 225 228 ret = EMULATE_DONE; 226 226 - else { 229 229 + } else 227 230 ret = EMULATE_DO_IOCSR; 228 228 - /* Save register id for iocsr read completion */ 229 229 - vcpu->arch.io_gpr = rd; 230 230 - } 231 231 trace_kvm_iocsr(KVM_TRACE_IOCSR_READ, run->iocsr_io.len, addr, NULL); 232 232 } 233 233 ··· 468 468 if (ret == EMULATE_DO_MMIO) { 469 469 trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len, run->mmio.phys_addr, NULL); 470 470 471 471 + vcpu->arch.io_gpr = rd; /* Set for kvm_complete_mmio_read() use */ 472 472 + 471 473 /* 472 474 * If mmio device such as PCH-PIC is emulated in KVM, 473 475 * it need not return to user space to handle the mmio ··· 477 475 */ 478 476 idx = srcu_read_lock(&vcpu->kvm->srcu); 479 477 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, vcpu->arch.badv, 480 480 - run->mmio.len, &vcpu->arch.gprs[rd]); 478 478 + run->mmio.len, run->mmio.data); 481 479 srcu_read_unlock(&vcpu->kvm->srcu, idx); 482 480 if (!ret) { 481 481 + kvm_complete_mmio_read(vcpu, run); 483 482 update_pc(&vcpu->arch); 484 483 vcpu->mmio_needed = 0; 485 484 return EMULATE_DONE; 486 485 } 487 486 488 488 - /* Set for kvm_complete_mmio_read() use */ 489 489 - vcpu->arch.io_gpr = rd; 490 487 run->mmio.is_write = 0; 491 488 vcpu->mmio_is_write = 0; 492 489 return EMULATE_DO_MMIO;

+48 -32

arch/loongarch/kvm/intc/ipi.c

reviewed

··· 7 7 #include <asm/kvm_ipi.h> 8 8 #include <asm/kvm_vcpu.h> 9 9 10 10 + static void ipi_set(struct kvm_vcpu *vcpu, uint32_t data) 11 11 + { 12 12 + uint32_t status; 13 13 + struct kvm_interrupt irq; 14 14 + 15 15 + spin_lock(&vcpu->arch.ipi_state.lock); 16 16 + status = vcpu->arch.ipi_state.status; 17 17 + vcpu->arch.ipi_state.status |= data; 18 18 + spin_unlock(&vcpu->arch.ipi_state.lock); 19 19 + if ((status == 0) && data) { 20 20 + irq.irq = LARCH_INT_IPI; 21 21 + kvm_vcpu_ioctl_interrupt(vcpu, &irq); 22 22 + } 23 23 + } 24 24 + 10 25 static void ipi_send(struct kvm *kvm, uint64_t data) 11 26 { 12 12 - int cpu, action; 13 13 - uint32_t status; 27 27 + int cpu; 14 28 struct kvm_vcpu *vcpu; 15 15 - struct kvm_interrupt irq; 16 29 17 30 cpu = ((data & 0xffffffff) >> 16) & 0x3ff; 18 31 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); ··· 34 21 return; 35 22 } 36 23 37 37 - action = BIT(data & 0x1f); 38 38 - spin_lock(&vcpu->arch.ipi_state.lock); 39 39 - status = vcpu->arch.ipi_state.status; 40 40 - vcpu->arch.ipi_state.status |= action; 41 41 - spin_unlock(&vcpu->arch.ipi_state.lock); 42 42 - if (status == 0) { 43 43 - irq.irq = LARCH_INT_IPI; 44 44 - kvm_vcpu_ioctl_interrupt(vcpu, &irq); 45 45 - } 24 24 + ipi_set(vcpu, BIT(data & 0x1f)); 46 25 } 47 26 48 27 static void ipi_clear(struct kvm_vcpu *vcpu, uint64_t data) ··· 101 96 spin_unlock(&vcpu->arch.ipi_state.lock); 102 97 } 103 98 99 99 + static int mail_send(struct kvm *kvm, uint64_t data) 100 100 + { 101 101 + int i, cpu, mailbox, offset; 102 102 + uint32_t val = 0, mask = 0; 103 103 + struct kvm_vcpu *vcpu; 104 104 + 105 105 + cpu = ((data & 0xffffffff) >> 16) & 0x3ff; 106 106 + vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); 107 107 + if (unlikely(vcpu == NULL)) { 108 108 + kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); 109 109 + return -EINVAL; 110 110 + } 111 111 + mailbox = ((data & 0xffffffff) >> 2) & 0x7; 112 112 + offset = IOCSR_IPI_BUF_20 + mailbox * 4; 113 113 + if ((data >> 27) & 0xf) { 114 114 + val = read_mailbox(vcpu, offset, 4); 115 115 + for (i = 0; i < 4; i++) 116 116 + if (data & (BIT(27 + i))) 117 117 + mask |= (0xff << (i * 8)); 118 118 + val &= mask; 119 119 + } 120 120 + 121 121 + val |= ((uint32_t)(data >> 32) & ~mask); 122 122 + write_mailbox(vcpu, offset, val, 4); 123 123 + 124 124 + return 0; 125 125 + } 126 126 + 104 127 static int send_ipi_data(struct kvm_vcpu *vcpu, gpa_t addr, uint64_t data) 105 128 { 106 129 int i, idx, ret; ··· 163 130 kvm_err("%s: : write data to addr %llx failed\n", __func__, addr); 164 131 165 132 return ret; 166 166 - } 167 167 - 168 168 - static int mail_send(struct kvm *kvm, uint64_t data) 169 169 - { 170 170 - int cpu, mailbox, offset; 171 171 - struct kvm_vcpu *vcpu; 172 172 - 173 173 - cpu = ((data & 0xffffffff) >> 16) & 0x3ff; 174 174 - vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); 175 175 - if (unlikely(vcpu == NULL)) { 176 176 - kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); 177 177 - return -EINVAL; 178 178 - } 179 179 - mailbox = ((data & 0xffffffff) >> 2) & 0x7; 180 180 - offset = IOCSR_IPI_BASE + IOCSR_IPI_BUF_20 + mailbox * 4; 181 181 - 182 182 - return send_ipi_data(vcpu, offset, data); 183 133 } 184 134 185 135 static int any_send(struct kvm *kvm, uint64_t data) ··· 247 231 spin_unlock(&vcpu->arch.ipi_state.lock); 248 232 break; 249 233 case IOCSR_IPI_SET: 250 250 - ret = -EINVAL; 234 234 + ipi_set(vcpu, data); 251 235 break; 252 236 case IOCSR_IPI_CLEAR: 253 237 /* Just clear the status of the current vcpu */ ··· 266 250 ipi_send(vcpu->kvm, data); 267 251 break; 268 252 case IOCSR_MAIL_SEND: 269 269 - ret = mail_send(vcpu->kvm, *(uint64_t *)val); 253 253 + ret = mail_send(vcpu->kvm, data); 270 254 break; 271 255 case IOCSR_ANY_SEND: 272 272 - ret = any_send(vcpu->kvm, *(uint64_t *)val); 256 256 + ret = any_send(vcpu->kvm, data); 273 257 break; 274 258 default: 275 259 kvm_err("%s: unknown addr: %llx\n", __func__, addr);

+97 -142

arch/loongarch/kvm/intc/pch_pic.c

reviewed

··· 35 35 /* update batch irqs, the irq_mask is a bitmap of irqs */ 36 36 static void pch_pic_update_batch_irqs(struct loongarch_pch_pic *s, u64 irq_mask, int level) 37 37 { 38 38 - int irq, bits; 38 38 + unsigned int irq; 39 39 + DECLARE_BITMAP(irqs, 64) = { BITMAP_FROM_U64(irq_mask) }; 39 40 40 40 - /* find each irq by irqs bitmap and update each irq */ 41 41 - bits = sizeof(irq_mask) * 8; 42 42 - irq = find_first_bit((void *)&irq_mask, bits); 43 43 - while (irq < bits) { 41 41 + for_each_set_bit(irq, irqs, 64) 44 42 pch_pic_update_irq(s, irq, level); 45 45 - bitmap_clear((void *)&irq_mask, irq, 1); 46 46 - irq = find_first_bit((void *)&irq_mask, bits); 47 47 - } 48 43 } 49 44 50 45 /* called when a irq is triggered in pch pic */ ··· 72 77 eiointc_set_irq(kvm->arch.eiointc, irq, level); 73 78 } 74 79 75 75 - /* 76 76 - * pch pic register is 64-bit, but it is accessed by 32-bit, 77 77 - * so we use high to get whether low or high 32 bits we want 78 78 - * to read. 79 79 - */ 80 80 - static u32 pch_pic_read_reg(u64 *s, int high) 81 81 - { 82 82 - u64 val = *s; 83 83 - 84 84 - /* read the high 32 bits when high is 1 */ 85 85 - return high ? (u32)(val >> 32) : (u32)val; 86 86 - } 87 87 - 88 88 - /* 89 89 - * pch pic register is 64-bit, but it is accessed by 32-bit, 90 90 - * so we use high to get whether low or high 32 bits we want 91 91 - * to write. 92 92 - */ 93 93 - static u32 pch_pic_write_reg(u64 *s, int high, u32 v) 94 94 - { 95 95 - u64 val = *s, data = v; 96 96 - 97 97 - if (high) { 98 98 - /* 99 99 - * Clear val high 32 bits 100 100 - * Write the high 32 bits when the high is 1 101 101 - */ 102 102 - *s = (val << 32 >> 32) | (data << 32); 103 103 - val >>= 32; 104 104 - } else 105 105 - /* 106 106 - * Clear val low 32 bits 107 107 - * Write the low 32 bits when the high is 0 108 108 - */ 109 109 - *s = (val >> 32 << 32) | v; 110 110 - 111 111 - return (u32)val; 112 112 - } 113 113 - 114 80 static int loongarch_pch_pic_read(struct loongarch_pch_pic *s, gpa_t addr, int len, void *val) 115 81 { 116 116 - int offset, index, ret = 0; 117 117 - u32 data = 0; 118 118 - u64 int_id = 0; 82 82 + int ret = 0, offset; 83 83 + u64 data = 0; 84 84 + void *ptemp; 119 85 120 86 offset = addr - s->pch_pic_base; 87 87 + offset -= offset & 7; 121 88 122 89 spin_lock(&s->lock); 123 90 switch (offset) { 124 91 case PCH_PIC_INT_ID_START ... PCH_PIC_INT_ID_END: 125 125 - /* int id version */ 126 126 - int_id |= (u64)PCH_PIC_INT_ID_VER << 32; 127 127 - /* irq number */ 128 128 - int_id |= (u64)31 << (32 + 16); 129 129 - /* int id value */ 130 130 - int_id |= PCH_PIC_INT_ID_VAL; 131 131 - *(u64 *)val = int_id; 92 92 + data = s->id.data; 132 93 break; 133 94 case PCH_PIC_MASK_START ... PCH_PIC_MASK_END: 134 134 - offset -= PCH_PIC_MASK_START; 135 135 - index = offset >> 2; 136 136 - /* read mask reg */ 137 137 - data = pch_pic_read_reg(&s->mask, index); 138 138 - *(u32 *)val = data; 95 95 + data = s->mask; 139 96 break; 140 97 case PCH_PIC_HTMSI_EN_START ... PCH_PIC_HTMSI_EN_END: 141 141 - offset -= PCH_PIC_HTMSI_EN_START; 142 142 - index = offset >> 2; 143 98 /* read htmsi enable reg */ 144 144 - data = pch_pic_read_reg(&s->htmsi_en, index); 145 145 - *(u32 *)val = data; 99 99 + data = s->htmsi_en; 146 100 break; 147 101 case PCH_PIC_EDGE_START ... PCH_PIC_EDGE_END: 148 148 - offset -= PCH_PIC_EDGE_START; 149 149 - index = offset >> 2; 150 102 /* read edge enable reg */ 151 151 - data = pch_pic_read_reg(&s->edge, index); 152 152 - *(u32 *)val = data; 103 103 + data = s->edge; 153 104 break; 154 105 case PCH_PIC_AUTO_CTRL0_START ... PCH_PIC_AUTO_CTRL0_END: 155 106 case PCH_PIC_AUTO_CTRL1_START ... PCH_PIC_AUTO_CTRL1_END: 156 107 /* we only use default mode: fixed interrupt distribution mode */ 157 157 - *(u32 *)val = 0; 158 108 break; 159 109 case PCH_PIC_ROUTE_ENTRY_START ... PCH_PIC_ROUTE_ENTRY_END: 160 110 /* only route to int0: eiointc */ 161 161 - *(u8 *)val = 1; 111 111 + ptemp = s->route_entry + (offset - PCH_PIC_ROUTE_ENTRY_START); 112 112 + data = *(u64 *)ptemp; 162 113 break; 163 114 case PCH_PIC_HTMSI_VEC_START ... PCH_PIC_HTMSI_VEC_END: 164 164 - offset -= PCH_PIC_HTMSI_VEC_START; 165 115 /* read htmsi vector */ 166 166 - data = s->htmsi_vector[offset]; 167 167 - *(u8 *)val = data; 116 116 + ptemp = s->htmsi_vector + (offset - PCH_PIC_HTMSI_VEC_START); 117 117 + data = *(u64 *)ptemp; 168 118 break; 169 119 case PCH_PIC_POLARITY_START ... PCH_PIC_POLARITY_END: 170 170 - /* we only use defalut value 0: high level triggered */ 171 171 - *(u32 *)val = 0; 120 120 + data = s->polarity; 121 121 + break; 122 122 + case PCH_PIC_INT_IRR_START: 123 123 + data = s->irr; 124 124 + break; 125 125 + case PCH_PIC_INT_ISR_START: 126 126 + data = s->isr; 172 127 break; 173 128 default: 174 129 ret = -EINVAL; 175 130 } 176 131 spin_unlock(&s->lock); 132 132 + 133 133 + if (ret == 0) { 134 134 + offset = (addr - s->pch_pic_base) & 7; 135 135 + data = data >> (offset * 8); 136 136 + memcpy(val, &data, len); 137 137 + } 177 138 178 139 return ret; 179 140 } ··· 161 210 static int loongarch_pch_pic_write(struct loongarch_pch_pic *s, gpa_t addr, 162 211 int len, const void *val) 163 212 { 164 164 - int ret; 165 165 - u32 old, data, offset, index; 166 166 - u64 irq; 213 213 + int ret = 0, offset; 214 214 + u64 old, data, mask; 215 215 + void *ptemp; 167 216 168 168 - ret = 0; 169 169 - data = *(u32 *)val; 170 170 - offset = addr - s->pch_pic_base; 217 217 + switch (len) { 218 218 + case 1: 219 219 + data = *(u8 *)val; 220 220 + mask = 0xFF; 221 221 + break; 222 222 + case 2: 223 223 + data = *(u16 *)val; 224 224 + mask = USHRT_MAX; 225 225 + break; 226 226 + case 4: 227 227 + data = *(u32 *)val; 228 228 + mask = UINT_MAX; 229 229 + break; 230 230 + case 8: 231 231 + default: 232 232 + data = *(u64 *)val; 233 233 + mask = ULONG_MAX; 234 234 + break; 235 235 + } 236 236 + 237 237 + offset = (addr - s->pch_pic_base) & 7; 238 238 + mask = mask << (offset * 8); 239 239 + data = data << (offset * 8); 240 240 + offset = (addr - s->pch_pic_base) - offset; 171 241 172 242 spin_lock(&s->lock); 173 243 switch (offset) { 174 174 - case PCH_PIC_MASK_START ... PCH_PIC_MASK_END: 175 175 - offset -= PCH_PIC_MASK_START; 176 176 - /* get whether high or low 32 bits we want to write */ 177 177 - index = offset >> 2; 178 178 - old = pch_pic_write_reg(&s->mask, index, data); 179 179 - /* enable irq when mask value change to 0 */ 180 180 - irq = (old & ~data) << (32 * index); 181 181 - pch_pic_update_batch_irqs(s, irq, 1); 182 182 - /* disable irq when mask value change to 1 */ 183 183 - irq = (~old & data) << (32 * index); 184 184 - pch_pic_update_batch_irqs(s, irq, 0); 244 244 + case PCH_PIC_MASK_START: 245 245 + old = s->mask; 246 246 + s->mask = (old & ~mask) | data; 247 247 + if (old & ~data) 248 248 + pch_pic_update_batch_irqs(s, old & ~data, 1); 249 249 + if (~old & data) 250 250 + pch_pic_update_batch_irqs(s, ~old & data, 0); 185 251 break; 186 186 - case PCH_PIC_HTMSI_EN_START ... PCH_PIC_HTMSI_EN_END: 187 187 - offset -= PCH_PIC_HTMSI_EN_START; 188 188 - index = offset >> 2; 189 189 - pch_pic_write_reg(&s->htmsi_en, index, data); 252 252 + case PCH_PIC_HTMSI_EN_START: 253 253 + s->htmsi_en = (s->htmsi_en & ~mask) | data; 190 254 break; 191 191 - case PCH_PIC_EDGE_START ... PCH_PIC_EDGE_END: 192 192 - offset -= PCH_PIC_EDGE_START; 193 193 - index = offset >> 2; 194 194 - /* 1: edge triggered, 0: level triggered */ 195 195 - pch_pic_write_reg(&s->edge, index, data); 255 255 + case PCH_PIC_EDGE_START: 256 256 + s->edge = (s->edge & ~mask) | data; 196 257 break; 197 197 - case PCH_PIC_CLEAR_START ... PCH_PIC_CLEAR_END: 198 198 - offset -= PCH_PIC_CLEAR_START; 199 199 - index = offset >> 2; 200 200 - /* write 1 to clear edge irq */ 201 201 - old = pch_pic_read_reg(&s->irr, index); 202 202 - /* 203 203 - * get the irq bitmap which is edge triggered and 204 204 - * already set and to be cleared 205 205 - */ 206 206 - irq = old & pch_pic_read_reg(&s->edge, index) & data; 207 207 - /* write irr to the new state where irqs have been cleared */ 208 208 - pch_pic_write_reg(&s->irr, index, old & ~irq); 209 209 - /* update cleared irqs */ 210 210 - pch_pic_update_batch_irqs(s, irq, 0); 258 258 + case PCH_PIC_POLARITY_START: 259 259 + s->polarity = (s->polarity & ~mask) | data; 211 260 break; 212 212 - case PCH_PIC_AUTO_CTRL0_START ... PCH_PIC_AUTO_CTRL0_END: 213 213 - offset -= PCH_PIC_AUTO_CTRL0_START; 214 214 - index = offset >> 2; 215 215 - /* we only use default mode: fixed interrupt distribution mode */ 216 216 - pch_pic_write_reg(&s->auto_ctrl0, index, 0); 217 217 - break; 218 218 - case PCH_PIC_AUTO_CTRL1_START ... PCH_PIC_AUTO_CTRL1_END: 219 219 - offset -= PCH_PIC_AUTO_CTRL1_START; 220 220 - index = offset >> 2; 221 221 - /* we only use default mode: fixed interrupt distribution mode */ 222 222 - pch_pic_write_reg(&s->auto_ctrl1, index, 0); 223 223 - break; 224 224 - case PCH_PIC_ROUTE_ENTRY_START ... PCH_PIC_ROUTE_ENTRY_END: 225 225 - offset -= PCH_PIC_ROUTE_ENTRY_START; 226 226 - /* only route to int0: eiointc */ 227 227 - s->route_entry[offset] = 1; 261 261 + case PCH_PIC_CLEAR_START: 262 262 + old = s->irr & s->edge & data; 263 263 + if (old) { 264 264 + s->irr &= ~old; 265 265 + pch_pic_update_batch_irqs(s, old, 0); 266 266 + } 228 267 break; 229 268 case PCH_PIC_HTMSI_VEC_START ... PCH_PIC_HTMSI_VEC_END: 230 230 - /* route table to eiointc */ 231 231 - offset -= PCH_PIC_HTMSI_VEC_START; 232 232 - s->htmsi_vector[offset] = (u8)data; 269 269 + ptemp = s->htmsi_vector + (offset - PCH_PIC_HTMSI_VEC_START); 270 270 + *(u64 *)ptemp = (*(u64 *)ptemp & ~mask) | data; 233 271 break; 234 234 - case PCH_PIC_POLARITY_START ... PCH_PIC_POLARITY_END: 235 235 - offset -= PCH_PIC_POLARITY_START; 236 236 - index = offset >> 2; 237 237 - /* we only use defalut value 0: high level triggered */ 238 238 - pch_pic_write_reg(&s->polarity, index, 0); 272 272 + /* Not implemented */ 273 273 + case PCH_PIC_AUTO_CTRL0_START: 274 274 + case PCH_PIC_AUTO_CTRL1_START: 275 275 + case PCH_PIC_ROUTE_ENTRY_START ... PCH_PIC_ROUTE_ENTRY_END: 239 276 break; 240 277 default: 241 278 ret = -EINVAL; ··· 423 484 424 485 static int kvm_pch_pic_create(struct kvm_device *dev, u32 type) 425 486 { 426 426 - int ret; 487 487 + int i, ret, irq_num; 427 488 struct kvm *kvm = dev->kvm; 428 489 struct loongarch_pch_pic *s; 429 490 ··· 439 500 if (!s) 440 501 return -ENOMEM; 441 502 503 503 + /* 504 504 + * Interrupt controller identification register 1 505 505 + * Bit 24-31 Interrupt Controller ID 506 506 + * Interrupt controller identification register 2 507 507 + * Bit 0-7 Interrupt Controller version number 508 508 + * Bit 16-23 The number of interrupt sources supported 509 509 + */ 510 510 + irq_num = 32; 511 511 + s->mask = -1UL; 512 512 + s->id.desc.id = PCH_PIC_INT_ID_VAL; 513 513 + s->id.desc.version = PCH_PIC_INT_ID_VER; 514 514 + s->id.desc.irq_num = irq_num - 1; 515 515 + for (i = 0; i < irq_num; i++) { 516 516 + s->route_entry[i] = 1; 517 517 + s->htmsi_vector[i] = i; 518 518 + } 442 519 spin_lock_init(&s->lock); 443 520 s->kvm = kvm; 444 521 kvm->arch.pch_pic = s;

+35

arch/loongarch/kvm/trace.h

reviewed

··· 161 161 __entry->pc) 162 162 ); 163 163 164 164 + #define KVM_TRACE_IOCSR_READ_UNSATISFIED 0 165 165 + #define KVM_TRACE_IOCSR_READ 1 166 166 + #define KVM_TRACE_IOCSR_WRITE 2 167 167 + 168 168 + #define kvm_trace_symbol_iocsr \ 169 169 + { KVM_TRACE_IOCSR_READ_UNSATISFIED, "unsatisfied-read" }, \ 170 170 + { KVM_TRACE_IOCSR_READ, "read" }, \ 171 171 + { KVM_TRACE_IOCSR_WRITE, "write" } 172 172 + 173 173 + TRACE_EVENT(kvm_iocsr, 174 174 + TP_PROTO(int type, int len, u64 gpa, void *val), 175 175 + TP_ARGS(type, len, gpa, val), 176 176 + 177 177 + TP_STRUCT__entry( 178 178 + __field( u32, type ) 179 179 + __field( u32, len ) 180 180 + __field( u64, gpa ) 181 181 + __field( u64, val ) 182 182 + ), 183 183 + 184 184 + TP_fast_assign( 185 185 + __entry->type = type; 186 186 + __entry->len = len; 187 187 + __entry->gpa = gpa; 188 188 + __entry->val = 0; 189 189 + if (val) 190 190 + memcpy(&__entry->val, val, 191 191 + min_t(u32, sizeof(__entry->val), len)); 192 192 + ), 193 193 + 194 194 + TP_printk("iocsr %s len %u gpa 0x%llx val 0x%llx", 195 195 + __print_symbolic(__entry->type, kvm_trace_symbol_iocsr), 196 196 + __entry->len, __entry->gpa, __entry->val) 197 197 + ); 198 198 + 164 199 TRACE_EVENT(kvm_vpid_change, 165 200 TP_PROTO(struct kvm_vcpu *vcpu, unsigned long vpid), 166 201 TP_ARGS(vcpu, vpid),

+2

arch/loongarch/kvm/vcpu.c

reviewed

··· 680 680 *v |= CPUCFG2_ARMBT; 681 681 if (cpu_has_lbt_mips) 682 682 *v |= CPUCFG2_MIPSBT; 683 683 + if (cpu_has_ptw) 684 684 + *v |= CPUCFG2_PTW; 683 685 684 686 return 0; 685 687 case LOONGARCH_CPUCFG3:

+4

arch/loongarch/kvm/vm.c

reviewed

··· 146 146 if (kvm_pvtime_supported()) 147 147 return 0; 148 148 return -ENXIO; 149 149 + case KVM_LOONGARCH_VM_FEAT_PTW: 150 150 + if (cpu_has_ptw) 151 151 + return 0; 152 152 + return -ENXIO; 149 153 default: 150 154 return -ENXIO; 151 155 }

+4

arch/riscv/include/asm/kvm_host.h

reviewed

··· 21 21 #include <asm/kvm_vcpu_fp.h> 22 22 #include <asm/kvm_vcpu_insn.h> 23 23 #include <asm/kvm_vcpu_sbi.h> 24 24 + #include <asm/kvm_vcpu_sbi_fwft.h> 24 25 #include <asm/kvm_vcpu_timer.h> 25 26 #include <asm/kvm_vcpu_pmu.h> 26 27 ··· 263 262 264 263 /* Performance monitoring context */ 265 264 struct kvm_pmu pmu_context; 265 265 + 266 266 + /* Firmware feature SBI extension context */ 267 267 + struct kvm_sbi_fwft fwft_context; 266 268 267 269 /* 'static' configurations which are set only once */ 268 270 struct kvm_vcpu_config cfg;

+3

arch/riscv/include/asm/kvm_vcpu_pmu.h

reviewed

··· 98 98 int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long saddr_low, 99 99 unsigned long saddr_high, unsigned long flags, 100 100 struct kvm_vcpu_sbi_return *retdata); 101 101 + int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low, 102 102 + unsigned long saddr_high, unsigned long num_events, 103 103 + unsigned long flags, struct kvm_vcpu_sbi_return *retdata); 101 104 void kvm_riscv_vcpu_pmu_deinit(struct kvm_vcpu *vcpu); 102 105 void kvm_riscv_vcpu_pmu_reset(struct kvm_vcpu *vcpu); 103 106

+14 -11

arch/riscv/include/asm/kvm_vcpu_sbi.h

reviewed

··· 11 11 12 12 #define KVM_SBI_IMPID 3 13 13 14 14 - #define KVM_SBI_VERSION_MAJOR 2 14 14 + #define KVM_SBI_VERSION_MAJOR 3 15 15 #define KVM_SBI_VERSION_MINOR 0 16 16 17 17 enum kvm_riscv_sbi_ext_status { ··· 59 59 void (*deinit)(struct kvm_vcpu *vcpu); 60 60 61 61 void (*reset)(struct kvm_vcpu *vcpu); 62 62 + 63 63 + unsigned long state_reg_subtype; 64 64 + unsigned long (*get_state_reg_count)(struct kvm_vcpu *vcpu); 65 65 + int (*get_state_reg_id)(struct kvm_vcpu *vcpu, int index, u64 *reg_id); 66 66 + int (*get_state_reg)(struct kvm_vcpu *vcpu, unsigned long reg_num, 67 67 + unsigned long reg_size, void *reg_val); 68 68 + int (*set_state_reg)(struct kvm_vcpu *vcpu, unsigned long reg_num, 69 69 + unsigned long reg_size, const void *reg_val); 62 70 }; 63 71 64 72 void kvm_riscv_vcpu_sbi_forward(struct kvm_vcpu *vcpu, struct kvm_run *run); ··· 77 69 unsigned long pc, unsigned long a1); 78 70 void kvm_riscv_vcpu_sbi_load_reset_state(struct kvm_vcpu *vcpu); 79 71 int kvm_riscv_vcpu_sbi_return(struct kvm_vcpu *vcpu, struct kvm_run *run); 72 72 + int kvm_riscv_vcpu_reg_indices_sbi_ext(struct kvm_vcpu *vcpu, u64 __user *uindices); 80 73 int kvm_riscv_vcpu_set_reg_sbi_ext(struct kvm_vcpu *vcpu, 81 74 const struct kvm_one_reg *reg); 82 75 int kvm_riscv_vcpu_get_reg_sbi_ext(struct kvm_vcpu *vcpu, 83 76 const struct kvm_one_reg *reg); 84 84 - int kvm_riscv_vcpu_set_reg_sbi(struct kvm_vcpu *vcpu, 85 85 - const struct kvm_one_reg *reg); 86 86 - int kvm_riscv_vcpu_get_reg_sbi(struct kvm_vcpu *vcpu, 87 87 - const struct kvm_one_reg *reg); 77 77 + int kvm_riscv_vcpu_reg_indices_sbi(struct kvm_vcpu *vcpu, u64 __user *uindices); 78 78 + int kvm_riscv_vcpu_set_reg_sbi(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); 79 79 + int kvm_riscv_vcpu_get_reg_sbi(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); 88 80 const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext( 89 81 struct kvm_vcpu *vcpu, unsigned long extid); 90 90 - bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx); 91 82 int kvm_riscv_vcpu_sbi_ecall(struct kvm_vcpu *vcpu, struct kvm_run *run); 92 83 void kvm_riscv_vcpu_sbi_init(struct kvm_vcpu *vcpu); 93 84 void kvm_riscv_vcpu_sbi_deinit(struct kvm_vcpu *vcpu); 94 85 void kvm_riscv_vcpu_sbi_reset(struct kvm_vcpu *vcpu); 95 95 - 96 96 - int kvm_riscv_vcpu_get_reg_sbi_sta(struct kvm_vcpu *vcpu, unsigned long reg_num, 97 97 - unsigned long *reg_val); 98 98 - int kvm_riscv_vcpu_set_reg_sbi_sta(struct kvm_vcpu *vcpu, unsigned long reg_num, 99 99 - unsigned long reg_val); 100 86 101 87 #ifdef CONFIG_RISCV_SBI_V01 102 88 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_v01; ··· 104 102 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_dbcn; 105 103 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_susp; 106 104 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta; 105 105 + extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_fwft; 107 106 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental; 108 107 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor; 109 108

+34

arch/riscv/include/asm/kvm_vcpu_sbi_fwft.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 2 + /* 3 3 + * Copyright (c) 2025 Rivos Inc. 4 4 + * 5 5 + * Authors: 6 6 + * Clément Léger <cleger@rivosinc.com> 7 7 + */ 8 8 + 9 9 + #ifndef __KVM_VCPU_RISCV_FWFT_H 10 10 + #define __KVM_VCPU_RISCV_FWFT_H 11 11 + 12 12 + #include <asm/sbi.h> 13 13 + 14 14 + struct kvm_sbi_fwft_feature; 15 15 + 16 16 + struct kvm_sbi_fwft_config { 17 17 + const struct kvm_sbi_fwft_feature *feature; 18 18 + bool supported; 19 19 + bool enabled; 20 20 + unsigned long flags; 21 21 + }; 22 22 + 23 23 + /* FWFT data structure per vcpu */ 24 24 + struct kvm_sbi_fwft { 25 25 + struct kvm_sbi_fwft_config *configs; 26 26 + #ifndef CONFIG_32BIT 27 27 + bool have_vs_pmlen_7; 28 28 + bool have_vs_pmlen_16; 29 29 + #endif 30 30 + }; 31 31 + 32 32 + #define vcpu_to_fwft(vcpu) (&(vcpu)->arch.fwft_context) 33 33 + 34 34 + #endif /* !__KVM_VCPU_RISCV_FWFT_H */

+13

arch/riscv/include/asm/sbi.h

reviewed

··· 136 136 SBI_EXT_PMU_COUNTER_FW_READ, 137 137 SBI_EXT_PMU_COUNTER_FW_READ_HI, 138 138 SBI_EXT_PMU_SNAPSHOT_SET_SHMEM, 139 139 + SBI_EXT_PMU_EVENT_GET_INFO, 139 140 }; 140 141 141 142 union sbi_pmu_ctr_info { ··· 160 159 u64 reserved[447]; 161 160 }; 162 161 162 162 + struct riscv_pmu_event_info { 163 163 + u32 event_idx; 164 164 + u32 output; 165 165 + u64 event_data; 166 166 + }; 167 167 + 168 168 + #define RISCV_PMU_EVENT_INFO_OUTPUT_MASK 0x01 169 169 + 163 170 #define RISCV_PMU_RAW_EVENT_MASK GENMASK_ULL(47, 0) 164 171 #define RISCV_PMU_PLAT_FW_EVENT_MASK GENMASK_ULL(61, 0) 172 172 + /* SBI v3.0 allows extended hpmeventX width value */ 173 173 + #define RISCV_PMU_RAW_EVENT_V2_MASK GENMASK_ULL(55, 0) 165 174 #define RISCV_PMU_RAW_EVENT_IDX 0x20000 175 175 + #define RISCV_PMU_RAW_EVENT_V2_IDX 0x30000 166 176 #define RISCV_PLAT_FW_EVENT 0xFFFF 167 177 168 178 /** General pmu event codes specified in SBI PMU extension */ ··· 231 219 SBI_PMU_EVENT_TYPE_HW = 0x0, 232 220 SBI_PMU_EVENT_TYPE_CACHE = 0x1, 233 221 SBI_PMU_EVENT_TYPE_RAW = 0x2, 222 222 + SBI_PMU_EVENT_TYPE_RAW_V2 = 0x3, 234 223 SBI_PMU_EVENT_TYPE_FW = 0xf, 235 224 }; 236 225

+21

arch/riscv/include/uapi/asm/kvm.h

reviewed

··· 56 56 unsigned long mimpid; 57 57 unsigned long zicboz_block_size; 58 58 unsigned long satp_mode; 59 59 + unsigned long zicbop_block_size; 59 60 }; 60 61 61 62 /* CORE registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */ ··· 186 185 KVM_RISCV_ISA_EXT_ZICCRSE, 187 186 KVM_RISCV_ISA_EXT_ZAAMO, 188 187 KVM_RISCV_ISA_EXT_ZALRSC, 188 188 + KVM_RISCV_ISA_EXT_ZICBOP, 189 189 + KVM_RISCV_ISA_EXT_ZFBFMIN, 190 190 + KVM_RISCV_ISA_EXT_ZVFBFMIN, 191 191 + KVM_RISCV_ISA_EXT_ZVFBFWMA, 189 192 KVM_RISCV_ISA_EXT_MAX, 190 193 }; 191 194 ··· 210 205 KVM_RISCV_SBI_EXT_DBCN, 211 206 KVM_RISCV_SBI_EXT_STA, 212 207 KVM_RISCV_SBI_EXT_SUSP, 208 208 + KVM_RISCV_SBI_EXT_FWFT, 213 209 KVM_RISCV_SBI_EXT_MAX, 214 210 }; 215 211 ··· 218 212 struct kvm_riscv_sbi_sta { 219 213 unsigned long shmem_lo; 220 214 unsigned long shmem_hi; 215 215 + }; 216 216 + 217 217 + struct kvm_riscv_sbi_fwft_feature { 218 218 + unsigned long enable; 219 219 + unsigned long flags; 220 220 + unsigned long value; 221 221 + }; 222 222 + 223 223 + /* SBI FWFT extension registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */ 224 224 + struct kvm_riscv_sbi_fwft { 225 225 + struct kvm_riscv_sbi_fwft_feature misaligned_deleg; 226 226 + struct kvm_riscv_sbi_fwft_feature pointer_masking; 221 227 }; 222 228 223 229 /* Possible states for kvm_riscv_timer */ ··· 315 297 #define KVM_REG_RISCV_SBI_STA (0x0 << KVM_REG_RISCV_SUBTYPE_SHIFT) 316 298 #define KVM_REG_RISCV_SBI_STA_REG(name) \ 317 299 (offsetof(struct kvm_riscv_sbi_sta, name) / sizeof(unsigned long)) 300 300 + #define KVM_REG_RISCV_SBI_FWFT (0x1 << KVM_REG_RISCV_SUBTYPE_SHIFT) 301 301 + #define KVM_REG_RISCV_SBI_FWFT_REG(name) \ 302 302 + (offsetof(struct kvm_riscv_sbi_fwft, name) / sizeof(unsigned long)) 318 303 319 304 /* Device Control API: RISC-V AIA */ 320 305 #define KVM_DEV_RISCV_APLIC_ALIGN 0x1000

+1

arch/riscv/kvm/Makefile

reviewed

··· 27 27 kvm-$(CONFIG_RISCV_PMU_SBI) += vcpu_pmu.o 28 28 kvm-y += vcpu_sbi.o 29 29 kvm-y += vcpu_sbi_base.o 30 30 + kvm-y += vcpu_sbi_fwft.o 30 31 kvm-y += vcpu_sbi_hsm.o 31 32 kvm-$(CONFIG_RISCV_PMU_SBI) += vcpu_sbi_pmu.o 32 33 kvm-y += vcpu_sbi_replace.o

+24 -3

arch/riscv/kvm/gstage.c

reviewed

··· 321 321 if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV57X4) { 322 322 kvm_riscv_gstage_mode = HGATP_MODE_SV57X4; 323 323 kvm_riscv_gstage_pgd_levels = 5; 324 324 - goto skip_sv48x4_test; 324 324 + goto done; 325 325 } 326 326 327 327 /* Try Sv48x4 G-stage mode */ ··· 329 329 if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) { 330 330 kvm_riscv_gstage_mode = HGATP_MODE_SV48X4; 331 331 kvm_riscv_gstage_pgd_levels = 4; 332 332 + goto done; 332 333 } 333 333 - skip_sv48x4_test: 334 334 335 335 + /* Try Sv39x4 G-stage mode */ 336 336 + csr_write(CSR_HGATP, HGATP_MODE_SV39X4 << HGATP_MODE_SHIFT); 337 337 + if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV39X4) { 338 338 + kvm_riscv_gstage_mode = HGATP_MODE_SV39X4; 339 339 + kvm_riscv_gstage_pgd_levels = 3; 340 340 + goto done; 341 341 + } 342 342 + #else /* CONFIG_32BIT */ 343 343 + /* Try Sv32x4 G-stage mode */ 344 344 + csr_write(CSR_HGATP, HGATP_MODE_SV32X4 << HGATP_MODE_SHIFT); 345 345 + if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV32X4) { 346 346 + kvm_riscv_gstage_mode = HGATP_MODE_SV32X4; 347 347 + kvm_riscv_gstage_pgd_levels = 2; 348 348 + goto done; 349 349 + } 350 350 + #endif 351 351 + 352 352 + /* KVM depends on !HGATP_MODE_OFF */ 353 353 + kvm_riscv_gstage_mode = HGATP_MODE_OFF; 354 354 + kvm_riscv_gstage_pgd_levels = 0; 355 355 + 356 356 + done: 335 357 csr_write(CSR_HGATP, 0); 336 358 kvm_riscv_local_hfence_gvma_all(); 337 337 - #endif 338 359 }

+17 -16

arch/riscv/kvm/main.c

reviewed

··· 93 93 return rc; 94 94 95 95 kvm_riscv_gstage_mode_detect(); 96 96 + switch (kvm_riscv_gstage_mode) { 97 97 + case HGATP_MODE_SV32X4: 98 98 + str = "Sv32x4"; 99 99 + break; 100 100 + case HGATP_MODE_SV39X4: 101 101 + str = "Sv39x4"; 102 102 + break; 103 103 + case HGATP_MODE_SV48X4: 104 104 + str = "Sv48x4"; 105 105 + break; 106 106 + case HGATP_MODE_SV57X4: 107 107 + str = "Sv57x4"; 108 108 + break; 109 109 + default: 110 110 + kvm_riscv_nacl_exit(); 111 111 + return -ENODEV; 112 112 + } 96 113 97 114 kvm_riscv_gstage_vmid_detect(); 98 115 ··· 152 135 (rc) ? slist : "no features"); 153 136 } 154 137 155 155 - switch (kvm_riscv_gstage_mode) { 156 156 - case HGATP_MODE_SV32X4: 157 157 - str = "Sv32x4"; 158 158 - break; 159 159 - case HGATP_MODE_SV39X4: 160 160 - str = "Sv39x4"; 161 161 - break; 162 162 - case HGATP_MODE_SV48X4: 163 163 - str = "Sv48x4"; 164 164 - break; 165 165 - case HGATP_MODE_SV57X4: 166 166 - str = "Sv57x4"; 167 167 - break; 168 168 - default: 169 169 - return -ENODEV; 170 170 - } 171 138 kvm_info("using %s G-stage page table format\n", str); 172 139 173 140 kvm_info("VMID %ld bits available\n", kvm_riscv_gstage_vmid_bits());

+2 -1

arch/riscv/kvm/vcpu.c

reviewed

··· 133 133 134 134 /* Mark this VCPU never ran */ 135 135 vcpu->arch.ran_atleast_once = false; 136 136 + 137 137 + vcpu->arch.cfg.hedeleg = KVM_HEDELEG_DEFAULT; 136 138 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; 137 139 bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX); 138 140 ··· 572 570 cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0; 573 571 } 574 572 575 575 - cfg->hedeleg = KVM_HEDELEG_DEFAULT; 576 573 if (vcpu->guest_debug) 577 574 cfg->hedeleg &= ~BIT(EXC_BREAKPOINT); 578 575 }

+32 -63

arch/riscv/kvm/vcpu_onereg.c

reviewed

··· 65 65 KVM_ISA_EXT_ARR(ZCF), 66 66 KVM_ISA_EXT_ARR(ZCMOP), 67 67 KVM_ISA_EXT_ARR(ZFA), 68 68 + KVM_ISA_EXT_ARR(ZFBFMIN), 68 69 KVM_ISA_EXT_ARR(ZFH), 69 70 KVM_ISA_EXT_ARR(ZFHMIN), 70 71 KVM_ISA_EXT_ARR(ZICBOM), 72 72 + KVM_ISA_EXT_ARR(ZICBOP), 71 73 KVM_ISA_EXT_ARR(ZICBOZ), 72 74 KVM_ISA_EXT_ARR(ZICCRSE), 73 75 KVM_ISA_EXT_ARR(ZICNTR), ··· 90 88 KVM_ISA_EXT_ARR(ZTSO), 91 89 KVM_ISA_EXT_ARR(ZVBB), 92 90 KVM_ISA_EXT_ARR(ZVBC), 91 91 + KVM_ISA_EXT_ARR(ZVFBFMIN), 92 92 + KVM_ISA_EXT_ARR(ZVFBFWMA), 93 93 KVM_ISA_EXT_ARR(ZVFH), 94 94 KVM_ISA_EXT_ARR(ZVFHMIN), 95 95 KVM_ISA_EXT_ARR(ZVKB), ··· 177 173 case KVM_RISCV_ISA_EXT_C: 178 174 case KVM_RISCV_ISA_EXT_I: 179 175 case KVM_RISCV_ISA_EXT_M: 180 180 - case KVM_RISCV_ISA_EXT_SMNPM: 181 176 /* There is not architectural config bit to disable sscofpmf completely */ 182 177 case KVM_RISCV_ISA_EXT_SSCOFPMF: 183 178 case KVM_RISCV_ISA_EXT_SSNPM: ··· 202 199 case KVM_RISCV_ISA_EXT_ZCF: 203 200 case KVM_RISCV_ISA_EXT_ZCMOP: 204 201 case KVM_RISCV_ISA_EXT_ZFA: 202 202 + case KVM_RISCV_ISA_EXT_ZFBFMIN: 205 203 case KVM_RISCV_ISA_EXT_ZFH: 206 204 case KVM_RISCV_ISA_EXT_ZFHMIN: 205 205 + case KVM_RISCV_ISA_EXT_ZICBOP: 207 206 case KVM_RISCV_ISA_EXT_ZICCRSE: 208 207 case KVM_RISCV_ISA_EXT_ZICNTR: 209 208 case KVM_RISCV_ISA_EXT_ZICOND: ··· 225 220 case KVM_RISCV_ISA_EXT_ZTSO: 226 221 case KVM_RISCV_ISA_EXT_ZVBB: 227 222 case KVM_RISCV_ISA_EXT_ZVBC: 223 223 + case KVM_RISCV_ISA_EXT_ZVFBFMIN: 224 224 + case KVM_RISCV_ISA_EXT_ZVFBFWMA: 228 225 case KVM_RISCV_ISA_EXT_ZVFH: 229 226 case KVM_RISCV_ISA_EXT_ZVFHMIN: 230 227 case KVM_RISCV_ISA_EXT_ZVKB: ··· 284 277 reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; 285 278 break; 286 279 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 287 287 - if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) 280 280 + if (!riscv_isa_extension_available(NULL, ZICBOM)) 288 281 return -ENOENT; 289 282 reg_val = riscv_cbom_block_size; 290 283 break; 291 284 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 292 292 - if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOZ)) 285 285 + if (!riscv_isa_extension_available(NULL, ZICBOZ)) 293 286 return -ENOENT; 294 287 reg_val = riscv_cboz_block_size; 288 288 + break; 289 289 + case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): 290 290 + if (!riscv_isa_extension_available(NULL, ZICBOP)) 291 291 + return -ENOENT; 292 292 + reg_val = riscv_cbop_block_size; 295 293 break; 296 294 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 297 295 reg_val = vcpu->arch.mvendorid; ··· 378 366 } 379 367 break; 380 368 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 381 381 - if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) 369 369 + if (!riscv_isa_extension_available(NULL, ZICBOM)) 382 370 return -ENOENT; 383 371 if (reg_val != riscv_cbom_block_size) 384 372 return -EINVAL; 385 373 break; 386 374 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 387 387 - if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOZ)) 375 375 + if (!riscv_isa_extension_available(NULL, ZICBOZ)) 388 376 return -ENOENT; 389 377 if (reg_val != riscv_cboz_block_size) 378 378 + return -EINVAL; 379 379 + break; 380 380 + case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): 381 381 + if (!riscv_isa_extension_available(NULL, ZICBOP)) 382 382 + return -ENOENT; 383 383 + if (reg_val != riscv_cbop_block_size) 390 384 return -EINVAL; 391 385 break; 392 386 case KVM_REG_RISCV_CONFIG_REG(mvendorid): ··· 835 817 * was not available. 836 818 */ 837 819 if (i == KVM_REG_RISCV_CONFIG_REG(zicbom_block_size) && 838 838 - !riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) 820 820 + !riscv_isa_extension_available(NULL, ZICBOM)) 839 821 continue; 840 822 else if (i == KVM_REG_RISCV_CONFIG_REG(zicboz_block_size) && 841 841 - !riscv_isa_extension_available(vcpu->arch.isa, ZICBOZ)) 823 823 + !riscv_isa_extension_available(NULL, ZICBOZ)) 824 824 + continue; 825 825 + else if (i == KVM_REG_RISCV_CONFIG_REG(zicbop_block_size) && 826 826 + !riscv_isa_extension_available(NULL, ZICBOP)) 842 827 continue; 843 828 844 829 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; ··· 1082 1061 return copy_isa_ext_reg_indices(vcpu, NULL); 1083 1062 } 1084 1063 1085 1085 - static int copy_sbi_ext_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) 1086 1086 - { 1087 1087 - unsigned int n = 0; 1088 1088 - 1089 1089 - for (int i = 0; i < KVM_RISCV_SBI_EXT_MAX; i++) { 1090 1090 - u64 size = IS_ENABLED(CONFIG_32BIT) ? 1091 1091 - KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 1092 1092 - u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_SBI_EXT | 1093 1093 - KVM_REG_RISCV_SBI_SINGLE | i; 1094 1094 - 1095 1095 - if (!riscv_vcpu_supports_sbi_ext(vcpu, i)) 1096 1096 - continue; 1097 1097 - 1098 1098 - if (uindices) { 1099 1099 - if (put_user(reg, uindices)) 1100 1100 - return -EFAULT; 1101 1101 - uindices++; 1102 1102 - } 1103 1103 - 1104 1104 - n++; 1105 1105 - } 1106 1106 - 1107 1107 - return n; 1108 1108 - } 1109 1109 - 1110 1064 static unsigned long num_sbi_ext_regs(struct kvm_vcpu *vcpu) 1111 1065 { 1112 1112 - return copy_sbi_ext_reg_indices(vcpu, NULL); 1113 1113 - } 1114 1114 - 1115 1115 - static int copy_sbi_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) 1116 1116 - { 1117 1117 - struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; 1118 1118 - int total = 0; 1119 1119 - 1120 1120 - if (scontext->ext_status[KVM_RISCV_SBI_EXT_STA] == KVM_RISCV_SBI_EXT_STATUS_ENABLED) { 1121 1121 - u64 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 1122 1122 - int n = sizeof(struct kvm_riscv_sbi_sta) / sizeof(unsigned long); 1123 1123 - 1124 1124 - for (int i = 0; i < n; i++) { 1125 1125 - u64 reg = KVM_REG_RISCV | size | 1126 1126 - KVM_REG_RISCV_SBI_STATE | 1127 1127 - KVM_REG_RISCV_SBI_STA | i; 1128 1128 - 1129 1129 - if (uindices) { 1130 1130 - if (put_user(reg, uindices)) 1131 1131 - return -EFAULT; 1132 1132 - uindices++; 1133 1133 - } 1134 1134 - } 1135 1135 - 1136 1136 - total += n; 1137 1137 - } 1138 1138 - 1139 1139 - return total; 1066 1066 + return kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, NULL); 1140 1067 } 1141 1068 1142 1069 static inline unsigned long num_sbi_regs(struct kvm_vcpu *vcpu) 1143 1070 { 1144 1144 - return copy_sbi_reg_indices(vcpu, NULL); 1071 1071 + return kvm_riscv_vcpu_reg_indices_sbi(vcpu, NULL); 1145 1072 } 1146 1073 1147 1074 static inline unsigned long num_vector_regs(const struct kvm_vcpu *vcpu) ··· 1212 1243 return ret; 1213 1244 uindices += ret; 1214 1245 1215 1215 - ret = copy_sbi_ext_reg_indices(vcpu, uindices); 1246 1246 + ret = kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, uindices); 1216 1247 if (ret < 0) 1217 1248 return ret; 1218 1249 uindices += ret; 1219 1250 1220 1220 - ret = copy_sbi_reg_indices(vcpu, uindices); 1251 1251 + ret = kvm_riscv_vcpu_reg_indices_sbi(vcpu, uindices); 1221 1252 if (ret < 0) 1222 1253 return ret; 1223 1254 uindices += ret;

+65 -9

arch/riscv/kvm/vcpu_pmu.c

reviewed

··· 60 60 type = PERF_TYPE_HW_CACHE; 61 61 break; 62 62 case SBI_PMU_EVENT_TYPE_RAW: 63 63 + case SBI_PMU_EVENT_TYPE_RAW_V2: 63 64 case SBI_PMU_EVENT_TYPE_FW: 64 65 type = PERF_TYPE_RAW; 65 66 break; ··· 128 127 break; 129 128 case SBI_PMU_EVENT_TYPE_RAW: 130 129 config = evt_data & RISCV_PMU_RAW_EVENT_MASK; 130 130 + break; 131 131 + case SBI_PMU_EVENT_TYPE_RAW_V2: 132 132 + config = evt_data & RISCV_PMU_RAW_EVENT_V2_MASK; 131 133 break; 132 134 case SBI_PMU_EVENT_TYPE_FW: 133 135 if (ecode < SBI_PMU_FW_MAX) ··· 409 405 int snapshot_area_size = sizeof(struct riscv_pmu_snapshot_data); 410 406 int sbiret = 0; 411 407 gpa_t saddr; 412 412 - unsigned long hva; 413 413 - bool writable; 414 408 415 409 if (!kvpmu || flags) { 416 410 sbiret = SBI_ERR_INVALID_PARAM; ··· 430 428 goto out; 431 429 } 432 430 433 433 - hva = kvm_vcpu_gfn_to_hva_prot(vcpu, saddr >> PAGE_SHIFT, &writable); 434 434 - if (kvm_is_error_hva(hva) || !writable) { 435 435 - sbiret = SBI_ERR_INVALID_ADDRESS; 436 436 - goto out; 437 437 - } 438 438 - 439 431 kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC); 440 432 if (!kvpmu->sdata) 441 433 return -ENOMEM; 442 434 435 435 + /* No need to check writable slot explicitly as kvm_vcpu_write_guest does it internally */ 443 436 if (kvm_vcpu_write_guest(vcpu, saddr, kvpmu->sdata, snapshot_area_size)) { 444 437 kfree(kvpmu->sdata); 445 445 - sbiret = SBI_ERR_FAILURE; 438 438 + sbiret = SBI_ERR_INVALID_ADDRESS; 446 439 goto out; 447 440 } 448 441 ··· 445 448 446 449 out: 447 450 retdata->err_val = sbiret; 451 451 + 452 452 + return 0; 453 453 + } 454 454 + 455 455 + int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low, 456 456 + unsigned long saddr_high, unsigned long num_events, 457 457 + unsigned long flags, struct kvm_vcpu_sbi_return *retdata) 458 458 + { 459 459 + struct riscv_pmu_event_info *einfo = NULL; 460 460 + int shmem_size = num_events * sizeof(*einfo); 461 461 + gpa_t shmem; 462 462 + u32 eidx, etype; 463 463 + u64 econfig; 464 464 + int ret; 465 465 + 466 466 + if (flags != 0 || (saddr_low & (SZ_16 - 1) || num_events == 0)) { 467 467 + ret = SBI_ERR_INVALID_PARAM; 468 468 + goto out; 469 469 + } 470 470 + 471 471 + shmem = saddr_low; 472 472 + if (saddr_high != 0) { 473 473 + if (IS_ENABLED(CONFIG_32BIT)) { 474 474 + shmem |= ((gpa_t)saddr_high << 32); 475 475 + } else { 476 476 + ret = SBI_ERR_INVALID_ADDRESS; 477 477 + goto out; 478 478 + } 479 479 + } 480 480 + 481 481 + einfo = kzalloc(shmem_size, GFP_KERNEL); 482 482 + if (!einfo) 483 483 + return -ENOMEM; 484 484 + 485 485 + ret = kvm_vcpu_read_guest(vcpu, shmem, einfo, shmem_size); 486 486 + if (ret) { 487 487 + ret = SBI_ERR_FAILURE; 488 488 + goto free_mem; 489 489 + } 490 490 + 491 491 + for (int i = 0; i < num_events; i++) { 492 492 + eidx = einfo[i].event_idx; 493 493 + etype = kvm_pmu_get_perf_event_type(eidx); 494 494 + econfig = kvm_pmu_get_perf_event_config(eidx, einfo[i].event_data); 495 495 + ret = riscv_pmu_get_event_info(etype, econfig, NULL); 496 496 + einfo[i].output = (ret > 0) ? 1 : 0; 497 497 + } 498 498 + 499 499 + ret = kvm_vcpu_write_guest(vcpu, shmem, einfo, shmem_size); 500 500 + if (ret) { 501 501 + ret = SBI_ERR_INVALID_ADDRESS; 502 502 + goto free_mem; 503 503 + } 504 504 + 505 505 + ret = 0; 506 506 + free_mem: 507 507 + kfree(einfo); 508 508 + out: 509 509 + retdata->err_val = ret; 448 510 449 511 return 0; 450 512 }

+163 -35

arch/riscv/kvm/vcpu_sbi.c

reviewed

··· 79 79 .ext_ptr = &vcpu_sbi_ext_sta, 80 80 }, 81 81 { 82 82 + .ext_idx = KVM_RISCV_SBI_EXT_FWFT, 83 83 + .ext_ptr = &vcpu_sbi_ext_fwft, 84 84 + }, 85 85 + { 82 86 .ext_idx = KVM_RISCV_SBI_EXT_EXPERIMENTAL, 83 87 .ext_ptr = &vcpu_sbi_ext_experimental, 84 88 }, ··· 110 106 return sext; 111 107 } 112 108 113 113 - bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx) 109 109 + static bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx) 114 110 { 115 111 struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; 116 112 const struct kvm_riscv_sbi_extension_entry *sext; ··· 288 284 return 0; 289 285 } 290 286 287 287 + int kvm_riscv_vcpu_reg_indices_sbi_ext(struct kvm_vcpu *vcpu, u64 __user *uindices) 288 288 + { 289 289 + unsigned int n = 0; 290 290 + 291 291 + for (int i = 0; i < KVM_RISCV_SBI_EXT_MAX; i++) { 292 292 + u64 size = IS_ENABLED(CONFIG_32BIT) ? 293 293 + KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 294 294 + u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_SBI_EXT | 295 295 + KVM_REG_RISCV_SBI_SINGLE | i; 296 296 + 297 297 + if (!riscv_vcpu_supports_sbi_ext(vcpu, i)) 298 298 + continue; 299 299 + 300 300 + if (uindices) { 301 301 + if (put_user(reg, uindices)) 302 302 + return -EFAULT; 303 303 + uindices++; 304 304 + } 305 305 + 306 306 + n++; 307 307 + } 308 308 + 309 309 + return n; 310 310 + } 311 311 + 291 312 int kvm_riscv_vcpu_set_reg_sbi_ext(struct kvm_vcpu *vcpu, 292 313 const struct kvm_one_reg *reg) 293 314 { ··· 389 360 return 0; 390 361 } 391 362 392 392 - int kvm_riscv_vcpu_set_reg_sbi(struct kvm_vcpu *vcpu, 393 393 - const struct kvm_one_reg *reg) 363 363 + int kvm_riscv_vcpu_reg_indices_sbi(struct kvm_vcpu *vcpu, u64 __user *uindices) 394 364 { 395 395 - unsigned long __user *uaddr = 396 396 - (unsigned long __user *)(unsigned long)reg->addr; 397 397 - unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 398 398 - KVM_REG_SIZE_MASK | 399 399 - KVM_REG_RISCV_SBI_STATE); 400 400 - unsigned long reg_subtype, reg_val; 365 365 + struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; 366 366 + const struct kvm_riscv_sbi_extension_entry *entry; 367 367 + const struct kvm_vcpu_sbi_extension *ext; 368 368 + unsigned long state_reg_count; 369 369 + int i, j, rc, count = 0; 370 370 + u64 reg; 401 371 402 402 - if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 403 403 - return -EINVAL; 372 372 + for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { 373 373 + entry = &sbi_ext[i]; 374 374 + ext = entry->ext_ptr; 404 375 405 405 - if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id))) 406 406 - return -EFAULT; 376 376 + if (!ext->get_state_reg_count || 377 377 + scontext->ext_status[entry->ext_idx] != KVM_RISCV_SBI_EXT_STATUS_ENABLED) 378 378 + continue; 407 379 408 408 - reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 409 409 - reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 380 380 + state_reg_count = ext->get_state_reg_count(vcpu); 381 381 + if (!uindices) 382 382 + goto skip_put_user; 410 383 411 411 - switch (reg_subtype) { 412 412 - case KVM_REG_RISCV_SBI_STA: 413 413 - return kvm_riscv_vcpu_set_reg_sbi_sta(vcpu, reg_num, reg_val); 414 414 - default: 415 415 - return -EINVAL; 384 384 + for (j = 0; j < state_reg_count; j++) { 385 385 + if (ext->get_state_reg_id) { 386 386 + rc = ext->get_state_reg_id(vcpu, j, &reg); 387 387 + if (rc) 388 388 + return rc; 389 389 + } else { 390 390 + reg = KVM_REG_RISCV | 391 391 + (IS_ENABLED(CONFIG_32BIT) ? 392 392 + KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64) | 393 393 + KVM_REG_RISCV_SBI_STATE | 394 394 + ext->state_reg_subtype | j; 395 395 + } 396 396 + 397 397 + if (put_user(reg, uindices)) 398 398 + return -EFAULT; 399 399 + uindices++; 400 400 + } 401 401 + 402 402 + skip_put_user: 403 403 + count += state_reg_count; 416 404 } 417 405 418 418 - return 0; 406 406 + return count; 419 407 } 420 408 421 421 - int kvm_riscv_vcpu_get_reg_sbi(struct kvm_vcpu *vcpu, 422 422 - const struct kvm_one_reg *reg) 409 409 + static const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext_withstate(struct kvm_vcpu *vcpu, 410 410 + unsigned long subtype) 411 411 + { 412 412 + struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; 413 413 + const struct kvm_riscv_sbi_extension_entry *entry; 414 414 + const struct kvm_vcpu_sbi_extension *ext; 415 415 + int i; 416 416 + 417 417 + for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { 418 418 + entry = &sbi_ext[i]; 419 419 + ext = entry->ext_ptr; 420 420 + 421 421 + if (ext->get_state_reg_count && 422 422 + ext->state_reg_subtype == subtype && 423 423 + scontext->ext_status[entry->ext_idx] == KVM_RISCV_SBI_EXT_STATUS_ENABLED) 424 424 + return ext; 425 425 + } 426 426 + 427 427 + return NULL; 428 428 + } 429 429 + 430 430 + int kvm_riscv_vcpu_set_reg_sbi(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 423 431 { 424 432 unsigned long __user *uaddr = 425 433 (unsigned long __user *)(unsigned long)reg->addr; 426 434 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 427 435 KVM_REG_SIZE_MASK | 428 436 KVM_REG_RISCV_SBI_STATE); 429 429 - unsigned long reg_subtype, reg_val; 430 430 - int ret; 437 437 + const struct kvm_vcpu_sbi_extension *ext; 438 438 + unsigned long reg_subtype; 439 439 + void *reg_val; 440 440 + u64 data64; 441 441 + u32 data32; 442 442 + u16 data16; 443 443 + u8 data8; 431 444 432 432 - if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 433 433 - return -EINVAL; 434 434 - 435 435 - reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 436 436 - reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 437 437 - 438 438 - switch (reg_subtype) { 439 439 - case KVM_REG_RISCV_SBI_STA: 440 440 - ret = kvm_riscv_vcpu_get_reg_sbi_sta(vcpu, reg_num, &reg_val); 445 445 + switch (KVM_REG_SIZE(reg->id)) { 446 446 + case 1: 447 447 + reg_val = &data8; 448 448 + break; 449 449 + case 2: 450 450 + reg_val = &data16; 451 451 + break; 452 452 + case 4: 453 453 + reg_val = &data32; 454 454 + break; 455 455 + case 8: 456 456 + reg_val = &data64; 441 457 break; 442 458 default: 443 459 return -EINVAL; 444 460 } 445 461 462 462 + if (copy_from_user(reg_val, uaddr, KVM_REG_SIZE(reg->id))) 463 463 + return -EFAULT; 464 464 + 465 465 + reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 466 466 + reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 467 467 + 468 468 + ext = kvm_vcpu_sbi_find_ext_withstate(vcpu, reg_subtype); 469 469 + if (!ext || !ext->set_state_reg) 470 470 + return -EINVAL; 471 471 + 472 472 + return ext->set_state_reg(vcpu, reg_num, KVM_REG_SIZE(reg->id), reg_val); 473 473 + } 474 474 + 475 475 + int kvm_riscv_vcpu_get_reg_sbi(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 476 476 + { 477 477 + unsigned long __user *uaddr = 478 478 + (unsigned long __user *)(unsigned long)reg->addr; 479 479 + unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 480 480 + KVM_REG_SIZE_MASK | 481 481 + KVM_REG_RISCV_SBI_STATE); 482 482 + const struct kvm_vcpu_sbi_extension *ext; 483 483 + unsigned long reg_subtype; 484 484 + void *reg_val; 485 485 + u64 data64; 486 486 + u32 data32; 487 487 + u16 data16; 488 488 + u8 data8; 489 489 + int ret; 490 490 + 491 491 + switch (KVM_REG_SIZE(reg->id)) { 492 492 + case 1: 493 493 + reg_val = &data8; 494 494 + break; 495 495 + case 2: 496 496 + reg_val = &data16; 497 497 + break; 498 498 + case 4: 499 499 + reg_val = &data32; 500 500 + break; 501 501 + case 8: 502 502 + reg_val = &data64; 503 503 + break; 504 504 + default: 505 505 + return -EINVAL; 506 506 + } 507 507 + 508 508 + reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 509 509 + reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 510 510 + 511 511 + ext = kvm_vcpu_sbi_find_ext_withstate(vcpu, reg_subtype); 512 512 + if (!ext || !ext->get_state_reg) 513 513 + return -EINVAL; 514 514 + 515 515 + ret = ext->get_state_reg(vcpu, reg_num, KVM_REG_SIZE(reg->id), reg_val); 446 516 if (ret) 447 517 return ret; 448 518 449 449 - if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id))) 519 519 + if (copy_to_user(uaddr, reg_val, KVM_REG_SIZE(reg->id))) 450 520 return -EFAULT; 451 521 452 522 return 0;

+544

arch/riscv/kvm/vcpu_sbi_fwft.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* 3 3 + * Copyright (c) 2025 Rivos Inc. 4 4 + * 5 5 + * Authors: 6 6 + * Clément Léger <cleger@rivosinc.com> 7 7 + */ 8 8 + 9 9 + #include <linux/errno.h> 10 10 + #include <linux/err.h> 11 11 + #include <linux/kvm_host.h> 12 12 + #include <asm/cpufeature.h> 13 13 + #include <asm/sbi.h> 14 14 + #include <asm/kvm_vcpu_sbi.h> 15 15 + #include <asm/kvm_vcpu_sbi_fwft.h> 16 16 + 17 17 + #define MIS_DELEG (BIT_ULL(EXC_LOAD_MISALIGNED) | BIT_ULL(EXC_STORE_MISALIGNED)) 18 18 + 19 19 + struct kvm_sbi_fwft_feature { 20 20 + /** 21 21 + * @id: Feature ID 22 22 + */ 23 23 + enum sbi_fwft_feature_t id; 24 24 + 25 25 + /** 26 26 + * @first_reg_num: ONE_REG index of the first ONE_REG register 27 27 + */ 28 28 + unsigned long first_reg_num; 29 29 + 30 30 + /** 31 31 + * @supported: Check if the feature is supported on the vcpu 32 32 + * 33 33 + * This callback is optional, if not provided the feature is assumed to 34 34 + * be supported 35 35 + */ 36 36 + bool (*supported)(struct kvm_vcpu *vcpu); 37 37 + 38 38 + /** 39 39 + * @reset: Reset the feature value irrespective whether feature is supported or not 40 40 + * 41 41 + * This callback is mandatory 42 42 + */ 43 43 + void (*reset)(struct kvm_vcpu *vcpu); 44 44 + 45 45 + /** 46 46 + * @set: Set the feature value 47 47 + * 48 48 + * Return SBI_SUCCESS on success or an SBI error (SBI_ERR_*) 49 49 + * 50 50 + * This callback is mandatory 51 51 + */ 52 52 + long (*set)(struct kvm_vcpu *vcpu, struct kvm_sbi_fwft_config *conf, 53 53 + bool one_reg_access, unsigned long value); 54 54 + 55 55 + /** 56 56 + * @get: Get the feature current value 57 57 + * 58 58 + * Return SBI_SUCCESS on success or an SBI error (SBI_ERR_*) 59 59 + * 60 60 + * This callback is mandatory 61 61 + */ 62 62 + long (*get)(struct kvm_vcpu *vcpu, struct kvm_sbi_fwft_config *conf, 63 63 + bool one_reg_access, unsigned long *value); 64 64 + }; 65 65 + 66 66 + static const enum sbi_fwft_feature_t kvm_fwft_defined_features[] = { 67 67 + SBI_FWFT_MISALIGNED_EXC_DELEG, 68 68 + SBI_FWFT_LANDING_PAD, 69 69 + SBI_FWFT_SHADOW_STACK, 70 70 + SBI_FWFT_DOUBLE_TRAP, 71 71 + SBI_FWFT_PTE_AD_HW_UPDATING, 72 72 + SBI_FWFT_POINTER_MASKING_PMLEN, 73 73 + }; 74 74 + 75 75 + static bool kvm_fwft_is_defined_feature(enum sbi_fwft_feature_t feature) 76 76 + { 77 77 + int i; 78 78 + 79 79 + for (i = 0; i < ARRAY_SIZE(kvm_fwft_defined_features); i++) { 80 80 + if (kvm_fwft_defined_features[i] == feature) 81 81 + return true; 82 82 + } 83 83 + 84 84 + return false; 85 85 + } 86 86 + 87 87 + static bool kvm_sbi_fwft_misaligned_delegation_supported(struct kvm_vcpu *vcpu) 88 88 + { 89 89 + return misaligned_traps_can_delegate(); 90 90 + } 91 91 + 92 92 + static void kvm_sbi_fwft_reset_misaligned_delegation(struct kvm_vcpu *vcpu) 93 93 + { 94 94 + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; 95 95 + 96 96 + cfg->hedeleg &= ~MIS_DELEG; 97 97 + } 98 98 + 99 99 + static long kvm_sbi_fwft_set_misaligned_delegation(struct kvm_vcpu *vcpu, 100 100 + struct kvm_sbi_fwft_config *conf, 101 101 + bool one_reg_access, unsigned long value) 102 102 + { 103 103 + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; 104 104 + 105 105 + if (value == 1) { 106 106 + cfg->hedeleg |= MIS_DELEG; 107 107 + if (!one_reg_access) 108 108 + csr_set(CSR_HEDELEG, MIS_DELEG); 109 109 + } else if (value == 0) { 110 110 + cfg->hedeleg &= ~MIS_DELEG; 111 111 + if (!one_reg_access) 112 112 + csr_clear(CSR_HEDELEG, MIS_DELEG); 113 113 + } else { 114 114 + return SBI_ERR_INVALID_PARAM; 115 115 + } 116 116 + 117 117 + return SBI_SUCCESS; 118 118 + } 119 119 + 120 120 + static long kvm_sbi_fwft_get_misaligned_delegation(struct kvm_vcpu *vcpu, 121 121 + struct kvm_sbi_fwft_config *conf, 122 122 + bool one_reg_access, unsigned long *value) 123 123 + { 124 124 + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; 125 125 + 126 126 + *value = (cfg->hedeleg & MIS_DELEG) == MIS_DELEG; 127 127 + return SBI_SUCCESS; 128 128 + } 129 129 + 130 130 + #ifndef CONFIG_32BIT 131 131 + 132 132 + static bool try_to_set_pmm(unsigned long value) 133 133 + { 134 134 + csr_set(CSR_HENVCFG, value); 135 135 + return (csr_read_clear(CSR_HENVCFG, ENVCFG_PMM) & ENVCFG_PMM) == value; 136 136 + } 137 137 + 138 138 + static bool kvm_sbi_fwft_pointer_masking_pmlen_supported(struct kvm_vcpu *vcpu) 139 139 + { 140 140 + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); 141 141 + 142 142 + if (!riscv_isa_extension_available(vcpu->arch.isa, SMNPM)) 143 143 + return false; 144 144 + 145 145 + fwft->have_vs_pmlen_7 = try_to_set_pmm(ENVCFG_PMM_PMLEN_7); 146 146 + fwft->have_vs_pmlen_16 = try_to_set_pmm(ENVCFG_PMM_PMLEN_16); 147 147 + 148 148 + return fwft->have_vs_pmlen_7 || fwft->have_vs_pmlen_16; 149 149 + } 150 150 + 151 151 + static void kvm_sbi_fwft_reset_pointer_masking_pmlen(struct kvm_vcpu *vcpu) 152 152 + { 153 153 + vcpu->arch.cfg.henvcfg &= ~ENVCFG_PMM; 154 154 + } 155 155 + 156 156 + static long kvm_sbi_fwft_set_pointer_masking_pmlen(struct kvm_vcpu *vcpu, 157 157 + struct kvm_sbi_fwft_config *conf, 158 158 + bool one_reg_access, unsigned long value) 159 159 + { 160 160 + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); 161 161 + unsigned long pmm; 162 162 + 163 163 + switch (value) { 164 164 + case 0: 165 165 + pmm = ENVCFG_PMM_PMLEN_0; 166 166 + break; 167 167 + case 7: 168 168 + if (!fwft->have_vs_pmlen_7) 169 169 + return SBI_ERR_INVALID_PARAM; 170 170 + pmm = ENVCFG_PMM_PMLEN_7; 171 171 + break; 172 172 + case 16: 173 173 + if (!fwft->have_vs_pmlen_16) 174 174 + return SBI_ERR_INVALID_PARAM; 175 175 + pmm = ENVCFG_PMM_PMLEN_16; 176 176 + break; 177 177 + default: 178 178 + return SBI_ERR_INVALID_PARAM; 179 179 + } 180 180 + 181 181 + vcpu->arch.cfg.henvcfg &= ~ENVCFG_PMM; 182 182 + vcpu->arch.cfg.henvcfg |= pmm; 183 183 + 184 184 + /* 185 185 + * Instead of waiting for vcpu_load/put() to update HENVCFG CSR, 186 186 + * update here so that VCPU see's pointer masking mode change 187 187 + * immediately. 188 188 + */ 189 189 + if (!one_reg_access) 190 190 + csr_write(CSR_HENVCFG, vcpu->arch.cfg.henvcfg); 191 191 + 192 192 + return SBI_SUCCESS; 193 193 + } 194 194 + 195 195 + static long kvm_sbi_fwft_get_pointer_masking_pmlen(struct kvm_vcpu *vcpu, 196 196 + struct kvm_sbi_fwft_config *conf, 197 197 + bool one_reg_access, unsigned long *value) 198 198 + { 199 199 + switch (vcpu->arch.cfg.henvcfg & ENVCFG_PMM) { 200 200 + case ENVCFG_PMM_PMLEN_0: 201 201 + *value = 0; 202 202 + break; 203 203 + case ENVCFG_PMM_PMLEN_7: 204 204 + *value = 7; 205 205 + break; 206 206 + case ENVCFG_PMM_PMLEN_16: 207 207 + *value = 16; 208 208 + break; 209 209 + default: 210 210 + return SBI_ERR_FAILURE; 211 211 + } 212 212 + 213 213 + return SBI_SUCCESS; 214 214 + } 215 215 + 216 216 + #endif 217 217 + 218 218 + static const struct kvm_sbi_fwft_feature features[] = { 219 219 + { 220 220 + .id = SBI_FWFT_MISALIGNED_EXC_DELEG, 221 221 + .first_reg_num = offsetof(struct kvm_riscv_sbi_fwft, misaligned_deleg.enable) / 222 222 + sizeof(unsigned long), 223 223 + .supported = kvm_sbi_fwft_misaligned_delegation_supported, 224 224 + .reset = kvm_sbi_fwft_reset_misaligned_delegation, 225 225 + .set = kvm_sbi_fwft_set_misaligned_delegation, 226 226 + .get = kvm_sbi_fwft_get_misaligned_delegation, 227 227 + }, 228 228 + #ifndef CONFIG_32BIT 229 229 + { 230 230 + .id = SBI_FWFT_POINTER_MASKING_PMLEN, 231 231 + .first_reg_num = offsetof(struct kvm_riscv_sbi_fwft, pointer_masking.enable) / 232 232 + sizeof(unsigned long), 233 233 + .supported = kvm_sbi_fwft_pointer_masking_pmlen_supported, 234 234 + .reset = kvm_sbi_fwft_reset_pointer_masking_pmlen, 235 235 + .set = kvm_sbi_fwft_set_pointer_masking_pmlen, 236 236 + .get = kvm_sbi_fwft_get_pointer_masking_pmlen, 237 237 + }, 238 238 + #endif 239 239 + }; 240 240 + 241 241 + static const struct kvm_sbi_fwft_feature *kvm_sbi_fwft_regnum_to_feature(unsigned long reg_num) 242 242 + { 243 243 + const struct kvm_sbi_fwft_feature *feature; 244 244 + int i; 245 245 + 246 246 + for (i = 0; i < ARRAY_SIZE(features); i++) { 247 247 + feature = &features[i]; 248 248 + if (feature->first_reg_num <= reg_num && reg_num < (feature->first_reg_num + 3)) 249 249 + return feature; 250 250 + } 251 251 + 252 252 + return NULL; 253 253 + } 254 254 + 255 255 + static struct kvm_sbi_fwft_config * 256 256 + kvm_sbi_fwft_get_config(struct kvm_vcpu *vcpu, enum sbi_fwft_feature_t feature) 257 257 + { 258 258 + int i; 259 259 + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); 260 260 + 261 261 + for (i = 0; i < ARRAY_SIZE(features); i++) { 262 262 + if (fwft->configs[i].feature->id == feature) 263 263 + return &fwft->configs[i]; 264 264 + } 265 265 + 266 266 + return NULL; 267 267 + } 268 268 + 269 269 + static int kvm_fwft_get_feature(struct kvm_vcpu *vcpu, u32 feature, 270 270 + struct kvm_sbi_fwft_config **conf) 271 271 + { 272 272 + struct kvm_sbi_fwft_config *tconf; 273 273 + 274 274 + tconf = kvm_sbi_fwft_get_config(vcpu, feature); 275 275 + if (!tconf) { 276 276 + if (kvm_fwft_is_defined_feature(feature)) 277 277 + return SBI_ERR_NOT_SUPPORTED; 278 278 + 279 279 + return SBI_ERR_DENIED; 280 280 + } 281 281 + 282 282 + if (!tconf->supported || !tconf->enabled) 283 283 + return SBI_ERR_NOT_SUPPORTED; 284 284 + 285 285 + *conf = tconf; 286 286 + 287 287 + return SBI_SUCCESS; 288 288 + } 289 289 + 290 290 + static int kvm_sbi_fwft_set(struct kvm_vcpu *vcpu, u32 feature, 291 291 + unsigned long value, unsigned long flags) 292 292 + { 293 293 + int ret; 294 294 + struct kvm_sbi_fwft_config *conf; 295 295 + 296 296 + ret = kvm_fwft_get_feature(vcpu, feature, &conf); 297 297 + if (ret) 298 298 + return ret; 299 299 + 300 300 + if ((flags & ~SBI_FWFT_SET_FLAG_LOCK) != 0) 301 301 + return SBI_ERR_INVALID_PARAM; 302 302 + 303 303 + if (conf->flags & SBI_FWFT_SET_FLAG_LOCK) 304 304 + return SBI_ERR_DENIED_LOCKED; 305 305 + 306 306 + conf->flags = flags; 307 307 + 308 308 + return conf->feature->set(vcpu, conf, false, value); 309 309 + } 310 310 + 311 311 + static int kvm_sbi_fwft_get(struct kvm_vcpu *vcpu, unsigned long feature, 312 312 + unsigned long *value) 313 313 + { 314 314 + int ret; 315 315 + struct kvm_sbi_fwft_config *conf; 316 316 + 317 317 + ret = kvm_fwft_get_feature(vcpu, feature, &conf); 318 318 + if (ret) 319 319 + return ret; 320 320 + 321 321 + return conf->feature->get(vcpu, conf, false, value); 322 322 + } 323 323 + 324 324 + static int kvm_sbi_ext_fwft_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, 325 325 + struct kvm_vcpu_sbi_return *retdata) 326 326 + { 327 327 + int ret; 328 328 + struct kvm_cpu_context *cp = &vcpu->arch.guest_context; 329 329 + unsigned long funcid = cp->a6; 330 330 + 331 331 + switch (funcid) { 332 332 + case SBI_EXT_FWFT_SET: 333 333 + ret = kvm_sbi_fwft_set(vcpu, cp->a0, cp->a1, cp->a2); 334 334 + break; 335 335 + case SBI_EXT_FWFT_GET: 336 336 + ret = kvm_sbi_fwft_get(vcpu, cp->a0, &retdata->out_val); 337 337 + break; 338 338 + default: 339 339 + ret = SBI_ERR_NOT_SUPPORTED; 340 340 + break; 341 341 + } 342 342 + 343 343 + retdata->err_val = ret; 344 344 + 345 345 + return 0; 346 346 + } 347 347 + 348 348 + static int kvm_sbi_ext_fwft_init(struct kvm_vcpu *vcpu) 349 349 + { 350 350 + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); 351 351 + const struct kvm_sbi_fwft_feature *feature; 352 352 + struct kvm_sbi_fwft_config *conf; 353 353 + int i; 354 354 + 355 355 + fwft->configs = kcalloc(ARRAY_SIZE(features), sizeof(struct kvm_sbi_fwft_config), 356 356 + GFP_KERNEL); 357 357 + if (!fwft->configs) 358 358 + return -ENOMEM; 359 359 + 360 360 + for (i = 0; i < ARRAY_SIZE(features); i++) { 361 361 + feature = &features[i]; 362 362 + conf = &fwft->configs[i]; 363 363 + if (feature->supported) 364 364 + conf->supported = feature->supported(vcpu); 365 365 + else 366 366 + conf->supported = true; 367 367 + 368 368 + conf->enabled = conf->supported; 369 369 + conf->feature = feature; 370 370 + } 371 371 + 372 372 + return 0; 373 373 + } 374 374 + 375 375 + static void kvm_sbi_ext_fwft_deinit(struct kvm_vcpu *vcpu) 376 376 + { 377 377 + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); 378 378 + 379 379 + kfree(fwft->configs); 380 380 + } 381 381 + 382 382 + static void kvm_sbi_ext_fwft_reset(struct kvm_vcpu *vcpu) 383 383 + { 384 384 + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); 385 385 + const struct kvm_sbi_fwft_feature *feature; 386 386 + int i; 387 387 + 388 388 + for (i = 0; i < ARRAY_SIZE(features); i++) { 389 389 + fwft->configs[i].flags = 0; 390 390 + feature = &features[i]; 391 391 + if (feature->reset) 392 392 + feature->reset(vcpu); 393 393 + } 394 394 + } 395 395 + 396 396 + static unsigned long kvm_sbi_ext_fwft_get_reg_count(struct kvm_vcpu *vcpu) 397 397 + { 398 398 + unsigned long max_reg_count = sizeof(struct kvm_riscv_sbi_fwft) / sizeof(unsigned long); 399 399 + const struct kvm_sbi_fwft_feature *feature; 400 400 + struct kvm_sbi_fwft_config *conf; 401 401 + unsigned long reg, ret = 0; 402 402 + 403 403 + for (reg = 0; reg < max_reg_count; reg++) { 404 404 + feature = kvm_sbi_fwft_regnum_to_feature(reg); 405 405 + if (!feature) 406 406 + continue; 407 407 + 408 408 + conf = kvm_sbi_fwft_get_config(vcpu, feature->id); 409 409 + if (!conf || !conf->supported) 410 410 + continue; 411 411 + 412 412 + ret++; 413 413 + } 414 414 + 415 415 + return ret; 416 416 + } 417 417 + 418 418 + static int kvm_sbi_ext_fwft_get_reg_id(struct kvm_vcpu *vcpu, int index, u64 *reg_id) 419 419 + { 420 420 + int reg, max_reg_count = sizeof(struct kvm_riscv_sbi_fwft) / sizeof(unsigned long); 421 421 + const struct kvm_sbi_fwft_feature *feature; 422 422 + struct kvm_sbi_fwft_config *conf; 423 423 + int idx = 0; 424 424 + 425 425 + for (reg = 0; reg < max_reg_count; reg++) { 426 426 + feature = kvm_sbi_fwft_regnum_to_feature(reg); 427 427 + if (!feature) 428 428 + continue; 429 429 + 430 430 + conf = kvm_sbi_fwft_get_config(vcpu, feature->id); 431 431 + if (!conf || !conf->supported) 432 432 + continue; 433 433 + 434 434 + if (index == idx) { 435 435 + *reg_id = KVM_REG_RISCV | 436 436 + (IS_ENABLED(CONFIG_32BIT) ? 437 437 + KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64) | 438 438 + KVM_REG_RISCV_SBI_STATE | 439 439 + KVM_REG_RISCV_SBI_FWFT | reg; 440 440 + return 0; 441 441 + } 442 442 + 443 443 + idx++; 444 444 + } 445 445 + 446 446 + return -ENOENT; 447 447 + } 448 448 + 449 449 + static int kvm_sbi_ext_fwft_get_reg(struct kvm_vcpu *vcpu, unsigned long reg_num, 450 450 + unsigned long reg_size, void *reg_val) 451 451 + { 452 452 + const struct kvm_sbi_fwft_feature *feature; 453 453 + struct kvm_sbi_fwft_config *conf; 454 454 + unsigned long *value; 455 455 + int ret = 0; 456 456 + 457 457 + if (reg_size != sizeof(unsigned long)) 458 458 + return -EINVAL; 459 459 + value = reg_val; 460 460 + 461 461 + feature = kvm_sbi_fwft_regnum_to_feature(reg_num); 462 462 + if (!feature) 463 463 + return -ENOENT; 464 464 + 465 465 + conf = kvm_sbi_fwft_get_config(vcpu, feature->id); 466 466 + if (!conf || !conf->supported) 467 467 + return -ENOENT; 468 468 + 469 469 + switch (reg_num - feature->first_reg_num) { 470 470 + case 0: 471 471 + *value = conf->enabled; 472 472 + break; 473 473 + case 1: 474 474 + *value = conf->flags; 475 475 + break; 476 476 + case 2: 477 477 + ret = conf->feature->get(vcpu, conf, true, value); 478 478 + break; 479 479 + default: 480 480 + return -ENOENT; 481 481 + } 482 482 + 483 483 + return sbi_err_map_linux_errno(ret); 484 484 + } 485 485 + 486 486 + static int kvm_sbi_ext_fwft_set_reg(struct kvm_vcpu *vcpu, unsigned long reg_num, 487 487 + unsigned long reg_size, const void *reg_val) 488 488 + { 489 489 + const struct kvm_sbi_fwft_feature *feature; 490 490 + struct kvm_sbi_fwft_config *conf; 491 491 + unsigned long value; 492 492 + int ret = 0; 493 493 + 494 494 + if (reg_size != sizeof(unsigned long)) 495 495 + return -EINVAL; 496 496 + value = *(const unsigned long *)reg_val; 497 497 + 498 498 + feature = kvm_sbi_fwft_regnum_to_feature(reg_num); 499 499 + if (!feature) 500 500 + return -ENOENT; 501 501 + 502 502 + conf = kvm_sbi_fwft_get_config(vcpu, feature->id); 503 503 + if (!conf || !conf->supported) 504 504 + return -ENOENT; 505 505 + 506 506 + switch (reg_num - feature->first_reg_num) { 507 507 + case 0: 508 508 + switch (value) { 509 509 + case 0: 510 510 + conf->enabled = false; 511 511 + break; 512 512 + case 1: 513 513 + conf->enabled = true; 514 514 + break; 515 515 + default: 516 516 + return -EINVAL; 517 517 + } 518 518 + break; 519 519 + case 1: 520 520 + conf->flags = value & SBI_FWFT_SET_FLAG_LOCK; 521 521 + break; 522 522 + case 2: 523 523 + ret = conf->feature->set(vcpu, conf, true, value); 524 524 + break; 525 525 + default: 526 526 + return -ENOENT; 527 527 + } 528 528 + 529 529 + return sbi_err_map_linux_errno(ret); 530 530 + } 531 531 + 532 532 + const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_fwft = { 533 533 + .extid_start = SBI_EXT_FWFT, 534 534 + .extid_end = SBI_EXT_FWFT, 535 535 + .handler = kvm_sbi_ext_fwft_handler, 536 536 + .init = kvm_sbi_ext_fwft_init, 537 537 + .deinit = kvm_sbi_ext_fwft_deinit, 538 538 + .reset = kvm_sbi_ext_fwft_reset, 539 539 + .state_reg_subtype = KVM_REG_RISCV_SBI_FWFT, 540 540 + .get_state_reg_count = kvm_sbi_ext_fwft_get_reg_count, 541 541 + .get_state_reg_id = kvm_sbi_ext_fwft_get_reg_id, 542 542 + .get_state_reg = kvm_sbi_ext_fwft_get_reg, 543 543 + .set_state_reg = kvm_sbi_ext_fwft_set_reg, 544 544 + };

+3

arch/riscv/kvm/vcpu_sbi_pmu.c

reviewed

··· 73 73 case SBI_EXT_PMU_SNAPSHOT_SET_SHMEM: 74 74 ret = kvm_riscv_vcpu_pmu_snapshot_set_shmem(vcpu, cp->a0, cp->a1, cp->a2, retdata); 75 75 break; 76 76 + case SBI_EXT_PMU_EVENT_GET_INFO: 77 77 + ret = kvm_riscv_vcpu_pmu_event_info(vcpu, cp->a0, cp->a1, cp->a2, cp->a3, retdata); 78 78 + break; 76 79 default: 77 80 retdata->err_val = SBI_ERR_NOT_SUPPORTED; 78 81 }

+44 -30

arch/riscv/kvm/vcpu_sbi_sta.c

reviewed

··· 85 85 unsigned long shmem_phys_hi = cp->a1; 86 86 u32 flags = cp->a2; 87 87 struct sbi_sta_struct zero_sta = {0}; 88 88 - unsigned long hva; 89 89 - bool writable; 90 88 gpa_t shmem; 91 89 int ret; 92 90 ··· 109 111 return SBI_ERR_INVALID_ADDRESS; 110 112 } 111 113 112 112 - hva = kvm_vcpu_gfn_to_hva_prot(vcpu, shmem >> PAGE_SHIFT, &writable); 113 113 - if (kvm_is_error_hva(hva) || !writable) 114 114 - return SBI_ERR_INVALID_ADDRESS; 115 115 - 114 114 + /* No need to check writable slot explicitly as kvm_vcpu_write_guest does it internally */ 116 115 ret = kvm_vcpu_write_guest(vcpu, shmem, &zero_sta, sizeof(zero_sta)); 117 116 if (ret) 118 118 - return SBI_ERR_FAILURE; 117 117 + return SBI_ERR_INVALID_ADDRESS; 119 118 120 119 vcpu->arch.sta.shmem = shmem; 121 120 vcpu->arch.sta.last_steal = current->sched_info.run_delay; ··· 146 151 return !!sched_info_on(); 147 152 } 148 153 149 149 - const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta = { 150 150 - .extid_start = SBI_EXT_STA, 151 151 - .extid_end = SBI_EXT_STA, 152 152 - .handler = kvm_sbi_ext_sta_handler, 153 153 - .probe = kvm_sbi_ext_sta_probe, 154 154 - .reset = kvm_riscv_vcpu_sbi_sta_reset, 155 155 - }; 156 156 - 157 157 - int kvm_riscv_vcpu_get_reg_sbi_sta(struct kvm_vcpu *vcpu, 158 158 - unsigned long reg_num, 159 159 - unsigned long *reg_val) 154 154 + static unsigned long kvm_sbi_ext_sta_get_state_reg_count(struct kvm_vcpu *vcpu) 160 155 { 156 156 + return sizeof(struct kvm_riscv_sbi_sta) / sizeof(unsigned long); 157 157 + } 158 158 + 159 159 + static int kvm_sbi_ext_sta_get_reg(struct kvm_vcpu *vcpu, unsigned long reg_num, 160 160 + unsigned long reg_size, void *reg_val) 161 161 + { 162 162 + unsigned long *value; 163 163 + 164 164 + if (reg_size != sizeof(unsigned long)) 165 165 + return -EINVAL; 166 166 + value = reg_val; 167 167 + 161 168 switch (reg_num) { 162 169 case KVM_REG_RISCV_SBI_STA_REG(shmem_lo): 163 163 - *reg_val = (unsigned long)vcpu->arch.sta.shmem; 170 170 + *value = (unsigned long)vcpu->arch.sta.shmem; 164 171 break; 165 172 case KVM_REG_RISCV_SBI_STA_REG(shmem_hi): 166 173 if (IS_ENABLED(CONFIG_32BIT)) 167 167 - *reg_val = upper_32_bits(vcpu->arch.sta.shmem); 174 174 + *value = upper_32_bits(vcpu->arch.sta.shmem); 168 175 else 169 169 - *reg_val = 0; 176 176 + *value = 0; 170 177 break; 171 178 default: 172 172 - return -EINVAL; 179 179 + return -ENOENT; 173 180 } 174 181 175 182 return 0; 176 183 } 177 184 178 178 - int kvm_riscv_vcpu_set_reg_sbi_sta(struct kvm_vcpu *vcpu, 179 179 - unsigned long reg_num, 180 180 - unsigned long reg_val) 185 185 + static int kvm_sbi_ext_sta_set_reg(struct kvm_vcpu *vcpu, unsigned long reg_num, 186 186 + unsigned long reg_size, const void *reg_val) 181 187 { 188 188 + unsigned long value; 189 189 + 190 190 + if (reg_size != sizeof(unsigned long)) 191 191 + return -EINVAL; 192 192 + value = *(const unsigned long *)reg_val; 193 193 + 182 194 switch (reg_num) { 183 195 case KVM_REG_RISCV_SBI_STA_REG(shmem_lo): 184 196 if (IS_ENABLED(CONFIG_32BIT)) { 185 197 gpa_t hi = upper_32_bits(vcpu->arch.sta.shmem); 186 198 187 187 - vcpu->arch.sta.shmem = reg_val; 199 199 + vcpu->arch.sta.shmem = value; 188 200 vcpu->arch.sta.shmem |= hi << 32; 189 201 } else { 190 190 - vcpu->arch.sta.shmem = reg_val; 202 202 + vcpu->arch.sta.shmem = value; 191 203 } 192 204 break; 193 205 case KVM_REG_RISCV_SBI_STA_REG(shmem_hi): 194 206 if (IS_ENABLED(CONFIG_32BIT)) { 195 207 gpa_t lo = lower_32_bits(vcpu->arch.sta.shmem); 196 208 197 197 - vcpu->arch.sta.shmem = ((gpa_t)reg_val << 32); 209 209 + vcpu->arch.sta.shmem = ((gpa_t)value << 32); 198 210 vcpu->arch.sta.shmem |= lo; 199 199 - } else if (reg_val != 0) { 211 211 + } else if (value != 0) { 200 212 return -EINVAL; 201 213 } 202 214 break; 203 215 default: 204 204 - return -EINVAL; 216 216 + return -ENOENT; 205 217 } 206 218 207 219 return 0; 208 220 } 221 221 + 222 222 + const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta = { 223 223 + .extid_start = SBI_EXT_STA, 224 224 + .extid_end = SBI_EXT_STA, 225 225 + .handler = kvm_sbi_ext_sta_handler, 226 226 + .probe = kvm_sbi_ext_sta_probe, 227 227 + .reset = kvm_riscv_vcpu_sbi_sta_reset, 228 228 + .state_reg_subtype = KVM_REG_RISCV_SBI_STA, 229 229 + .get_state_reg_count = kvm_sbi_ext_sta_get_state_reg_count, 230 230 + .get_state_reg = kvm_sbi_ext_sta_get_reg, 231 231 + .set_state_reg = kvm_sbi_ext_sta_set_reg, 232 232 + };

+3 -5

arch/riscv/kvm/vmid.c

reviewed

··· 14 14 #include <linux/smp.h> 15 15 #include <linux/kvm_host.h> 16 16 #include <asm/csr.h> 17 17 + #include <asm/kvm_mmu.h> 17 18 #include <asm/kvm_tlb.h> 18 19 #include <asm/kvm_vmid.h> 19 20 ··· 25 24 26 25 void __init kvm_riscv_gstage_vmid_detect(void) 27 26 { 28 28 - unsigned long old; 29 29 - 30 27 /* Figure-out number of VMID bits in HW */ 31 31 - old = csr_read(CSR_HGATP); 32 32 - csr_write(CSR_HGATP, old | HGATP_VMID); 28 28 + csr_write(CSR_HGATP, (kvm_riscv_gstage_mode << HGATP_MODE_SHIFT) | HGATP_VMID); 33 29 vmid_bits = csr_read(CSR_HGATP); 34 30 vmid_bits = (vmid_bits & HGATP_VMID) >> HGATP_VMID_SHIFT; 35 31 vmid_bits = fls_long(vmid_bits); 36 36 - csr_write(CSR_HGATP, old); 32 32 + csr_write(CSR_HGATP, 0); 37 33 38 34 /* We polluted local TLB so flush all guest TLB */ 39 35 kvm_riscv_local_hfence_gvma_all();

+1 -1

arch/s390/include/asm/kvm_host.h

reviewed

··· 356 356 int counters[FIRQ_MAX_COUNT]; 357 357 struct kvm_s390_mchk_info mchk; 358 358 struct kvm_s390_ext_info srv_signal; 359 359 - int next_rr_cpu; 359 359 + int last_sleep_cpu; 360 360 struct mutex ais_lock; 361 361 u8 simm; 362 362 u8 nimm;

+22

arch/s390/include/asm/pgtable.h

reviewed

··· 2055 2055 return res; 2056 2056 } 2057 2057 2058 2058 + static inline pgste_t pgste_get_lock(pte_t *ptep) 2059 2059 + { 2060 2060 + unsigned long value = 0; 2061 2061 + #ifdef CONFIG_PGSTE 2062 2062 + unsigned long *ptr = (unsigned long *)(ptep + PTRS_PER_PTE); 2063 2063 + 2064 2064 + do { 2065 2065 + value = __atomic64_or_barrier(PGSTE_PCL_BIT, ptr); 2066 2066 + } while (value & PGSTE_PCL_BIT); 2067 2067 + value |= PGSTE_PCL_BIT; 2068 2068 + #endif 2069 2069 + return __pgste(value); 2070 2070 + } 2071 2071 + 2072 2072 + static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) 2073 2073 + { 2074 2074 + #ifdef CONFIG_PGSTE 2075 2075 + barrier(); 2076 2076 + WRITE_ONCE(*(unsigned long *)(ptep + PTRS_PER_PTE), pgste_val(pgste) & ~PGSTE_PCL_BIT); 2077 2077 + #endif 2078 2078 + } 2079 2079 + 2058 2080 #endif /* _S390_PAGE_H */

+9 -11

arch/s390/kvm/interrupt.c

reviewed

··· 1323 1323 VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime); 1324 1324 no_timer: 1325 1325 kvm_vcpu_srcu_read_unlock(vcpu); 1326 1326 + vcpu->kvm->arch.float_int.last_sleep_cpu = vcpu->vcpu_idx; 1326 1327 kvm_vcpu_halt(vcpu); 1327 1328 vcpu->valid_wakeup = false; 1328 1329 __unset_cpu_idle(vcpu); ··· 1950 1949 if (!online_vcpus) 1951 1950 return; 1952 1951 1953 1953 - /* find idle VCPUs first, then round robin */ 1954 1954 - sigcpu = find_first_bit(kvm->arch.idle_mask, online_vcpus); 1955 1955 - if (sigcpu == online_vcpus) { 1956 1956 - do { 1957 1957 - sigcpu = kvm->arch.float_int.next_rr_cpu++; 1958 1958 - kvm->arch.float_int.next_rr_cpu %= online_vcpus; 1959 1959 - /* avoid endless loops if all vcpus are stopped */ 1960 1960 - if (nr_tries++ >= online_vcpus) 1961 1961 - return; 1962 1962 - } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu))); 1952 1952 + for (sigcpu = kvm->arch.float_int.last_sleep_cpu; ; sigcpu++) { 1953 1953 + sigcpu %= online_vcpus; 1954 1954 + dst_vcpu = kvm_get_vcpu(kvm, sigcpu); 1955 1955 + if (!is_vcpu_stopped(dst_vcpu)) 1956 1956 + break; 1957 1957 + /* avoid endless loops if all vcpus are stopped */ 1958 1958 + if (nr_tries++ >= online_vcpus) 1959 1959 + return; 1963 1960 } 1964 1964 - dst_vcpu = kvm_get_vcpu(kvm, sigcpu); 1965 1961 1966 1962 /* make the VCPU drop out of the SIE, or wake it up if sleeping */ 1967 1963 switch (type) {

+11 -1

arch/s390/mm/gmap_helpers.c

reviewed

··· 15 15 #include <linux/pagewalk.h> 16 16 #include <linux/ksm.h> 17 17 #include <asm/gmap_helpers.h> 18 18 + #include <asm/pgtable.h> 18 19 19 20 /** 20 21 * ptep_zap_swap_entry() - discard a swap entry. ··· 48 47 { 49 48 struct vm_area_struct *vma; 50 49 spinlock_t *ptl; 50 50 + pgste_t pgste; 51 51 pte_t *ptep; 52 52 53 53 mmap_assert_locked(mm); ··· 62 60 ptep = get_locked_pte(mm, vmaddr, &ptl); 63 61 if (unlikely(!ptep)) 64 62 return; 65 65 - if (pte_swap(*ptep)) 63 63 + if (pte_swap(*ptep)) { 64 64 + preempt_disable(); 65 65 + pgste = pgste_get_lock(ptep); 66 66 + 66 67 ptep_zap_swap_entry(mm, pte_to_swp_entry(*ptep)); 68 68 + pte_clear(mm, vmaddr, ptep); 69 69 + 70 70 + pgste_set_unlock(ptep, pgste); 71 71 + preempt_enable(); 72 72 + } 67 73 pte_unmap_unlock(ptep, ptl); 68 74 } 69 75 EXPORT_SYMBOL_GPL(gmap_helper_zap_one_page);

+1 -22

arch/s390/mm/pgtable.c

reviewed

··· 24 24 #include <asm/tlbflush.h> 25 25 #include <asm/mmu_context.h> 26 26 #include <asm/page-states.h> 27 27 + #include <asm/pgtable.h> 27 28 #include <asm/machine.h> 28 29 29 30 pgprot_t pgprot_writecombine(pgprot_t prot) ··· 114 113 ptep_ipte_global(mm, addr, ptep, nodat); 115 114 atomic_dec(&mm->context.flush_count); 116 115 return old; 117 117 - } 118 118 - 119 119 - static inline pgste_t pgste_get_lock(pte_t *ptep) 120 120 - { 121 121 - unsigned long value = 0; 122 122 - #ifdef CONFIG_PGSTE 123 123 - unsigned long *ptr = (unsigned long *)(ptep + PTRS_PER_PTE); 124 124 - 125 125 - do { 126 126 - value = __atomic64_or_barrier(PGSTE_PCL_BIT, ptr); 127 127 - } while (value & PGSTE_PCL_BIT); 128 128 - value |= PGSTE_PCL_BIT; 129 129 - #endif 130 130 - return __pgste(value); 131 131 - } 132 132 - 133 133 - static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) 134 134 - { 135 135 - #ifdef CONFIG_PGSTE 136 136 - barrier(); 137 137 - WRITE_ONCE(*(unsigned long *)(ptep + PTRS_PER_PTE), pgste_val(pgste) & ~PGSTE_PCL_BIT); 138 138 - #endif 139 116 } 140 117 141 118 static inline pgste_t pgste_get(pte_t *ptep)

+1 -1

arch/x86/include/asm/kvm-x86-ops.h

reviewed

··· 145 145 KVM_X86_OP_OPTIONAL(get_untagged_addr) 146 146 KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) 147 147 KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) 148 148 - KVM_X86_OP_OPTIONAL_RET0(private_max_mapping_level) 148 148 + KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level) 149 149 KVM_X86_OP_OPTIONAL(gmem_invalidate) 150 150 151 151 #undef KVM_X86_OP

+2 -4

arch/x86/include/asm/kvm_host.h

reviewed

··· 1922 1922 void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); 1923 1923 int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); 1924 1924 void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end); 1925 1925 - int (*private_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn); 1925 1925 + int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); 1926 1926 }; 1927 1927 1928 1928 struct kvm_x86_nested_ops { ··· 2276 2276 int tdp_max_root_level, int tdp_huge_page_level); 2277 2277 2278 2278 2279 2279 - #ifdef CONFIG_KVM_PRIVATE_MEM 2279 2279 + #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES 2280 2280 #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem) 2281 2281 - #else 2282 2282 - #define kvm_arch_has_private_mem(kvm) false 2283 2281 #endif 2284 2282 2285 2283 #define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state)

-2

arch/x86/include/asm/kvm_para.h

reviewed

··· 124 124 unsigned int kvm_arch_para_features(void); 125 125 unsigned int kvm_arch_para_hints(void); 126 126 void kvm_async_pf_task_wait_schedule(u32 token); 127 127 - void kvm_async_pf_task_wake(u32 token); 128 127 u32 kvm_read_and_reset_apf_flags(void); 129 128 bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token); 130 129 ··· 147 148 148 149 #else /* CONFIG_KVM_GUEST */ 149 150 #define kvm_async_pf_task_wait_schedule(T) do {} while(0) 150 150 - #define kvm_async_pf_task_wake(T) do {} while(0) 151 151 152 152 static inline bool kvm_para_available(void) 153 153 {

+30 -14

arch/x86/kernel/kvm.c

reviewed

··· 190 190 } 191 191 } 192 192 193 193 - void kvm_async_pf_task_wake(u32 token) 193 193 + static void kvm_async_pf_task_wake(u32 token) 194 194 { 195 195 u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); 196 196 struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; ··· 241 241 /* A dummy token might be allocated and ultimately not used. */ 242 242 kfree(dummy); 243 243 } 244 244 - EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake); 245 244 246 245 noinstr u32 kvm_read_and_reset_apf_flags(void) 247 246 { ··· 932 933 933 934 static void __init kvm_init_platform(void) 934 935 { 936 936 + u64 tolud = PFN_PHYS(e820__end_of_low_ram_pfn()); 937 937 + /* 938 938 + * Note, hardware requires variable MTRR ranges to be power-of-2 sized 939 939 + * and naturally aligned. But when forcing guest MTRR state, Linux 940 940 + * doesn't program the forced ranges into hardware. Don't bother doing 941 941 + * the math to generate a technically-legal range. 942 942 + */ 943 943 + struct mtrr_var_range pci_hole = { 944 944 + .base_lo = tolud | X86_MEMTYPE_UC, 945 945 + .mask_lo = (u32)(~(SZ_4G - tolud - 1)) | MTRR_PHYSMASK_V, 946 946 + .mask_hi = (BIT_ULL(boot_cpu_data.x86_phys_bits) - 1) >> 32, 947 947 + }; 948 948 + 935 949 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) && 936 950 kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL)) { 937 951 unsigned long nr_pages; ··· 994 982 kvmclock_init(); 995 983 x86_platform.apic_post_init = kvm_apic_init; 996 984 997 997 - /* Set WB as the default cache mode for SEV-SNP and TDX */ 998 998 - guest_force_mtrr_state(NULL, 0, MTRR_TYPE_WRBACK); 985 985 + /* 986 986 + * Set WB as the default cache mode for SEV-SNP and TDX, with a single 987 987 + * UC range for the legacy PCI hole, e.g. so that devices that expect 988 988 + * to get UC/WC mappings don't get surprised with WB. 989 989 + */ 990 990 + guest_force_mtrr_state(&pci_hole, 1, MTRR_TYPE_WRBACK); 999 991 } 1000 992 1001 993 #if defined(CONFIG_AMD_MEM_ENCRYPT) ··· 1089 1073 void __init kvm_spinlock_init(void) 1090 1074 { 1091 1075 /* 1092 1092 - * In case host doesn't support KVM_FEATURE_PV_UNHALT there is still an 1093 1093 - * advantage of keeping virt_spin_lock_key enabled: virt_spin_lock() is 1094 1094 - * preferred over native qspinlock when vCPU is preempted. 1095 1095 - */ 1096 1096 - if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT)) { 1097 1097 - pr_info("PV spinlocks disabled, no host support\n"); 1098 1098 - return; 1099 1099 - } 1100 1100 - 1101 1101 - /* 1102 1076 * Disable PV spinlocks and use native qspinlock when dedicated pCPUs 1103 1077 * are available. 1104 1078 */ ··· 1105 1099 if (nopvspin) { 1106 1100 pr_info("PV spinlocks disabled, forced by \"nopvspin\" parameter\n"); 1107 1101 goto out; 1102 1102 + } 1103 1103 + 1104 1104 + /* 1105 1105 + * In case host doesn't support KVM_FEATURE_PV_UNHALT there is still an 1106 1106 + * advantage of keeping virt_spin_lock_key enabled: virt_spin_lock() is 1107 1107 + * preferred over native qspinlock when vCPU is preempted. 1108 1108 + */ 1109 1109 + if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT)) { 1110 1110 + pr_info("PV spinlocks disabled, no host support\n"); 1111 1111 + return; 1108 1112 } 1109 1113 1110 1114 pr_info("PV spinlocks enabled\n");

+14 -12

arch/x86/kvm/Kconfig

reviewed

··· 46 46 select HAVE_KVM_PM_NOTIFIER if PM 47 47 select KVM_GENERIC_HARDWARE_ENABLING 48 48 select KVM_GENERIC_PRE_FAULT_MEMORY 49 49 - select KVM_GENERIC_PRIVATE_MEM if KVM_SW_PROTECTED_VM 50 49 select KVM_WERROR if WERROR 50 50 + select KVM_GUEST_MEMFD if X86_64 51 51 52 52 config KVM 53 53 tristate "Kernel-based Virtual Machine (KVM) support" ··· 74 74 # FRAME_WARN, i.e. KVM_WERROR=y with KASAN=y requires special tuning. 75 75 # Building KVM with -Werror and KASAN is still doable via enabling 76 76 # the kernel-wide WERROR=y. 77 77 - depends on KVM && ((EXPERT && !KASAN) || WERROR) 77 77 + depends on KVM_X86 && ((EXPERT && !KASAN) || WERROR) 78 78 help 79 79 Add -Werror to the build flags for KVM. 80 80 ··· 83 83 config KVM_SW_PROTECTED_VM 84 84 bool "Enable support for KVM software-protected VMs" 85 85 depends on EXPERT 86 86 - depends on KVM && X86_64 86 86 + depends on KVM_X86 && X86_64 87 87 + select KVM_GENERIC_MEMORY_ATTRIBUTES 87 88 help 88 89 Enable support for KVM software-protected VMs. Currently, software- 89 90 protected VMs are purely a development and testing vehicle for ··· 96 95 config KVM_INTEL 97 96 tristate "KVM for Intel (and compatible) processors support" 98 97 depends on KVM && IA32_FEAT_CTL 99 99 - select KVM_GENERIC_PRIVATE_MEM if INTEL_TDX_HOST 100 100 - select KVM_GENERIC_MEMORY_ATTRIBUTES if INTEL_TDX_HOST 101 98 help 102 99 Provides support for KVM on processors equipped with Intel's VT 103 100 extensions, a.k.a. Virtual Machine Extensions (VMX). ··· 134 135 bool "Intel Trust Domain Extensions (TDX) support" 135 136 default y 136 137 depends on INTEL_TDX_HOST 138 138 + select KVM_GENERIC_MEMORY_ATTRIBUTES 139 139 + select HAVE_KVM_ARCH_GMEM_POPULATE 137 140 help 138 141 Provides support for launching Intel Trust Domain Extensions (TDX) 139 142 confidential VMs on Intel processors. ··· 158 157 depends on KVM_AMD && X86_64 159 158 depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m) 160 159 select ARCH_HAS_CC_PLATFORM 161 161 - select KVM_GENERIC_PRIVATE_MEM 160 160 + select KVM_GENERIC_MEMORY_ATTRIBUTES 162 161 select HAVE_KVM_ARCH_GMEM_PREPARE 163 162 select HAVE_KVM_ARCH_GMEM_INVALIDATE 163 163 + select HAVE_KVM_ARCH_GMEM_POPULATE 164 164 help 165 165 Provides support for launching encrypted VMs which use Secure 166 166 Encrypted Virtualization (SEV), Secure Encrypted Virtualization with ··· 171 169 config KVM_IOAPIC 172 170 bool "I/O APIC, PIC, and PIT emulation" 173 171 default y 174 174 - depends on KVM 172 172 + depends on KVM_X86 175 173 help 176 174 Provides support for KVM to emulate an I/O APIC, PIC, and PIT, i.e. 177 175 for full in-kernel APIC emulation. ··· 181 179 config KVM_SMM 182 180 bool "System Management Mode emulation" 183 181 default y 184 184 - depends on KVM 182 182 + depends on KVM_X86 185 183 help 186 184 Provides support for KVM to emulate System Management Mode (SMM) 187 185 in virtual machines. This can be used by the virtual machine ··· 191 189 192 190 config KVM_HYPERV 193 191 bool "Support for Microsoft Hyper-V emulation" 194 194 - depends on KVM 192 192 + depends on KVM_X86 195 193 default y 196 194 help 197 195 Provides KVM support for emulating Microsoft Hyper-V. This allows KVM ··· 205 203 206 204 config KVM_XEN 207 205 bool "Support for Xen hypercall interface" 208 208 - depends on KVM 206 206 + depends on KVM_X86 209 207 help 210 208 Provides KVM support for the hosting Xen HVM guests and 211 209 passing Xen hypercalls to userspace. ··· 215 213 config KVM_PROVE_MMU 216 214 bool "Prove KVM MMU correctness" 217 215 depends on DEBUG_KERNEL 218 218 - depends on KVM 216 216 + depends on KVM_X86 219 217 depends on EXPERT 220 218 help 221 219 Enables runtime assertions in KVM's MMU that are too costly to enable ··· 230 228 231 229 config KVM_MAX_NR_VCPUS 232 230 int "Maximum number of vCPUs per KVM guest" 233 233 - depends on KVM 231 231 + depends on KVM_X86 234 232 range 1024 4096 235 233 default 4096 if MAXSMP 236 234 default 1024

+78 -64

arch/x86/kvm/mmu/mmu.c

reviewed

··· 3285 3285 return level; 3286 3286 } 3287 3287 3288 3288 - static int __kvm_mmu_max_mapping_level(struct kvm *kvm, 3289 3289 - const struct kvm_memory_slot *slot, 3290 3290 - gfn_t gfn, int max_level, bool is_private) 3288 3288 + static u8 kvm_max_level_for_order(int order) 3289 3289 + { 3290 3290 + BUILD_BUG_ON(KVM_MAX_HUGEPAGE_LEVEL > PG_LEVEL_1G); 3291 3291 + 3292 3292 + KVM_MMU_WARN_ON(order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_1G) && 3293 3293 + order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M) && 3294 3294 + order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_4K)); 3295 3295 + 3296 3296 + if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_1G)) 3297 3297 + return PG_LEVEL_1G; 3298 3298 + 3299 3299 + if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M)) 3300 3300 + return PG_LEVEL_2M; 3301 3301 + 3302 3302 + return PG_LEVEL_4K; 3303 3303 + } 3304 3304 + 3305 3305 + static u8 kvm_gmem_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault, 3306 3306 + const struct kvm_memory_slot *slot, gfn_t gfn, 3307 3307 + bool is_private) 3308 3308 + { 3309 3309 + u8 max_level, coco_level; 3310 3310 + kvm_pfn_t pfn; 3311 3311 + 3312 3312 + /* For faults, use the gmem information that was resolved earlier. */ 3313 3313 + if (fault) { 3314 3314 + pfn = fault->pfn; 3315 3315 + max_level = fault->max_level; 3316 3316 + } else { 3317 3317 + /* TODO: Call into guest_memfd once hugepages are supported. */ 3318 3318 + WARN_ONCE(1, "Get pfn+order from guest_memfd"); 3319 3319 + pfn = KVM_PFN_ERR_FAULT; 3320 3320 + max_level = PG_LEVEL_4K; 3321 3321 + } 3322 3322 + 3323 3323 + if (max_level == PG_LEVEL_4K) 3324 3324 + return max_level; 3325 3325 + 3326 3326 + /* 3327 3327 + * CoCo may influence the max mapping level, e.g. due to RMP or S-EPT 3328 3328 + * restrictions. A return of '0' means "no additional restrictions", to 3329 3329 + * allow for using an optional "ret0" static call. 3330 3330 + */ 3331 3331 + coco_level = kvm_x86_call(gmem_max_mapping_level)(kvm, pfn, is_private); 3332 3332 + if (coco_level) 3333 3333 + max_level = min(max_level, coco_level); 3334 3334 + 3335 3335 + return max_level; 3336 3336 + } 3337 3337 + 3338 3338 + int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault, 3339 3339 + const struct kvm_memory_slot *slot, gfn_t gfn) 3291 3340 { 3292 3341 struct kvm_lpage_info *linfo; 3293 3293 - int host_level; 3342 3342 + int host_level, max_level; 3343 3343 + bool is_private; 3344 3344 + 3345 3345 + lockdep_assert_held(&kvm->mmu_lock); 3346 3346 + 3347 3347 + if (fault) { 3348 3348 + max_level = fault->max_level; 3349 3349 + is_private = fault->is_private; 3350 3350 + } else { 3351 3351 + max_level = PG_LEVEL_NUM; 3352 3352 + is_private = kvm_mem_is_private(kvm, gfn); 3353 3353 + } 3294 3354 3295 3355 max_level = min(max_level, max_huge_page_level); 3296 3356 for ( ; max_level > PG_LEVEL_4K; max_level--) { ··· 3359 3299 break; 3360 3300 } 3361 3301 3362 3362 - if (is_private) 3363 3363 - return max_level; 3364 3364 - 3365 3302 if (max_level == PG_LEVEL_4K) 3366 3303 return PG_LEVEL_4K; 3367 3304 3368 3368 - host_level = host_pfn_mapping_level(kvm, gfn, slot); 3305 3305 + if (is_private || kvm_memslot_is_gmem_only(slot)) 3306 3306 + host_level = kvm_gmem_max_mapping_level(kvm, fault, slot, gfn, 3307 3307 + is_private); 3308 3308 + else 3309 3309 + host_level = host_pfn_mapping_level(kvm, gfn, slot); 3369 3310 return min(host_level, max_level); 3370 3370 - } 3371 3371 - 3372 3372 - int kvm_mmu_max_mapping_level(struct kvm *kvm, 3373 3373 - const struct kvm_memory_slot *slot, gfn_t gfn) 3374 3374 - { 3375 3375 - bool is_private = kvm_slot_can_be_private(slot) && 3376 3376 - kvm_mem_is_private(kvm, gfn); 3377 3377 - 3378 3378 - return __kvm_mmu_max_mapping_level(kvm, slot, gfn, PG_LEVEL_NUM, is_private); 3379 3311 } 3380 3312 3381 3313 void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) ··· 3390 3338 * Enforce the iTLB multihit workaround after capturing the requested 3391 3339 * level, which will be used to do precise, accurate accounting. 3392 3340 */ 3393 3393 - fault->req_level = __kvm_mmu_max_mapping_level(vcpu->kvm, slot, 3394 3394 - fault->gfn, fault->max_level, 3395 3395 - fault->is_private); 3341 3341 + fault->req_level = kvm_mmu_max_mapping_level(vcpu->kvm, fault, 3342 3342 + fault->slot, fault->gfn); 3396 3343 if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed) 3397 3344 return; 3398 3345 ··· 4554 4503 vcpu->stat.pf_fixed++; 4555 4504 } 4556 4505 4557 4557 - static inline u8 kvm_max_level_for_order(int order) 4558 4558 - { 4559 4559 - BUILD_BUG_ON(KVM_MAX_HUGEPAGE_LEVEL > PG_LEVEL_1G); 4560 4560 - 4561 4561 - KVM_MMU_WARN_ON(order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_1G) && 4562 4562 - order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M) && 4563 4563 - order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_4K)); 4564 4564 - 4565 4565 - if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_1G)) 4566 4566 - return PG_LEVEL_1G; 4567 4567 - 4568 4568 - if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M)) 4569 4569 - return PG_LEVEL_2M; 4570 4570 - 4571 4571 - return PG_LEVEL_4K; 4572 4572 - } 4573 4573 - 4574 4574 - static u8 kvm_max_private_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, 4575 4575 - u8 max_level, int gmem_order) 4576 4576 - { 4577 4577 - u8 req_max_level; 4578 4578 - 4579 4579 - if (max_level == PG_LEVEL_4K) 4580 4580 - return PG_LEVEL_4K; 4581 4581 - 4582 4582 - max_level = min(kvm_max_level_for_order(gmem_order), max_level); 4583 4583 - if (max_level == PG_LEVEL_4K) 4584 4584 - return PG_LEVEL_4K; 4585 4585 - 4586 4586 - req_max_level = kvm_x86_call(private_max_mapping_level)(kvm, pfn); 4587 4587 - if (req_max_level) 4588 4588 - max_level = min(max_level, req_max_level); 4589 4589 - 4590 4590 - return max_level; 4591 4591 - } 4592 4592 - 4593 4506 static void kvm_mmu_finish_page_fault(struct kvm_vcpu *vcpu, 4594 4507 struct kvm_page_fault *fault, int r) 4595 4508 { ··· 4561 4546 r == RET_PF_RETRY, fault->map_writable); 4562 4547 } 4563 4548 4564 4564 - static int kvm_mmu_faultin_pfn_private(struct kvm_vcpu *vcpu, 4565 4565 - struct kvm_page_fault *fault) 4549 4549 + static int kvm_mmu_faultin_pfn_gmem(struct kvm_vcpu *vcpu, 4550 4550 + struct kvm_page_fault *fault) 4566 4551 { 4567 4552 int max_order, r; 4568 4553 4569 4569 - if (!kvm_slot_can_be_private(fault->slot)) { 4554 4554 + if (!kvm_slot_has_gmem(fault->slot)) { 4570 4555 kvm_mmu_prepare_memory_fault_exit(vcpu, fault); 4571 4556 return -EFAULT; 4572 4557 } ··· 4579 4564 } 4580 4565 4581 4566 fault->map_writable = !(fault->slot->flags & KVM_MEM_READONLY); 4582 4582 - fault->max_level = kvm_max_private_mapping_level(vcpu->kvm, fault->pfn, 4583 4583 - fault->max_level, max_order); 4567 4567 + fault->max_level = kvm_max_level_for_order(max_order); 4584 4568 4585 4569 return RET_PF_CONTINUE; 4586 4570 } ··· 4589 4575 { 4590 4576 unsigned int foll = fault->write ? FOLL_WRITE : 0; 4591 4577 4592 4592 - if (fault->is_private) 4593 4593 - return kvm_mmu_faultin_pfn_private(vcpu, fault); 4578 4578 + if (fault->is_private || kvm_memslot_is_gmem_only(fault->slot)) 4579 4579 + return kvm_mmu_faultin_pfn_gmem(vcpu, fault); 4594 4580 4595 4581 foll |= FOLL_NOWAIT; 4596 4582 fault->pfn = __kvm_faultin_pfn(fault->slot, fault->gfn, foll, ··· 7179 7165 * mapping if the indirect sp has level = 1. 7180 7166 */ 7181 7167 if (sp->role.direct && 7182 7182 - sp->role.level < kvm_mmu_max_mapping_level(kvm, slot, sp->gfn)) { 7168 7168 + sp->role.level < kvm_mmu_max_mapping_level(kvm, NULL, slot, sp->gfn)) { 7183 7169 kvm_zap_one_rmap_spte(kvm, rmap_head, sptep); 7184 7170 7185 7171 if (kvm_available_flush_remote_tlbs_range())

+1 -1

arch/x86/kvm/mmu/mmu_internal.h

reviewed

··· 411 411 return r; 412 412 } 413 413 414 414 - int kvm_mmu_max_mapping_level(struct kvm *kvm, 414 414 + int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault, 415 415 const struct kvm_memory_slot *slot, gfn_t gfn); 416 416 void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); 417 417 void disallowed_hugepage_adjust(struct kvm_page_fault *fault, u64 spte, int cur_level);

+1 -1

arch/x86/kvm/mmu/tdp_mmu.c

reviewed

··· 1813 1813 if (iter.gfn < start || iter.gfn >= end) 1814 1814 continue; 1815 1815 1816 1816 - max_mapping_level = kvm_mmu_max_mapping_level(kvm, slot, iter.gfn); 1816 1816 + max_mapping_level = kvm_mmu_max_mapping_level(kvm, NULL, slot, iter.gfn); 1817 1817 if (max_mapping_level < iter.level) 1818 1818 continue; 1819 1819

+3 -3

arch/x86/kvm/svm/sev.c

reviewed

··· 2361 2361 mutex_lock(&kvm->slots_lock); 2362 2362 2363 2363 memslot = gfn_to_memslot(kvm, params.gfn_start); 2364 2364 - if (!kvm_slot_can_be_private(memslot)) { 2364 2364 + if (!kvm_slot_has_gmem(memslot)) { 2365 2365 ret = -EINVAL; 2366 2366 goto out; 2367 2367 } ··· 4715 4715 } 4716 4716 4717 4717 slot = gfn_to_memslot(kvm, gfn); 4718 4718 - if (!kvm_slot_can_be_private(slot)) { 4718 4718 + if (!kvm_slot_has_gmem(slot)) { 4719 4719 pr_warn_ratelimited("SEV: Unexpected RMP fault, non-private slot for GPA 0x%llx\n", 4720 4720 gpa); 4721 4721 return; ··· 4943 4943 } 4944 4944 } 4945 4945 4946 4946 - int sev_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn) 4946 4946 + int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) 4947 4947 { 4948 4948 int level, rc; 4949 4949 bool assigned;

+1 -1

arch/x86/kvm/svm/svm.c

reviewed

··· 5179 5179 5180 5180 .gmem_prepare = sev_gmem_prepare, 5181 5181 .gmem_invalidate = sev_gmem_invalidate, 5182 5182 - .private_max_mapping_level = sev_private_max_mapping_level, 5182 5182 + .gmem_max_mapping_level = sev_gmem_max_mapping_level, 5183 5183 }; 5184 5184 5185 5185 /*

+2 -2

arch/x86/kvm/svm/svm.h

reviewed

··· 866 866 void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu); 867 867 int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); 868 868 void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); 869 869 - int sev_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn); 869 869 + int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); 870 870 struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu); 871 871 void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_save_area *vmsa); 872 872 #else ··· 895 895 return 0; 896 896 } 897 897 static inline void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) {} 898 898 - static inline int sev_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn) 898 898 + static inline int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) 899 899 { 900 900 return 0; 901 901 }

+4 -3

arch/x86/kvm/vmx/main.c

reviewed

··· 831 831 return tdx_vcpu_ioctl(vcpu, argp); 832 832 } 833 833 834 834 - static int vt_gmem_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn) 834 834 + static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, 835 835 + bool is_private) 835 836 { 836 837 if (is_td(kvm)) 837 837 - return tdx_gmem_private_max_mapping_level(kvm, pfn); 838 838 + return tdx_gmem_max_mapping_level(kvm, pfn, is_private); 838 839 839 840 return 0; 840 841 } ··· 1006 1005 .mem_enc_ioctl = vt_op_tdx_only(mem_enc_ioctl), 1007 1006 .vcpu_mem_enc_ioctl = vt_op_tdx_only(vcpu_mem_enc_ioctl), 1008 1007 1009 1009 - .private_max_mapping_level = vt_op_tdx_only(gmem_private_max_mapping_level) 1008 1008 + .gmem_max_mapping_level = vt_op_tdx_only(gmem_max_mapping_level) 1010 1009 }; 1011 1010 1012 1011 struct kvm_x86_init_ops vt_init_ops __initdata = {

+4 -1

arch/x86/kvm/vmx/tdx.c

reviewed

··· 3318 3318 return ret; 3319 3319 } 3320 3320 3321 3321 - int tdx_gmem_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn) 3321 3321 + int tdx_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) 3322 3322 { 3323 3323 + if (!is_private) 3324 3324 + return 0; 3325 3325 + 3323 3326 return PG_LEVEL_4K; 3324 3327 } 3325 3328

+7

arch/x86/kvm/vmx/vmx.c

reviewed

··· 5785 5785 if (kvm_test_request(KVM_REQ_EVENT, vcpu)) 5786 5786 return 1; 5787 5787 5788 5788 + /* 5789 5789 + * Ensure that any updates to kvm->buses[] observed by the 5790 5790 + * previous instruction (emulated or otherwise) are also 5791 5791 + * visible to the instruction KVM is about to emulate. 5792 5792 + */ 5793 5793 + smp_rmb(); 5794 5794 + 5788 5795 if (!kvm_emulate_instruction(vcpu, 0)) 5789 5796 return 0; 5790 5797

+1 -1

arch/x86/kvm/vmx/x86_ops.h

reviewed

··· 153 153 void tdx_flush_tlb_current(struct kvm_vcpu *vcpu); 154 154 void tdx_flush_tlb_all(struct kvm_vcpu *vcpu); 155 155 void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level); 156 156 - int tdx_gmem_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn); 156 156 + int tdx_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); 157 157 #endif 158 158 159 159 #endif /* __KVM_X86_VMX_X86_OPS_H */

+11

arch/x86/kvm/x86.c

reviewed

··· 13530 13530 } 13531 13531 EXPORT_SYMBOL_GPL(kvm_arch_no_poll); 13532 13532 13533 13533 + #ifdef CONFIG_KVM_GUEST_MEMFD 13534 13534 + /* 13535 13535 + * KVM doesn't yet support mmap() on guest_memfd for VMs with private memory 13536 13536 + * (the private vs. shared tracking needs to be moved into guest_memfd). 13537 13537 + */ 13538 13538 + bool kvm_arch_supports_gmem_mmap(struct kvm *kvm) 13539 13539 + { 13540 13540 + return !kvm_arch_has_private_mem(kvm); 13541 13541 + } 13542 13542 + 13533 13543 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE 13534 13544 int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order) 13535 13545 { ··· 13552 13542 { 13553 13543 kvm_x86_call(gmem_invalidate)(start, end); 13554 13544 } 13545 13545 + #endif 13555 13546 #endif 13556 13547 13557 13548 int kvm_spec_ctrl_test_value(u64 value)

-7

drivers/irqchip/irq-gic-v5.c

reviewed

··· 1062 1062 #ifdef CONFIG_KVM 1063 1063 static struct gic_kvm_info gic_v5_kvm_info __initdata; 1064 1064 1065 1065 - static bool __init gicv5_cpuif_has_gcie_legacy(void) 1066 1066 - { 1067 1067 - u64 idr0 = read_sysreg_s(SYS_ICC_IDR0_EL1); 1068 1068 - return !!FIELD_GET(ICC_IDR0_EL1_GCIE_LEGACY, idr0); 1069 1069 - } 1070 1070 - 1071 1065 static void __init gic_of_setup_kvm_info(struct device_node *node) 1072 1066 { 1073 1067 gic_v5_kvm_info.type = GIC_V5; 1074 1074 - gic_v5_kvm_info.has_gcie_v3_compat = gicv5_cpuif_has_gcie_legacy(); 1075 1068 1076 1069 /* GIC Virtual CPU interface maintenance interrupt */ 1077 1070 gic_v5_kvm_info.no_maint_irq_mask = false;

+141 -50

drivers/perf/riscv_pmu_sbi.c

reviewed

··· 59 59 #define PERF_EVENT_FLAG_USER_ACCESS BIT(SYSCTL_USER_ACCESS) 60 60 #define PERF_EVENT_FLAG_LEGACY BIT(SYSCTL_LEGACY) 61 61 62 62 - PMU_FORMAT_ATTR(event, "config:0-47"); 62 62 + PMU_FORMAT_ATTR(event, "config:0-55"); 63 63 PMU_FORMAT_ATTR(firmware, "config:62-63"); 64 64 65 65 static bool sbi_v2_available; 66 66 + static bool sbi_v3_available; 66 67 static DEFINE_STATIC_KEY_FALSE(sbi_pmu_snapshot_available); 67 68 #define sbi_pmu_snapshot_available() \ 68 69 static_branch_unlikely(&sbi_pmu_snapshot_available) ··· 100 99 /* Cache the available counters in a bitmask */ 101 100 static unsigned long cmask; 102 101 102 102 + static int pmu_event_find_cache(u64 config); 103 103 struct sbi_pmu_event_data { 104 104 union { 105 105 union { ··· 300 298 }, 301 299 }; 302 300 301 301 + static int pmu_sbi_check_event_info(void) 302 302 + { 303 303 + int num_events = ARRAY_SIZE(pmu_hw_event_map) + PERF_COUNT_HW_CACHE_MAX * 304 304 + PERF_COUNT_HW_CACHE_OP_MAX * PERF_COUNT_HW_CACHE_RESULT_MAX; 305 305 + struct riscv_pmu_event_info *event_info_shmem; 306 306 + phys_addr_t base_addr; 307 307 + int i, j, k, result = 0, count = 0; 308 308 + struct sbiret ret; 309 309 + 310 310 + event_info_shmem = kcalloc(num_events, sizeof(*event_info_shmem), GFP_KERNEL); 311 311 + if (!event_info_shmem) 312 312 + return -ENOMEM; 313 313 + 314 314 + for (i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++) 315 315 + event_info_shmem[count++].event_idx = pmu_hw_event_map[i].event_idx; 316 316 + 317 317 + for (i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++) { 318 318 + for (j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++) { 319 319 + for (k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++) 320 320 + event_info_shmem[count++].event_idx = 321 321 + pmu_cache_event_map[i][j][k].event_idx; 322 322 + } 323 323 + } 324 324 + 325 325 + base_addr = __pa(event_info_shmem); 326 326 + if (IS_ENABLED(CONFIG_32BIT)) 327 327 + ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_EVENT_GET_INFO, lower_32_bits(base_addr), 328 328 + upper_32_bits(base_addr), count, 0, 0, 0); 329 329 + else 330 330 + ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_EVENT_GET_INFO, base_addr, 0, 331 331 + count, 0, 0, 0); 332 332 + if (ret.error) { 333 333 + result = -EOPNOTSUPP; 334 334 + goto free_mem; 335 335 + } 336 336 + 337 337 + for (i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++) { 338 338 + if (!(event_info_shmem[i].output & RISCV_PMU_EVENT_INFO_OUTPUT_MASK)) 339 339 + pmu_hw_event_map[i].event_idx = -ENOENT; 340 340 + } 341 341 + 342 342 + count = ARRAY_SIZE(pmu_hw_event_map); 343 343 + 344 344 + for (i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++) { 345 345 + for (j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++) { 346 346 + for (k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++) { 347 347 + if (!(event_info_shmem[count].output & 348 348 + RISCV_PMU_EVENT_INFO_OUTPUT_MASK)) 349 349 + pmu_cache_event_map[i][j][k].event_idx = -ENOENT; 350 350 + count++; 351 351 + } 352 352 + } 353 353 + } 354 354 + 355 355 + free_mem: 356 356 + kfree(event_info_shmem); 357 357 + 358 358 + return result; 359 359 + } 360 360 + 303 361 static void pmu_sbi_check_event(struct sbi_pmu_event_data *edata) 304 362 { 305 363 struct sbiret ret; ··· 377 315 378 316 static void pmu_sbi_check_std_events(struct work_struct *work) 379 317 { 318 318 + int ret; 319 319 + 320 320 + if (sbi_v3_available) { 321 321 + ret = pmu_sbi_check_event_info(); 322 322 + if (ret) 323 323 + pr_err("pmu_sbi_check_event_info failed with error %d\n", ret); 324 324 + return; 325 325 + } 326 326 + 380 327 for (int i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++) 381 328 pmu_sbi_check_event(&pmu_hw_event_map[i]); 382 329 ··· 412 341 413 342 return info->type == SBI_PMU_CTR_TYPE_FW; 414 343 } 344 344 + 345 345 + int riscv_pmu_get_event_info(u32 type, u64 config, u64 *econfig) 346 346 + { 347 347 + int ret = -ENOENT; 348 348 + 349 349 + switch (type) { 350 350 + case PERF_TYPE_HARDWARE: 351 351 + if (config >= PERF_COUNT_HW_MAX) 352 352 + return -EINVAL; 353 353 + ret = pmu_hw_event_map[config].event_idx; 354 354 + break; 355 355 + case PERF_TYPE_HW_CACHE: 356 356 + ret = pmu_event_find_cache(config); 357 357 + break; 358 358 + case PERF_TYPE_RAW: 359 359 + /* 360 360 + * As per SBI v0.3 specification, 361 361 + * -- the upper 16 bits must be unused for a hardware raw event. 362 362 + * As per SBI v2.0 specification, 363 363 + * -- the upper 8 bits must be unused for a hardware raw event. 364 364 + * Bits 63:62 are used to distinguish between raw events 365 365 + * 00 - Hardware raw event 366 366 + * 10 - SBI firmware events 367 367 + * 11 - Risc-V platform specific firmware event 368 368 + */ 369 369 + switch (config >> 62) { 370 370 + case 0: 371 371 + if (sbi_v3_available) { 372 372 + /* Return error any bits [56-63] is set as it is not allowed by the spec */ 373 373 + if (!(config & ~RISCV_PMU_RAW_EVENT_V2_MASK)) { 374 374 + if (econfig) 375 375 + *econfig = config & RISCV_PMU_RAW_EVENT_V2_MASK; 376 376 + ret = RISCV_PMU_RAW_EVENT_V2_IDX; 377 377 + } 378 378 + /* Return error any bits [48-63] is set as it is not allowed by the spec */ 379 379 + } else if (!(config & ~RISCV_PMU_RAW_EVENT_MASK)) { 380 380 + if (econfig) 381 381 + *econfig = config & RISCV_PMU_RAW_EVENT_MASK; 382 382 + ret = RISCV_PMU_RAW_EVENT_IDX; 383 383 + } 384 384 + break; 385 385 + case 2: 386 386 + ret = (config & 0xFFFF) | (SBI_PMU_EVENT_TYPE_FW << 16); 387 387 + break; 388 388 + case 3: 389 389 + /* 390 390 + * For Risc-V platform specific firmware events 391 391 + * Event code - 0xFFFF 392 392 + * Event data - raw event encoding 393 393 + */ 394 394 + ret = SBI_PMU_EVENT_TYPE_FW << 16 | RISCV_PLAT_FW_EVENT; 395 395 + if (econfig) 396 396 + *econfig = config & RISCV_PMU_PLAT_FW_EVENT_MASK; 397 397 + break; 398 398 + default: 399 399 + break; 400 400 + } 401 401 + break; 402 402 + default: 403 403 + break; 404 404 + } 405 405 + 406 406 + return ret; 407 407 + } 408 408 + EXPORT_SYMBOL_GPL(riscv_pmu_get_event_info); 415 409 416 410 /* 417 411 * Returns the counter width of a programmable counter and number of hardware ··· 643 507 { 644 508 u32 type = event->attr.type; 645 509 u64 config = event->attr.config; 646 646 - int ret = -ENOENT; 647 510 648 511 /* 649 512 * Ensure we are finished checking standard hardware events for ··· 650 515 */ 651 516 flush_work(&check_std_events_work); 652 517 653 653 - switch (type) { 654 654 - case PERF_TYPE_HARDWARE: 655 655 - if (config >= PERF_COUNT_HW_MAX) 656 656 - return -EINVAL; 657 657 - ret = pmu_hw_event_map[event->attr.config].event_idx; 658 658 - break; 659 659 - case PERF_TYPE_HW_CACHE: 660 660 - ret = pmu_event_find_cache(config); 661 661 - break; 662 662 - case PERF_TYPE_RAW: 663 663 - /* 664 664 - * As per SBI specification, the upper 16 bits must be unused 665 665 - * for a hardware raw event. 666 666 - * Bits 63:62 are used to distinguish between raw events 667 667 - * 00 - Hardware raw event 668 668 - * 10 - SBI firmware events 669 669 - * 11 - Risc-V platform specific firmware event 670 670 - */ 671 671 - 672 672 - switch (config >> 62) { 673 673 - case 0: 674 674 - /* Return error any bits [48-63] is set as it is not allowed by the spec */ 675 675 - if (!(config & ~RISCV_PMU_RAW_EVENT_MASK)) { 676 676 - *econfig = config & RISCV_PMU_RAW_EVENT_MASK; 677 677 - ret = RISCV_PMU_RAW_EVENT_IDX; 678 678 - } 679 679 - break; 680 680 - case 2: 681 681 - ret = (config & 0xFFFF) | (SBI_PMU_EVENT_TYPE_FW << 16); 682 682 - break; 683 683 - case 3: 684 684 - /* 685 685 - * For Risc-V platform specific firmware events 686 686 - * Event code - 0xFFFF 687 687 - * Event data - raw event encoding 688 688 - */ 689 689 - ret = SBI_PMU_EVENT_TYPE_FW << 16 | RISCV_PLAT_FW_EVENT; 690 690 - *econfig = config & RISCV_PMU_PLAT_FW_EVENT_MASK; 691 691 - break; 692 692 - default: 693 693 - break; 694 694 - } 695 695 - break; 696 696 - default: 697 697 - break; 698 698 - } 699 699 - 700 700 - return ret; 518 518 + return riscv_pmu_get_event_info(type, config, econfig); 701 519 } 702 520 703 521 static void pmu_sbi_snapshot_free(struct riscv_pmu *pmu) ··· 1541 1453 1542 1454 if (sbi_spec_version >= sbi_mk_version(2, 0)) 1543 1455 sbi_v2_available = true; 1456 1456 + 1457 1457 + if (sbi_spec_version >= sbi_mk_version(3, 0)) 1458 1458 + sbi_v3_available = true; 1544 1459 1545 1460 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_RISCV_STARTING, 1546 1461 "perf/riscv/pmu:starting",

+1 -1

include/kvm/arm_vgic.h

reviewed

··· 378 378 379 379 extern struct static_key_false vgic_v2_cpuif_trap; 380 380 extern struct static_key_false vgic_v3_cpuif_trap; 381 381 + extern struct static_key_false vgic_v3_has_v2_compat; 381 382 382 383 int kvm_set_legacy_vgic_v2_addr(struct kvm *kvm, struct kvm_arm_device_addr *dev_addr); 383 384 void kvm_vgic_early_init(struct kvm *kvm); ··· 410 409 411 410 #define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel)) 412 411 #define vgic_initialized(k) ((k)->arch.vgic.initialized) 413 413 - #define vgic_ready(k) ((k)->arch.vgic.ready) 414 412 #define vgic_valid_spi(k, i) (((i) >= VGIC_NR_PRIVATE_IRQS) && \ 415 413 ((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) 416 414

+1

include/linux/arm_ffa.h

reviewed

··· 128 128 #define FFA_FEAT_RXTX_MIN_SZ_4K 0 129 129 #define FFA_FEAT_RXTX_MIN_SZ_64K 1 130 130 #define FFA_FEAT_RXTX_MIN_SZ_16K 2 131 131 + #define FFA_FEAT_RXTX_MIN_SZ_MASK GENMASK(1, 0) 131 132 132 133 /* FFA Bus/Device/Driver related */ 133 134 struct ffa_device {

-2

include/linux/irqchip/arm-vgic-info.h

reviewed

··· 36 36 bool has_v4_1; 37 37 /* Deactivation impared, subpar stuff */ 38 38 bool no_hw_deactivation; 39 39 - /* v3 compat support (GICv5 hosts, only) */ 40 40 - bool has_gcie_v3_compat; 41 39 }; 42 40 43 41 #ifdef CONFIG_KVM

+31 -18

include/linux/kvm_host.h

reviewed

··· 52 52 /* 53 53 * The bit 16 ~ bit 31 of kvm_userspace_memory_region::flags are internally 54 54 * used in kvm, other bits are visible for userspace which are defined in 55 55 - * include/linux/kvm_h. 55 55 + * include/uapi/linux/kvm.h. 56 56 */ 57 57 - #define KVM_MEMSLOT_INVALID (1UL << 16) 57 57 + #define KVM_MEMSLOT_INVALID (1UL << 16) 58 58 + #define KVM_MEMSLOT_GMEM_ONLY (1UL << 17) 58 59 59 60 /* 60 61 * Bit 63 of the memslot generation number is an "update in-progress flag", ··· 207 206 struct kvm_io_bus { 208 207 int dev_count; 209 208 int ioeventfd_count; 209 209 + struct rcu_head rcu; 210 210 struct kvm_io_range range[]; 211 211 }; 212 212 ··· 604 602 short id; 605 603 u16 as_id; 606 604 607 607 - #ifdef CONFIG_KVM_PRIVATE_MEM 605 605 + #ifdef CONFIG_KVM_GUEST_MEMFD 608 606 struct { 609 607 /* 610 608 * Writes protected by kvm->slots_lock. Acquiring a ··· 617 615 #endif 618 616 }; 619 617 620 620 - static inline bool kvm_slot_can_be_private(const struct kvm_memory_slot *slot) 618 618 + static inline bool kvm_slot_has_gmem(const struct kvm_memory_slot *slot) 621 619 { 622 620 return slot && (slot->flags & KVM_MEM_GUEST_MEMFD); 623 621 } ··· 721 719 } 722 720 #endif 723 721 724 724 - /* 725 725 - * Arch code must define kvm_arch_has_private_mem if support for private memory 726 726 - * is enabled. 727 727 - */ 728 728 - #if !defined(kvm_arch_has_private_mem) && !IS_ENABLED(CONFIG_KVM_PRIVATE_MEM) 722 722 + #ifndef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES 729 723 static inline bool kvm_arch_has_private_mem(struct kvm *kvm) 730 724 { 731 725 return false; 732 726 } 727 727 + #endif 728 728 + 729 729 + #ifdef CONFIG_KVM_GUEST_MEMFD 730 730 + bool kvm_arch_supports_gmem_mmap(struct kvm *kvm); 733 731 #endif 734 732 735 733 #ifndef kvm_arch_has_readonly_mem ··· 862 860 struct notifier_block pm_notifier; 863 861 #endif 864 862 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES 865 865 - /* Protected by slots_locks (for writes) and RCU (for reads) */ 863 863 + /* Protected by slots_lock (for writes) and RCU (for reads) */ 866 864 struct xarray mem_attr_array; 867 865 #endif 868 866 char stats_id[KVM_STATS_NAME_SIZE]; ··· 968 966 return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET); 969 967 } 970 968 969 969 + /* 970 970 + * Get a bus reference under the update-side lock. No long-term SRCU reader 971 971 + * references are permitted, to avoid stale reads vs concurrent IO 972 972 + * registrations. 973 973 + */ 971 974 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) 972 975 { 973 973 - return srcu_dereference_check(kvm->buses[idx], &kvm->srcu, 974 974 - lockdep_is_held(&kvm->slots_lock) || 975 975 - !refcount_read(&kvm->users_count)); 976 976 + return rcu_dereference_protected(kvm->buses[idx], 977 977 + lockdep_is_held(&kvm->slots_lock)); 976 978 } 977 979 978 980 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) ··· 2496 2490 vcpu->run->memory_fault.flags |= KVM_MEMORY_EXIT_FLAG_PRIVATE; 2497 2491 } 2498 2492 2493 2493 + static inline bool kvm_memslot_is_gmem_only(const struct kvm_memory_slot *slot) 2494 2494 + { 2495 2495 + if (!IS_ENABLED(CONFIG_KVM_GUEST_MEMFD)) 2496 2496 + return false; 2497 2497 + 2498 2498 + return slot->flags & KVM_MEMSLOT_GMEM_ONLY; 2499 2499 + } 2500 2500 + 2499 2501 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES 2500 2502 static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn) 2501 2503 { ··· 2519 2505 2520 2506 static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) 2521 2507 { 2522 2522 - return IS_ENABLED(CONFIG_KVM_PRIVATE_MEM) && 2523 2523 - kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; 2508 2508 + return kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; 2524 2509 } 2525 2510 #else 2526 2511 static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) ··· 2528 2515 } 2529 2516 #endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ 2530 2517 2531 2531 - #ifdef CONFIG_KVM_PRIVATE_MEM 2518 2518 + #ifdef CONFIG_KVM_GUEST_MEMFD 2532 2519 int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, 2533 2520 gfn_t gfn, kvm_pfn_t *pfn, struct page **page, 2534 2521 int *max_order); ··· 2541 2528 KVM_BUG_ON(1, kvm); 2542 2529 return -EIO; 2543 2530 } 2544 2544 - #endif /* CONFIG_KVM_PRIVATE_MEM */ 2531 2531 + #endif /* CONFIG_KVM_GUEST_MEMFD */ 2545 2532 2546 2533 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE 2547 2534 int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order); 2548 2535 #endif 2549 2536 2550 2550 - #ifdef CONFIG_KVM_GENERIC_PRIVATE_MEM 2537 2537 + #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE 2551 2538 /** 2552 2539 * kvm_gmem_populate() - Populate/prepare a GPA range with guest data 2553 2540 *

+1

include/linux/perf/riscv_pmu.h

reviewed

··· 89 89 struct riscv_pmu *riscv_pmu_alloc(void); 90 90 #ifdef CONFIG_RISCV_PMU_SBI 91 91 int riscv_pmu_get_hpm_info(u32 *hw_ctr_width, u32 *num_hw_ctr); 92 92 + int riscv_pmu_get_event_info(u32 type, u64 config, u64 *econfig); 92 93 #endif 93 94 94 95 #endif /* CONFIG_RISCV_PMU */

-35

include/trace/events/kvm.h

reviewed

··· 156 156 __entry->len, __entry->gpa, __entry->val) 157 157 ); 158 158 159 159 - #define KVM_TRACE_IOCSR_READ_UNSATISFIED 0 160 160 - #define KVM_TRACE_IOCSR_READ 1 161 161 - #define KVM_TRACE_IOCSR_WRITE 2 162 162 - 163 163 - #define kvm_trace_symbol_iocsr \ 164 164 - { KVM_TRACE_IOCSR_READ_UNSATISFIED, "unsatisfied-read" }, \ 165 165 - { KVM_TRACE_IOCSR_READ, "read" }, \ 166 166 - { KVM_TRACE_IOCSR_WRITE, "write" } 167 167 - 168 168 - TRACE_EVENT(kvm_iocsr, 169 169 - TP_PROTO(int type, int len, u64 gpa, void *val), 170 170 - TP_ARGS(type, len, gpa, val), 171 171 - 172 172 - TP_STRUCT__entry( 173 173 - __field( u32, type ) 174 174 - __field( u32, len ) 175 175 - __field( u64, gpa ) 176 176 - __field( u64, val ) 177 177 - ), 178 178 - 179 179 - TP_fast_assign( 180 180 - __entry->type = type; 181 181 - __entry->len = len; 182 182 - __entry->gpa = gpa; 183 183 - __entry->val = 0; 184 184 - if (val) 185 185 - memcpy(&__entry->val, val, 186 186 - min_t(u32, sizeof(__entry->val), len)); 187 187 - ), 188 188 - 189 189 - TP_printk("iocsr %s len %u gpa 0x%llx val 0x%llx", 190 190 - __print_symbolic(__entry->type, kvm_trace_symbol_iocsr), 191 191 - __entry->len, __entry->gpa, __entry->val) 192 192 - ); 193 193 - 194 159 #define kvm_fpu_load_symbol \ 195 160 {0, "unload"}, \ 196 161 {1, "load"}

+2

include/uapi/linux/kvm.h

reviewed

··· 962 962 #define KVM_CAP_ARM_EL2_E2H0 241 963 963 #define KVM_CAP_RISCV_MP_STATE_RESET 242 964 964 #define KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED 243 965 965 + #define KVM_CAP_GUEST_MEMFD_MMAP 244 965 966 966 967 struct kvm_irq_routing_irqchip { 967 968 __u32 irqchip; ··· 1599 1598 #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) 1600 1599 1601 1600 #define KVM_CREATE_GUEST_MEMFD _IOWR(KVMIO, 0xd4, struct kvm_create_guest_memfd) 1601 1601 + #define GUEST_MEMFD_FLAG_MMAP (1ULL << 0) 1602 1602 1603 1603 struct kvm_create_guest_memfd { 1604 1604 __u64 size;

+8

tools/testing/selftests/kvm/Makefile.kvm

reviewed

··· 156 156 TEST_GEN_PROGS_arm64 += arm64/aarch32_id_regs 157 157 TEST_GEN_PROGS_arm64 += arm64/arch_timer_edge_cases 158 158 TEST_GEN_PROGS_arm64 += arm64/debug-exceptions 159 159 + TEST_GEN_PROGS_arm64 += arm64/hello_el2 159 160 TEST_GEN_PROGS_arm64 += arm64/host_sve 160 161 TEST_GEN_PROGS_arm64 += arm64/hypercalls 161 162 TEST_GEN_PROGS_arm64 += arm64/external_aborts ··· 176 175 TEST_GEN_PROGS_arm64 += coalesced_io_test 177 176 TEST_GEN_PROGS_arm64 += dirty_log_perf_test 178 177 TEST_GEN_PROGS_arm64 += get-reg-list 178 178 + TEST_GEN_PROGS_arm64 += guest_memfd_test 179 179 TEST_GEN_PROGS_arm64 += memslot_modification_stress_test 180 180 TEST_GEN_PROGS_arm64 += memslot_perf_test 181 181 TEST_GEN_PROGS_arm64 += mmu_stress_test ··· 198 196 TEST_GEN_PROGS_riscv = $(TEST_GEN_PROGS_COMMON) 199 197 TEST_GEN_PROGS_riscv += riscv/sbi_pmu_test 200 198 TEST_GEN_PROGS_riscv += riscv/ebreak_test 199 199 + TEST_GEN_PROGS_riscv += access_tracking_perf_test 201 200 TEST_GEN_PROGS_riscv += arch_timer 202 201 TEST_GEN_PROGS_riscv += coalesced_io_test 202 202 + TEST_GEN_PROGS_riscv += dirty_log_perf_test 203 203 TEST_GEN_PROGS_riscv += get-reg-list 204 204 + TEST_GEN_PROGS_riscv += memslot_modification_stress_test 205 205 + TEST_GEN_PROGS_riscv += memslot_perf_test 206 206 + TEST_GEN_PROGS_riscv += mmu_stress_test 207 207 + TEST_GEN_PROGS_riscv += rseq_test 204 208 TEST_GEN_PROGS_riscv += steal_time 205 209 206 210 TEST_GEN_PROGS_loongarch += coalesced_io_test

+1

tools/testing/selftests/kvm/access_tracking_perf_test.c

reviewed

··· 50 50 #include "memstress.h" 51 51 #include "guest_modes.h" 52 52 #include "processor.h" 53 53 + #include "ucall_common.h" 53 54 54 55 #include "cgroup_util.h" 55 56 #include "lru_gen_util.h"

+4 -9

tools/testing/selftests/kvm/arm64/arch_timer.c

reviewed

··· 165 165 static void test_init_timer_irq(struct kvm_vm *vm) 166 166 { 167 167 /* Timer initid should be same for all the vCPUs, so query only vCPU-0 */ 168 168 - vcpu_device_attr_get(vcpus[0], KVM_ARM_VCPU_TIMER_CTRL, 169 169 - KVM_ARM_VCPU_TIMER_IRQ_PTIMER, &ptimer_irq); 170 170 - vcpu_device_attr_get(vcpus[0], KVM_ARM_VCPU_TIMER_CTRL, 171 171 - KVM_ARM_VCPU_TIMER_IRQ_VTIMER, &vtimer_irq); 168 168 + ptimer_irq = vcpu_get_ptimer_irq(vcpus[0]); 169 169 + vtimer_irq = vcpu_get_vtimer_irq(vcpus[0]); 172 170 173 171 sync_global_to_guest(vm, ptimer_irq); 174 172 sync_global_to_guest(vm, vtimer_irq); ··· 174 176 pr_debug("ptimer_irq: %d; vtimer_irq: %d\n", ptimer_irq, vtimer_irq); 175 177 } 176 178 177 177 - static int gic_fd; 178 178 - 179 179 struct kvm_vm *test_vm_create(void) 180 180 { 181 181 struct kvm_vm *vm; 182 182 unsigned int i; 183 183 int nr_vcpus = test_args.nr_vcpus; 184 184 + 185 185 + TEST_REQUIRE(kvm_supports_vgic_v3()); 184 186 185 187 vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus); 186 188 ··· 202 204 vcpu_init_descriptor_tables(vcpus[i]); 203 205 204 206 test_init_timer_irq(vm); 205 205 - gic_fd = vgic_v3_setup(vm, nr_vcpus, 64); 206 206 - __TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3"); 207 207 208 208 /* Make all the test's cmdline args visible to the guest */ 209 209 sync_global_to_guest(vm, test_args); ··· 211 215 212 216 void test_vm_cleanup(struct kvm_vm *vm) 213 217 { 214 214 - close(gic_fd); 215 218 kvm_vm_free(vm); 216 219 }

+4 -9

tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c

reviewed

··· 924 924 925 925 static void test_init_timer_irq(struct kvm_vm *vm, struct kvm_vcpu *vcpu) 926 926 { 927 927 - vcpu_device_attr_get(vcpu, KVM_ARM_VCPU_TIMER_CTRL, 928 928 - KVM_ARM_VCPU_TIMER_IRQ_PTIMER, &ptimer_irq); 929 929 - vcpu_device_attr_get(vcpu, KVM_ARM_VCPU_TIMER_CTRL, 930 930 - KVM_ARM_VCPU_TIMER_IRQ_VTIMER, &vtimer_irq); 927 927 + ptimer_irq = vcpu_get_ptimer_irq(vcpu); 928 928 + vtimer_irq = vcpu_get_vtimer_irq(vcpu); 931 929 932 930 sync_global_to_guest(vm, ptimer_irq); 933 931 sync_global_to_guest(vm, vtimer_irq); 934 932 935 933 pr_debug("ptimer_irq: %d; vtimer_irq: %d\n", ptimer_irq, vtimer_irq); 936 934 } 937 937 - 938 938 - static int gic_fd; 939 935 940 936 static void test_vm_create(struct kvm_vm **vm, struct kvm_vcpu **vcpu, 941 937 enum arch_timer timer) ··· 947 951 vcpu_args_set(*vcpu, 1, timer); 948 952 949 953 test_init_timer_irq(*vm, *vcpu); 950 950 - gic_fd = vgic_v3_setup(*vm, 1, 64); 951 951 - __TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3"); 952 954 953 955 sync_global_to_guest(*vm, test_args); 954 956 sync_global_to_guest(*vm, CVAL_MAX); ··· 955 961 956 962 static void test_vm_cleanup(struct kvm_vm *vm) 957 963 { 958 958 - close(gic_fd); 959 964 kvm_vm_free(vm); 960 965 } 961 966 ··· 1034 1041 1035 1042 /* Tell stdout not to buffer its content */ 1036 1043 setbuf(stdout, NULL); 1044 1044 + 1045 1045 + TEST_REQUIRE(kvm_supports_vgic_v3()); 1037 1046 1038 1047 if (!parse_args(argc, argv)) 1039 1048 exit(KSFT_SKIP);

+42

tools/testing/selftests/kvm/arm64/external_aborts.c

reviewed

··· 250 250 kvm_vm_free(vm); 251 251 } 252 252 253 253 + static void expect_sea_s1ptw_handler(struct ex_regs *regs) 254 254 + { 255 255 + u64 esr = read_sysreg(esr_el1); 256 256 + 257 257 + GUEST_ASSERT_EQ(regs->pc, expected_abort_pc); 258 258 + GUEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_DABT_CUR); 259 259 + GUEST_ASSERT_EQ((esr & ESR_ELx_FSC), ESR_ELx_FSC_SEA_TTW(3)); 260 260 + 261 261 + GUEST_DONE(); 262 262 + } 263 263 + 264 264 + static noinline void test_s1ptw_abort_guest(void) 265 265 + { 266 266 + extern char test_s1ptw_abort_insn; 267 267 + 268 268 + WRITE_ONCE(expected_abort_pc, (u64)&test_s1ptw_abort_insn); 269 269 + 270 270 + asm volatile("test_s1ptw_abort_insn:\n\t" 271 271 + "ldr x0, [%0]\n\t" 272 272 + : : "r" (MMIO_ADDR) : "x0", "memory"); 273 273 + 274 274 + GUEST_FAIL("Load on S1PTW abort should not retire"); 275 275 + } 276 276 + 277 277 + static void test_s1ptw_abort(void) 278 278 + { 279 279 + struct kvm_vcpu *vcpu; 280 280 + u64 *ptep, bad_pa; 281 281 + struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_s1ptw_abort_guest, 282 282 + expect_sea_s1ptw_handler); 283 283 + 284 284 + ptep = virt_get_pte_hva_at_level(vm, MMIO_ADDR, 2); 285 285 + bad_pa = BIT(vm->pa_bits) - vm->page_size; 286 286 + 287 287 + *ptep &= ~GENMASK(47, 12); 288 288 + *ptep |= bad_pa; 289 289 + 290 290 + vcpu_run_expect_done(vcpu); 291 291 + kvm_vm_free(vm); 292 292 + } 293 293 + 253 294 static void test_serror_emulated_guest(void) 254 295 { 255 296 GUEST_ASSERT(!(read_sysreg(isr_el1) & ISR_EL1_A)); ··· 368 327 test_serror_masked(); 369 328 test_serror_emulated(); 370 329 test_mmio_ease(); 330 330 + test_s1ptw_abort(); 371 331 }

+71

tools/testing/selftests/kvm/arm64/hello_el2.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * hello_el2 - Basic KVM selftest for VM running at EL2 with E2H=RES1 4 4 + * 5 5 + * Copyright 2025 Google LLC 6 6 + */ 7 7 + #include "kvm_util.h" 8 8 + #include "processor.h" 9 9 + #include "test_util.h" 10 10 + #include "ucall.h" 11 11 + 12 12 + #include <asm/sysreg.h> 13 13 + 14 14 + static void guest_code(void) 15 15 + { 16 16 + u64 mmfr0 = read_sysreg_s(SYS_ID_AA64MMFR0_EL1); 17 17 + u64 mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1); 18 18 + u64 mmfr4 = read_sysreg_s(SYS_ID_AA64MMFR4_EL1); 19 19 + u8 e2h0 = SYS_FIELD_GET(ID_AA64MMFR4_EL1, E2H0, mmfr4); 20 20 + 21 21 + GUEST_ASSERT_EQ(get_current_el(), 2); 22 22 + GUEST_ASSERT(read_sysreg(hcr_el2) & HCR_EL2_E2H); 23 23 + GUEST_ASSERT_EQ(SYS_FIELD_GET(ID_AA64MMFR1_EL1, VH, mmfr1), 24 24 + ID_AA64MMFR1_EL1_VH_IMP); 25 25 + 26 26 + /* 27 27 + * Traps of the complete ID register space are IMPDEF without FEAT_FGT, 28 28 + * which is really annoying to deal with in KVM describing E2H as RES1. 29 29 + * 30 30 + * If the implementation doesn't honor the trap then expect the register 31 31 + * to return all zeros. 32 32 + */ 33 33 + if (e2h0 == ID_AA64MMFR4_EL1_E2H0_IMP) 34 34 + GUEST_ASSERT_EQ(SYS_FIELD_GET(ID_AA64MMFR0_EL1, FGT, mmfr0), 35 35 + ID_AA64MMFR0_EL1_FGT_NI); 36 36 + else 37 37 + GUEST_ASSERT_EQ(e2h0, ID_AA64MMFR4_EL1_E2H0_NI_NV1); 38 38 + 39 39 + GUEST_DONE(); 40 40 + } 41 41 + 42 42 + int main(void) 43 43 + { 44 44 + struct kvm_vcpu_init init; 45 45 + struct kvm_vcpu *vcpu; 46 46 + struct kvm_vm *vm; 47 47 + struct ucall uc; 48 48 + 49 49 + TEST_REQUIRE(kvm_check_cap(KVM_CAP_ARM_EL2)); 50 50 + 51 51 + vm = vm_create(1); 52 52 + 53 53 + kvm_get_default_vcpu_target(vm, &init); 54 54 + init.features[0] |= BIT(KVM_ARM_VCPU_HAS_EL2); 55 55 + vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); 56 56 + kvm_arch_vm_finalize_vcpus(vm); 57 57 + 58 58 + vcpu_run(vcpu); 59 59 + switch (get_ucall(vcpu, &uc)) { 60 60 + case UCALL_DONE: 61 61 + break; 62 62 + case UCALL_ABORT: 63 63 + REPORT_GUEST_ASSERT(uc); 64 64 + break; 65 65 + default: 66 66 + TEST_FAIL("Unhandled ucall: %ld\n", uc.cmd); 67 67 + } 68 68 + 69 69 + kvm_vm_free(vm); 70 70 + return 0; 71 71 + }

+1 -1

tools/testing/selftests/kvm/arm64/hypercalls.c

reviewed

··· 108 108 109 109 for (i = 0; i < hvc_info_arr_sz; i++, hc_info++) { 110 110 memset(&res, 0, sizeof(res)); 111 111 - smccc_hvc(hc_info->func_id, hc_info->arg1, 0, 0, 0, 0, 0, 0, &res); 111 111 + do_smccc(hc_info->func_id, hc_info->arg1, 0, 0, 0, 0, 0, 0, &res); 112 112 113 113 switch (stage) { 114 114 case TEST_STAGE_HVC_IFACE_FEAT_DISABLED:

+1 -1

tools/testing/selftests/kvm/arm64/kvm-uuid.c

reviewed

··· 25 25 { 26 26 struct arm_smccc_res res = {}; 27 27 28 28 - smccc_hvc(ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID, 0, 0, 0, 0, 0, 0, 0, &res); 28 28 + do_smccc(ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID, 0, 0, 0, 0, 0, 0, 0, &res); 29 29 30 30 __GUEST_ASSERT(res.a0 == ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_0 && 31 31 res.a1 == ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_1 &&

+2

tools/testing/selftests/kvm/arm64/no-vgic-v3.c

reviewed

··· 163 163 struct kvm_vm *vm; 164 164 uint64_t pfr0; 165 165 166 166 + test_disable_default_vgic(); 167 167 + 166 168 vm = vm_create_with_one_vcpu(&vcpu, NULL); 167 169 pfr0 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR0_EL1)); 168 170 __TEST_REQUIRE(FIELD_GET(ID_AA64PFR0_EL1_GIC, pfr0),

+7 -6

tools/testing/selftests/kvm/arm64/psci_test.c

reviewed

··· 27 27 { 28 28 struct arm_smccc_res res; 29 29 30 30 - smccc_hvc(PSCI_0_2_FN64_CPU_ON, target_cpu, entry_addr, context_id, 30 30 + do_smccc(PSCI_0_2_FN64_CPU_ON, target_cpu, entry_addr, context_id, 31 31 0, 0, 0, 0, &res); 32 32 33 33 return res.a0; ··· 38 38 { 39 39 struct arm_smccc_res res; 40 40 41 41 - smccc_hvc(PSCI_0_2_FN64_AFFINITY_INFO, target_affinity, lowest_affinity_level, 41 41 + do_smccc(PSCI_0_2_FN64_AFFINITY_INFO, target_affinity, lowest_affinity_level, 42 42 0, 0, 0, 0, 0, &res); 43 43 44 44 return res.a0; ··· 48 48 { 49 49 struct arm_smccc_res res; 50 50 51 51 - smccc_hvc(PSCI_1_0_FN64_SYSTEM_SUSPEND, entry_addr, context_id, 51 51 + do_smccc(PSCI_1_0_FN64_SYSTEM_SUSPEND, entry_addr, context_id, 52 52 0, 0, 0, 0, 0, &res); 53 53 54 54 return res.a0; ··· 58 58 { 59 59 struct arm_smccc_res res; 60 60 61 61 - smccc_hvc(PSCI_1_3_FN64_SYSTEM_OFF2, type, cookie, 0, 0, 0, 0, 0, &res); 61 61 + do_smccc(PSCI_1_3_FN64_SYSTEM_OFF2, type, cookie, 0, 0, 0, 0, 0, &res); 62 62 63 63 return res.a0; 64 64 } ··· 67 67 { 68 68 struct arm_smccc_res res; 69 69 70 70 - smccc_hvc(PSCI_1_0_FN_PSCI_FEATURES, func_id, 0, 0, 0, 0, 0, 0, &res); 70 70 + do_smccc(PSCI_1_0_FN_PSCI_FEATURES, func_id, 0, 0, 0, 0, 0, 0, &res); 71 71 72 72 return res.a0; 73 73 } ··· 89 89 90 90 vm = vm_create(2); 91 91 92 92 - vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init); 92 92 + kvm_get_default_vcpu_target(vm, &init); 93 93 init.features[0] |= (1 << KVM_ARM_VCPU_PSCI_0_2); 94 94 95 95 *source = aarch64_vcpu_add(vm, 0, &init, guest_code); 96 96 *target = aarch64_vcpu_add(vm, 1, &init, guest_code); 97 97 98 98 + kvm_arch_vm_finalize_vcpus(vm); 98 99 return vm; 99 100 } 100 101

+22 -22

tools/testing/selftests/kvm/arm64/set_id_regs.c

reviewed

··· 15 15 #include "test_util.h" 16 16 #include <linux/bitfield.h> 17 17 18 18 - bool have_cap_arm_mte; 19 19 - 20 18 enum ftr_type { 21 19 FTR_EXACT, /* Use a predefined safe value */ 22 20 FTR_LOWER_SAFE, /* Smaller value is safe */ ··· 123 125 REG_FTR_END, 124 126 }; 125 127 128 128 + static const struct reg_ftr_bits ftr_id_aa64isar3_el1[] = { 129 129 + REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64ISAR3_EL1, FPRCVT, 0), 130 130 + REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64ISAR3_EL1, LSFE, 0), 131 131 + REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64ISAR3_EL1, FAMINMAX, 0), 132 132 + REG_FTR_END, 133 133 + }; 134 134 + 126 135 static const struct reg_ftr_bits ftr_id_aa64pfr0_el1[] = { 127 136 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, CSV3, 0), 128 137 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, CSV2, 0), ··· 170 165 static const struct reg_ftr_bits ftr_id_aa64mmfr1_el1[] = { 171 166 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR1_EL1, TIDCP1, 0), 172 167 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR1_EL1, AFP, 0), 168 168 + REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR1_EL1, HCX, 0), 173 169 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR1_EL1, ETS, 0), 170 170 + REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR1_EL1, TWED, 0), 174 171 REG_FTR_BITS(FTR_HIGHER_SAFE, ID_AA64MMFR1_EL1, SpecSEI, 0), 175 172 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR1_EL1, PAN, 0), 176 173 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR1_EL1, LO, 0), ··· 228 221 TEST_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0_el1), 229 222 TEST_REG(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1_el1), 230 223 TEST_REG(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2_el1), 224 224 + TEST_REG(SYS_ID_AA64ISAR3_EL1, ftr_id_aa64isar3_el1), 231 225 TEST_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0_el1), 232 226 TEST_REG(SYS_ID_AA64PFR1_EL1, ftr_id_aa64pfr1_el1), 233 227 TEST_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0_el1), ··· 247 239 GUEST_REG_SYNC(SYS_ID_AA64ISAR0_EL1); 248 240 GUEST_REG_SYNC(SYS_ID_AA64ISAR1_EL1); 249 241 GUEST_REG_SYNC(SYS_ID_AA64ISAR2_EL1); 242 242 + GUEST_REG_SYNC(SYS_ID_AA64ISAR3_EL1); 250 243 GUEST_REG_SYNC(SYS_ID_AA64PFR0_EL1); 251 244 GUEST_REG_SYNC(SYS_ID_AA64MMFR0_EL1); 252 245 GUEST_REG_SYNC(SYS_ID_AA64MMFR1_EL1); ··· 577 568 uint64_t mte_frac; 578 569 int idx, err; 579 570 580 580 - if (!have_cap_arm_mte) { 571 571 + val = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR1_EL1)); 572 572 + mte = FIELD_GET(ID_AA64PFR1_EL1_MTE, val); 573 573 + if (!mte) { 581 574 ksft_test_result_skip("MTE capability not supported, nothing to test\n"); 582 575 return; 583 576 } ··· 604 593 * from unsupported (0xF) to supported (0). 605 594 * 606 595 */ 607 607 - val = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR1_EL1)); 608 608 - 609 609 - mte = FIELD_GET(ID_AA64PFR1_EL1_MTE, val); 610 596 mte_frac = FIELD_GET(ID_AA64PFR1_EL1_MTE_frac, val); 611 597 if (mte != ID_AA64PFR1_EL1_MTE_MTE2 || 612 598 mte_frac != ID_AA64PFR1_EL1_MTE_frac_NI) { ··· 758 750 ksft_test_result_pass("%s\n", __func__); 759 751 } 760 752 761 761 - void kvm_arch_vm_post_create(struct kvm_vm *vm) 762 762 - { 763 763 - if (vm_check_cap(vm, KVM_CAP_ARM_MTE)) { 764 764 - vm_enable_cap(vm, KVM_CAP_ARM_MTE, 0); 765 765 - have_cap_arm_mte = true; 766 766 - } 767 767 - } 768 768 - 769 753 int main(void) 770 754 { 771 755 struct kvm_vcpu *vcpu; 772 756 struct kvm_vm *vm; 773 757 bool aarch64_only; 774 758 uint64_t val, el0; 775 775 - int test_cnt; 759 759 + int test_cnt, i, j; 776 760 777 761 TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES)); 778 762 TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_WRITABLE_IMP_ID_REGS)); 779 763 764 764 + test_wants_mte(); 765 765 + 780 766 vm = vm_create(1); 781 767 vm_enable_cap(vm, KVM_CAP_ARM_WRITABLE_IMP_ID_REGS, 0); 782 768 vcpu = vm_vcpu_add(vm, 0, guest_code); 769 769 + kvm_arch_vm_finalize_vcpus(vm); 783 770 784 771 /* Check for AARCH64 only system */ 785 772 val = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR0_EL1)); ··· 783 780 784 781 ksft_print_header(); 785 782 786 786 - test_cnt = ARRAY_SIZE(ftr_id_aa64dfr0_el1) + ARRAY_SIZE(ftr_id_dfr0_el1) + 787 787 - ARRAY_SIZE(ftr_id_aa64isar0_el1) + ARRAY_SIZE(ftr_id_aa64isar1_el1) + 788 788 - ARRAY_SIZE(ftr_id_aa64isar2_el1) + ARRAY_SIZE(ftr_id_aa64pfr0_el1) + 789 789 - ARRAY_SIZE(ftr_id_aa64pfr1_el1) + ARRAY_SIZE(ftr_id_aa64mmfr0_el1) + 790 790 - ARRAY_SIZE(ftr_id_aa64mmfr1_el1) + ARRAY_SIZE(ftr_id_aa64mmfr2_el1) + 791 791 - ARRAY_SIZE(ftr_id_aa64mmfr3_el1) + ARRAY_SIZE(ftr_id_aa64zfr0_el1) - 792 792 - ARRAY_SIZE(test_regs) + 3 + MPAM_IDREG_TEST + MTE_IDREG_TEST; 783 783 + test_cnt = 3 + MPAM_IDREG_TEST + MTE_IDREG_TEST; 784 784 + for (i = 0; i < ARRAY_SIZE(test_regs); i++) 785 785 + for (j = 0; test_regs[i].ftr_bits[j].type != FTR_END; j++) 786 786 + test_cnt++; 793 787 794 788 ksft_set_plan(test_cnt); 795 789

+15 -2

tools/testing/selftests/kvm/arm64/smccc_filter.c

reviewed

··· 22 22 SMC_INSN, 23 23 }; 24 24 25 25 + static bool test_runs_at_el2(void) 26 26 + { 27 27 + struct kvm_vm *vm = vm_create(1); 28 28 + struct kvm_vcpu_init init; 29 29 + 30 30 + kvm_get_default_vcpu_target(vm, &init); 31 31 + kvm_vm_free(vm); 32 32 + 33 33 + return init.features[0] & BIT(KVM_ARM_VCPU_HAS_EL2); 34 34 + } 35 35 + 25 36 #define for_each_conduit(conduit) \ 26 26 - for (conduit = HVC_INSN; conduit <= SMC_INSN; conduit++) 37 37 + for (conduit = test_runs_at_el2() ? SMC_INSN : HVC_INSN; \ 38 38 + conduit <= SMC_INSN; conduit++) 27 39 28 40 static void guest_main(uint32_t func_id, enum smccc_conduit conduit) 29 41 { ··· 76 64 struct kvm_vm *vm; 77 65 78 66 vm = vm_create(1); 79 79 - vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init); 67 67 + kvm_get_default_vcpu_target(vm, &init); 80 68 81 69 /* 82 70 * Enable in-kernel emulation of PSCI to ensure that calls are denied ··· 85 73 init.features[0] |= (1 << KVM_ARM_VCPU_PSCI_0_2); 86 74 87 75 *vcpu = aarch64_vcpu_add(vm, 0, &init, guest_main); 76 76 + kvm_arch_vm_finalize_vcpus(vm); 88 77 return vm; 89 78 } 90 79

+2

tools/testing/selftests/kvm/arm64/vgic_init.c

reviewed

··· 994 994 int pa_bits; 995 995 int cnt_impl = 0; 996 996 997 997 + test_disable_default_vgic(); 998 998 + 997 999 pa_bits = vm_guest_mode_params[VM_MODE_DEFAULT].pa_bits; 998 1000 max_phys_size = 1ULL << pa_bits; 999 1001

+3 -1

tools/testing/selftests/kvm/arm64/vgic_irq.c

reviewed

··· 752 752 vcpu_args_set(vcpu, 1, args_gva); 753 753 754 754 gic_fd = vgic_v3_setup(vm, 1, nr_irqs); 755 755 - __TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3, skipping"); 756 755 757 756 vm_install_exception_handler(vm, VECTOR_IRQ_CURRENT, 758 757 guest_irq_handlers[args.eoi_split][args.level_sensitive]); ··· 800 801 bool level_sensitive = false; 801 802 int opt; 802 803 bool eoi_split = false; 804 804 + 805 805 + TEST_REQUIRE(kvm_supports_vgic_v3()); 806 806 + test_disable_default_vgic(); 803 807 804 808 while ((opt = getopt(argc, argv, "hn:e:l:")) != -1) { 805 809 switch (opt) {

+3 -5

tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c

reviewed

··· 27 27 28 28 static struct kvm_vm *vm; 29 29 static struct kvm_vcpu **vcpus; 30 30 - static int gic_fd, its_fd; 30 30 + static int its_fd; 31 31 32 32 static struct test_data { 33 33 bool request_vcpus_stop; ··· 214 214 215 215 static void setup_gic(void) 216 216 { 217 217 - gic_fd = vgic_v3_setup(vm, test_data.nr_cpus, 64); 218 218 - __TEST_REQUIRE(gic_fd >= 0, "Failed to create GICv3"); 219 219 - 220 217 its_fd = vgic_its_setup(vm); 221 218 } 222 219 ··· 352 355 static void destroy_vm(void) 353 356 { 354 357 close(its_fd); 355 355 - close(gic_fd); 356 358 kvm_vm_free(vm); 357 359 free(vcpus); 358 360 } ··· 369 373 { 370 374 u32 nr_threads; 371 375 int c; 376 376 + 377 377 + TEST_REQUIRE(kvm_supports_vgic_v3()); 372 378 373 379 while ((c = getopt(argc, argv, "hv:d:e:i:")) != -1) { 374 380 switch (c) {

+35 -40

tools/testing/selftests/kvm/arm64/vpmu_counter_access.c

reviewed

··· 28 28 struct vpmu_vm { 29 29 struct kvm_vm *vm; 30 30 struct kvm_vcpu *vcpu; 31 31 - int gic_fd; 32 31 }; 33 32 34 33 static struct vpmu_vm vpmu_vm; ··· 42 43 static uint64_t get_pmcr_n(uint64_t pmcr) 43 44 { 44 45 return FIELD_GET(ARMV8_PMU_PMCR_N, pmcr); 45 45 - } 46 46 - 47 47 - static void set_pmcr_n(uint64_t *pmcr, uint64_t pmcr_n) 48 48 - { 49 49 - u64p_replace_bits((__u64 *) pmcr, pmcr_n, ARMV8_PMU_PMCR_N); 50 46 } 51 47 52 48 static uint64_t get_counters_mask(uint64_t n) ··· 409 415 .attr = KVM_ARM_VCPU_PMU_V3_IRQ, 410 416 .addr = (uint64_t)&irq, 411 417 }; 412 412 - struct kvm_device_attr init_attr = { 413 413 - .group = KVM_ARM_VCPU_PMU_V3_CTRL, 414 414 - .attr = KVM_ARM_VCPU_PMU_V3_INIT, 415 415 - }; 416 418 417 419 /* The test creates the vpmu_vm multiple times. Ensure a clean state */ 418 420 memset(&vpmu_vm, 0, sizeof(vpmu_vm)); ··· 421 431 } 422 432 423 433 /* Create vCPU with PMUv3 */ 424 424 - vm_ioctl(vpmu_vm.vm, KVM_ARM_PREFERRED_TARGET, &init); 434 434 + kvm_get_default_vcpu_target(vpmu_vm.vm, &init); 425 435 init.features[0] |= (1 << KVM_ARM_VCPU_PMU_V3); 426 436 vpmu_vm.vcpu = aarch64_vcpu_add(vpmu_vm.vm, 0, &init, guest_code); 427 437 vcpu_init_descriptor_tables(vpmu_vm.vcpu); 428 428 - vpmu_vm.gic_fd = vgic_v3_setup(vpmu_vm.vm, 1, 64); 429 429 - __TEST_REQUIRE(vpmu_vm.gic_fd >= 0, 430 430 - "Failed to create vgic-v3, skipping"); 438 438 + 439 439 + kvm_arch_vm_finalize_vcpus(vpmu_vm.vm); 431 440 432 441 /* Make sure that PMUv3 support is indicated in the ID register */ 433 442 dfr0 = vcpu_get_reg(vpmu_vm.vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64DFR0_EL1)); ··· 435 446 pmuver >= ID_AA64DFR0_EL1_PMUVer_IMP, 436 447 "Unexpected PMUVER (0x%x) on the vCPU with PMUv3", pmuver); 437 448 438 438 - /* Initialize vPMU */ 439 449 vcpu_ioctl(vpmu_vm.vcpu, KVM_SET_DEVICE_ATTR, &irq_attr); 440 440 - vcpu_ioctl(vpmu_vm.vcpu, KVM_SET_DEVICE_ATTR, &init_attr); 441 450 } 442 451 443 452 static void destroy_vpmu_vm(void) 444 453 { 445 445 - close(vpmu_vm.gic_fd); 446 454 kvm_vm_free(vpmu_vm.vm); 447 455 } 448 456 ··· 461 475 } 462 476 } 463 477 464 464 - static void test_create_vpmu_vm_with_pmcr_n(uint64_t pmcr_n, bool expect_fail) 478 478 + static void test_create_vpmu_vm_with_nr_counters(unsigned int nr_counters, bool expect_fail) 465 479 { 466 480 struct kvm_vcpu *vcpu; 467 467 - uint64_t pmcr, pmcr_orig; 481 481 + unsigned int prev; 482 482 + int ret; 468 483 469 484 create_vpmu_vm(guest_code); 470 485 vcpu = vpmu_vm.vcpu; 471 486 472 472 - pmcr_orig = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_PMCR_EL0)); 473 473 - pmcr = pmcr_orig; 487 487 + prev = get_pmcr_n(vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_PMCR_EL0))); 474 488 475 475 - /* 476 476 - * Setting a larger value of PMCR.N should not modify the field, and 477 477 - * return a success. 478 478 - */ 479 479 - set_pmcr_n(&pmcr, pmcr_n); 480 480 - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_PMCR_EL0), pmcr); 481 481 - pmcr = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_PMCR_EL0)); 489 489 + ret = __vcpu_device_attr_set(vcpu, KVM_ARM_VCPU_PMU_V3_CTRL, 490 490 + KVM_ARM_VCPU_PMU_V3_SET_NR_COUNTERS, &nr_counters); 482 491 483 492 if (expect_fail) 484 484 - TEST_ASSERT(pmcr_orig == pmcr, 485 485 - "PMCR.N modified by KVM to a larger value (PMCR: 0x%lx) for pmcr_n: 0x%lx", 486 486 - pmcr, pmcr_n); 493 493 + TEST_ASSERT(ret && errno == EINVAL, 494 494 + "Setting more PMU counters (%u) than available (%u) unexpectedly succeeded", 495 495 + nr_counters, prev); 487 496 else 488 488 - TEST_ASSERT(pmcr_n == get_pmcr_n(pmcr), 489 489 - "Failed to update PMCR.N to %lu (received: %lu)", 490 490 - pmcr_n, get_pmcr_n(pmcr)); 497 497 + TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret)); 498 498 + 499 499 + vcpu_device_attr_set(vcpu, KVM_ARM_VCPU_PMU_V3_CTRL, KVM_ARM_VCPU_PMU_V3_INIT, NULL); 491 500 } 492 501 493 502 /* ··· 497 516 498 517 pr_debug("Test with pmcr_n %lu\n", pmcr_n); 499 518 500 500 - test_create_vpmu_vm_with_pmcr_n(pmcr_n, false); 519 519 + test_create_vpmu_vm_with_nr_counters(pmcr_n, false); 501 520 vcpu = vpmu_vm.vcpu; 502 521 503 522 /* Save the initial sp to restore them later to run the guest again */ 504 504 - sp = vcpu_get_reg(vcpu, ARM64_CORE_REG(sp_el1)); 523 523 + sp = vcpu_get_reg(vcpu, ctxt_reg_alias(vcpu, SYS_SP_EL1)); 505 524 506 525 run_vcpu(vcpu, pmcr_n); 507 526 ··· 509 528 * Reset and re-initialize the vCPU, and run the guest code again to 510 529 * check if PMCR_EL0.N is preserved. 511 530 */ 512 512 - vm_ioctl(vpmu_vm.vm, KVM_ARM_PREFERRED_TARGET, &init); 531 531 + kvm_get_default_vcpu_target(vpmu_vm.vm, &init); 513 532 init.features[0] |= (1 << KVM_ARM_VCPU_PMU_V3); 514 533 aarch64_vcpu_setup(vcpu, &init); 515 534 vcpu_init_descriptor_tables(vcpu); 516 516 - vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), sp); 535 535 + vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_SP_EL1), sp); 517 536 vcpu_set_reg(vcpu, ARM64_CORE_REG(regs.pc), (uint64_t)guest_code); 518 537 519 538 run_vcpu(vcpu, pmcr_n); ··· 538 557 uint64_t set_reg_id, clr_reg_id, reg_val; 539 558 uint64_t valid_counters_mask, max_counters_mask; 540 559 541 541 - test_create_vpmu_vm_with_pmcr_n(pmcr_n, false); 560 560 + test_create_vpmu_vm_with_nr_counters(pmcr_n, false); 542 561 vcpu = vpmu_vm.vcpu; 543 562 544 563 valid_counters_mask = get_counters_mask(pmcr_n); ··· 592 611 { 593 612 pr_debug("Error test with pmcr_n %lu (larger than the host)\n", pmcr_n); 594 613 595 595 - test_create_vpmu_vm_with_pmcr_n(pmcr_n, true); 614 614 + test_create_vpmu_vm_with_nr_counters(pmcr_n, true); 596 615 destroy_vpmu_vm(); 597 616 } 598 617 ··· 610 629 return get_pmcr_n(pmcr); 611 630 } 612 631 632 632 + static bool kvm_supports_nr_counters_attr(void) 633 633 + { 634 634 + bool supported; 635 635 + 636 636 + create_vpmu_vm(NULL); 637 637 + supported = !__vcpu_has_device_attr(vpmu_vm.vcpu, KVM_ARM_VCPU_PMU_V3_CTRL, 638 638 + KVM_ARM_VCPU_PMU_V3_SET_NR_COUNTERS); 639 639 + destroy_vpmu_vm(); 640 640 + 641 641 + return supported; 642 642 + } 643 643 + 613 644 int main(void) 614 645 { 615 646 uint64_t i, pmcr_n; 616 647 617 648 TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_PMU_V3)); 649 649 + TEST_REQUIRE(kvm_supports_vgic_v3()); 650 650 + TEST_REQUIRE(kvm_supports_nr_counters_attr()); 618 651 619 652 pmcr_n = get_pmcr_n_limit(); 620 653 for (i = 0; i <= pmcr_n; i++) {

-35

tools/testing/selftests/kvm/dirty_log_perf_test.c

reviewed

··· 20 20 #include "guest_modes.h" 21 21 #include "ucall_common.h" 22 22 23 23 - #ifdef __aarch64__ 24 24 - #include "arm64/vgic.h" 25 25 - 26 26 - static int gic_fd; 27 27 - 28 28 - static void arch_setup_vm(struct kvm_vm *vm, unsigned int nr_vcpus) 29 29 - { 30 30 - /* 31 31 - * The test can still run even if hardware does not support GICv3, as it 32 32 - * is only an optimization to reduce guest exits. 33 33 - */ 34 34 - gic_fd = vgic_v3_setup(vm, nr_vcpus, 64); 35 35 - } 36 36 - 37 37 - static void arch_cleanup_vm(struct kvm_vm *vm) 38 38 - { 39 39 - if (gic_fd > 0) 40 40 - close(gic_fd); 41 41 - } 42 42 - 43 43 - #else /* __aarch64__ */ 44 44 - 45 45 - static void arch_setup_vm(struct kvm_vm *vm, unsigned int nr_vcpus) 46 46 - { 47 47 - } 48 48 - 49 49 - static void arch_cleanup_vm(struct kvm_vm *vm) 50 50 - { 51 51 - } 52 52 - 53 53 - #endif 54 54 - 55 23 /* How many host loops to run by default (one KVM_GET_DIRTY_LOG for each loop)*/ 56 24 #define TEST_HOST_LOOP_N 2UL 57 25 ··· 133 165 if (dirty_log_manual_caps) 134 166 vm_enable_cap(vm, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2, 135 167 dirty_log_manual_caps); 136 136 - 137 137 - arch_setup_vm(vm, nr_vcpus); 138 168 139 169 /* Start the iterations */ 140 170 iteration = 0; ··· 251 285 } 252 286 253 287 memstress_free_bitmaps(bitmaps, p->slots); 254 254 - arch_cleanup_vm(vm); 255 288 memstress_destroy_vm(vm); 256 289 } 257 290

+1

tools/testing/selftests/kvm/dirty_log_test.c

reviewed

··· 585 585 586 586 log_mode_create_vm_done(vm); 587 587 *vcpu = vm_vcpu_add(vm, 0, guest_code); 588 588 + kvm_arch_vm_finalize_vcpus(vm); 588 589 return vm; 589 590 } 590 591

+6 -3

tools/testing/selftests/kvm/get-reg-list.c

reviewed

··· 116 116 } 117 117 118 118 #ifdef __aarch64__ 119 119 - static void prepare_vcpu_init(struct vcpu_reg_list *c, struct kvm_vcpu_init *init) 119 119 + static void prepare_vcpu_init(struct kvm_vm *vm, struct vcpu_reg_list *c, 120 120 + struct kvm_vcpu_init *init) 120 121 { 121 122 struct vcpu_reg_sublist *s; 123 123 + 124 124 + vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, init); 122 125 123 126 for_each_sublist(c, s) 124 127 if (s->capability) ··· 130 127 131 128 static struct kvm_vcpu *vcpu_config_get_vcpu(struct vcpu_reg_list *c, struct kvm_vm *vm) 132 129 { 133 133 - struct kvm_vcpu_init init = { .target = -1, }; 130 130 + struct kvm_vcpu_init init; 134 131 struct kvm_vcpu *vcpu; 135 132 136 136 - prepare_vcpu_init(c, &init); 133 133 + prepare_vcpu_init(vm, c, &init); 137 134 vcpu = __vm_vcpu_add(vm, 0); 138 135 aarch64_vcpu_setup(vcpu, &init); 139 136

+209 -27

tools/testing/selftests/kvm/guest_memfd_test.c

reviewed

··· 13 13 14 14 #include <linux/bitmap.h> 15 15 #include <linux/falloc.h> 16 16 + #include <linux/sizes.h> 17 17 + #include <setjmp.h> 18 18 + #include <signal.h> 16 19 #include <sys/mman.h> 17 20 #include <sys/types.h> 18 21 #include <sys/stat.h> 19 22 20 23 #include "kvm_util.h" 21 24 #include "test_util.h" 25 25 + #include "ucall_common.h" 22 26 23 27 static void test_file_read_write(int fd) 24 28 { ··· 38 34 "pwrite on a guest_mem fd should fail"); 39 35 } 40 36 41 41 - static void test_mmap(int fd, size_t page_size) 37 37 + static void test_mmap_supported(int fd, size_t page_size, size_t total_size) 38 38 + { 39 39 + const char val = 0xaa; 40 40 + char *mem; 41 41 + size_t i; 42 42 + int ret; 43 43 + 44 44 + mem = mmap(NULL, total_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 45 45 + TEST_ASSERT(mem == MAP_FAILED, "Copy-on-write not allowed by guest_memfd."); 46 46 + 47 47 + mem = mmap(NULL, total_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 48 48 + TEST_ASSERT(mem != MAP_FAILED, "mmap() for guest_memfd should succeed."); 49 49 + 50 50 + memset(mem, val, total_size); 51 51 + for (i = 0; i < total_size; i++) 52 52 + TEST_ASSERT_EQ(READ_ONCE(mem[i]), val); 53 53 + 54 54 + ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 55 55 + page_size); 56 56 + TEST_ASSERT(!ret, "fallocate the first page should succeed."); 57 57 + 58 58 + for (i = 0; i < page_size; i++) 59 59 + TEST_ASSERT_EQ(READ_ONCE(mem[i]), 0x00); 60 60 + for (; i < total_size; i++) 61 61 + TEST_ASSERT_EQ(READ_ONCE(mem[i]), val); 62 62 + 63 63 + memset(mem, val, page_size); 64 64 + for (i = 0; i < total_size; i++) 65 65 + TEST_ASSERT_EQ(READ_ONCE(mem[i]), val); 66 66 + 67 67 + ret = munmap(mem, total_size); 68 68 + TEST_ASSERT(!ret, "munmap() should succeed."); 69 69 + } 70 70 + 71 71 + static sigjmp_buf jmpbuf; 72 72 + void fault_sigbus_handler(int signum) 73 73 + { 74 74 + siglongjmp(jmpbuf, 1); 75 75 + } 76 76 + 77 77 + static void test_fault_overflow(int fd, size_t page_size, size_t total_size) 78 78 + { 79 79 + struct sigaction sa_old, sa_new = { 80 80 + .sa_handler = fault_sigbus_handler, 81 81 + }; 82 82 + size_t map_size = total_size * 4; 83 83 + const char val = 0xaa; 84 84 + char *mem; 85 85 + size_t i; 86 86 + int ret; 87 87 + 88 88 + mem = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 89 89 + TEST_ASSERT(mem != MAP_FAILED, "mmap() for guest_memfd should succeed."); 90 90 + 91 91 + sigaction(SIGBUS, &sa_new, &sa_old); 92 92 + if (sigsetjmp(jmpbuf, 1) == 0) { 93 93 + memset(mem, 0xaa, map_size); 94 94 + TEST_ASSERT(false, "memset() should have triggered SIGBUS."); 95 95 + } 96 96 + sigaction(SIGBUS, &sa_old, NULL); 97 97 + 98 98 + for (i = 0; i < total_size; i++) 99 99 + TEST_ASSERT_EQ(READ_ONCE(mem[i]), val); 100 100 + 101 101 + ret = munmap(mem, map_size); 102 102 + TEST_ASSERT(!ret, "munmap() should succeed."); 103 103 + } 104 104 + 105 105 + static void test_mmap_not_supported(int fd, size_t page_size, size_t total_size) 42 106 { 43 107 char *mem; 44 108 45 109 mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 110 110 + TEST_ASSERT_EQ(mem, MAP_FAILED); 111 111 + 112 112 + mem = mmap(NULL, total_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 46 113 TEST_ASSERT_EQ(mem, MAP_FAILED); 47 114 } 48 115 ··· 195 120 } 196 121 } 197 122 198 198 - static void test_create_guest_memfd_invalid(struct kvm_vm *vm) 123 123 + static void test_create_guest_memfd_invalid_sizes(struct kvm_vm *vm, 124 124 + uint64_t guest_memfd_flags, 125 125 + size_t page_size) 199 126 { 200 200 - size_t page_size = getpagesize(); 201 201 - uint64_t flag; 202 127 size_t size; 203 128 int fd; 204 129 205 130 for (size = 1; size < page_size; size++) { 206 206 - fd = __vm_create_guest_memfd(vm, size, 0); 207 207 - TEST_ASSERT(fd == -1 && errno == EINVAL, 131 131 + fd = __vm_create_guest_memfd(vm, size, guest_memfd_flags); 132 132 + TEST_ASSERT(fd < 0 && errno == EINVAL, 208 133 "guest_memfd() with non-page-aligned page size '0x%lx' should fail with EINVAL", 209 134 size); 210 210 - } 211 211 - 212 212 - for (flag = BIT(0); flag; flag <<= 1) { 213 213 - fd = __vm_create_guest_memfd(vm, page_size, flag); 214 214 - TEST_ASSERT(fd == -1 && errno == EINVAL, 215 215 - "guest_memfd() with flag '0x%lx' should fail with EINVAL", 216 216 - flag); 217 135 } 218 136 } 219 137 ··· 214 146 { 215 147 int fd1, fd2, ret; 216 148 struct stat st1, st2; 149 149 + size_t page_size = getpagesize(); 217 150 218 218 - fd1 = __vm_create_guest_memfd(vm, 4096, 0); 151 151 + fd1 = __vm_create_guest_memfd(vm, page_size, 0); 219 152 TEST_ASSERT(fd1 != -1, "memfd creation should succeed"); 220 153 221 154 ret = fstat(fd1, &st1); 222 155 TEST_ASSERT(ret != -1, "memfd fstat should succeed"); 223 223 - TEST_ASSERT(st1.st_size == 4096, "memfd st_size should match requested size"); 156 156 + TEST_ASSERT(st1.st_size == page_size, "memfd st_size should match requested size"); 224 157 225 225 - fd2 = __vm_create_guest_memfd(vm, 8192, 0); 158 158 + fd2 = __vm_create_guest_memfd(vm, page_size * 2, 0); 226 159 TEST_ASSERT(fd2 != -1, "memfd creation should succeed"); 227 160 228 161 ret = fstat(fd2, &st2); 229 162 TEST_ASSERT(ret != -1, "memfd fstat should succeed"); 230 230 - TEST_ASSERT(st2.st_size == 8192, "second memfd st_size should match requested size"); 163 163 + TEST_ASSERT(st2.st_size == page_size * 2, "second memfd st_size should match requested size"); 231 164 232 165 ret = fstat(fd1, &st1); 233 166 TEST_ASSERT(ret != -1, "memfd fstat should succeed"); 234 234 - TEST_ASSERT(st1.st_size == 4096, "first memfd st_size should still match requested size"); 167 167 + TEST_ASSERT(st1.st_size == page_size, "first memfd st_size should still match requested size"); 235 168 TEST_ASSERT(st1.st_ino != st2.st_ino, "different memfd should have different inode numbers"); 236 169 237 170 close(fd2); 238 171 close(fd1); 239 172 } 240 173 241 241 - int main(int argc, char *argv[]) 174 174 + static void test_guest_memfd_flags(struct kvm_vm *vm, uint64_t valid_flags) 242 175 { 243 243 - size_t page_size; 244 244 - size_t total_size; 176 176 + size_t page_size = getpagesize(); 177 177 + uint64_t flag; 245 178 int fd; 246 246 - struct kvm_vm *vm; 247 179 248 248 - TEST_REQUIRE(kvm_has_cap(KVM_CAP_GUEST_MEMFD)); 180 180 + for (flag = BIT(0); flag; flag <<= 1) { 181 181 + fd = __vm_create_guest_memfd(vm, page_size, flag); 182 182 + if (flag & valid_flags) { 183 183 + TEST_ASSERT(fd >= 0, 184 184 + "guest_memfd() with flag '0x%lx' should succeed", 185 185 + flag); 186 186 + close(fd); 187 187 + } else { 188 188 + TEST_ASSERT(fd < 0 && errno == EINVAL, 189 189 + "guest_memfd() with flag '0x%lx' should fail with EINVAL", 190 190 + flag); 191 191 + } 192 192 + } 193 193 + } 194 194 + 195 195 + static void test_guest_memfd(unsigned long vm_type) 196 196 + { 197 197 + uint64_t flags = 0; 198 198 + struct kvm_vm *vm; 199 199 + size_t total_size; 200 200 + size_t page_size; 201 201 + int fd; 249 202 250 203 page_size = getpagesize(); 251 204 total_size = page_size * 4; 252 205 253 253 - vm = vm_create_barebones(); 206 206 + vm = vm_create_barebones_type(vm_type); 254 207 255 255 - test_create_guest_memfd_invalid(vm); 208 208 + if (vm_check_cap(vm, KVM_CAP_GUEST_MEMFD_MMAP)) 209 209 + flags |= GUEST_MEMFD_FLAG_MMAP; 210 210 + 256 211 test_create_guest_memfd_multiple(vm); 212 212 + test_create_guest_memfd_invalid_sizes(vm, flags, page_size); 257 213 258 258 - fd = vm_create_guest_memfd(vm, total_size, 0); 214 214 + fd = vm_create_guest_memfd(vm, total_size, flags); 259 215 260 216 test_file_read_write(fd); 261 261 - test_mmap(fd, page_size); 217 217 + 218 218 + if (flags & GUEST_MEMFD_FLAG_MMAP) { 219 219 + test_mmap_supported(fd, page_size, total_size); 220 220 + test_fault_overflow(fd, page_size, total_size); 221 221 + } else { 222 222 + test_mmap_not_supported(fd, page_size, total_size); 223 223 + } 224 224 + 262 225 test_file_size(fd, page_size, total_size); 263 226 test_fallocate(fd, page_size, total_size); 264 227 test_invalid_punch_hole(fd, page_size, total_size); 265 228 229 229 + test_guest_memfd_flags(vm, flags); 230 230 + 266 231 close(fd); 232 232 + kvm_vm_free(vm); 233 233 + } 234 234 + 235 235 + static void guest_code(uint8_t *mem, uint64_t size) 236 236 + { 237 237 + size_t i; 238 238 + 239 239 + for (i = 0; i < size; i++) 240 240 + __GUEST_ASSERT(mem[i] == 0xaa, 241 241 + "Guest expected 0xaa at offset %lu, got 0x%x", i, mem[i]); 242 242 + 243 243 + memset(mem, 0xff, size); 244 244 + GUEST_DONE(); 245 245 + } 246 246 + 247 247 + static void test_guest_memfd_guest(void) 248 248 + { 249 249 + /* 250 250 + * Skip the first 4gb and slot0. slot0 maps <1gb and is used to back 251 251 + * the guest's code, stack, and page tables, and low memory contains 252 252 + * the PCI hole and other MMIO regions that need to be avoided. 253 253 + */ 254 254 + const uint64_t gpa = SZ_4G; 255 255 + const int slot = 1; 256 256 + 257 257 + struct kvm_vcpu *vcpu; 258 258 + struct kvm_vm *vm; 259 259 + uint8_t *mem; 260 260 + size_t size; 261 261 + int fd, i; 262 262 + 263 263 + if (!kvm_has_cap(KVM_CAP_GUEST_MEMFD_MMAP)) 264 264 + return; 265 265 + 266 266 + vm = __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, &vcpu, 1, guest_code); 267 267 + 268 268 + TEST_ASSERT(vm_check_cap(vm, KVM_CAP_GUEST_MEMFD_MMAP), 269 269 + "Default VM type should always support guest_memfd mmap()"); 270 270 + 271 271 + size = vm->page_size; 272 272 + fd = vm_create_guest_memfd(vm, size, GUEST_MEMFD_FLAG_MMAP); 273 273 + vm_set_user_memory_region2(vm, slot, KVM_MEM_GUEST_MEMFD, gpa, size, NULL, fd, 0); 274 274 + 275 275 + mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 276 276 + TEST_ASSERT(mem != MAP_FAILED, "mmap() on guest_memfd failed"); 277 277 + memset(mem, 0xaa, size); 278 278 + munmap(mem, size); 279 279 + 280 280 + virt_pg_map(vm, gpa, gpa); 281 281 + vcpu_args_set(vcpu, 2, gpa, size); 282 282 + vcpu_run(vcpu); 283 283 + 284 284 + TEST_ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_DONE); 285 285 + 286 286 + mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 287 287 + TEST_ASSERT(mem != MAP_FAILED, "mmap() on guest_memfd failed"); 288 288 + for (i = 0; i < size; i++) 289 289 + TEST_ASSERT_EQ(mem[i], 0xff); 290 290 + 291 291 + close(fd); 292 292 + kvm_vm_free(vm); 293 293 + } 294 294 + 295 295 + int main(int argc, char *argv[]) 296 296 + { 297 297 + unsigned long vm_types, vm_type; 298 298 + 299 299 + TEST_REQUIRE(kvm_has_cap(KVM_CAP_GUEST_MEMFD)); 300 300 + 301 301 + /* 302 302 + * Not all architectures support KVM_CAP_VM_TYPES. However, those that 303 303 + * support guest_memfd have that support for the default VM type. 304 304 + */ 305 305 + vm_types = kvm_check_cap(KVM_CAP_VM_TYPES); 306 306 + if (!vm_types) 307 307 + vm_types = BIT(VM_TYPE_DEFAULT); 308 308 + 309 309 + for_each_set_bit(vm_type, &vm_types, BITS_PER_TYPE(vm_types)) 310 310 + test_guest_memfd(vm_type); 311 311 + 312 312 + test_guest_memfd_guest(); 267 313 }

+24

tools/testing/selftests/kvm/include/arm64/arch_timer.h

reviewed

··· 155 155 timer_set_tval(timer, msec_to_cycles(msec)); 156 156 } 157 157 158 158 + static inline u32 vcpu_get_vtimer_irq(struct kvm_vcpu *vcpu) 159 159 + { 160 160 + u32 intid; 161 161 + u64 attr; 162 162 + 163 163 + attr = vcpu_has_el2(vcpu) ? KVM_ARM_VCPU_TIMER_IRQ_HVTIMER : 164 164 + KVM_ARM_VCPU_TIMER_IRQ_VTIMER; 165 165 + vcpu_device_attr_get(vcpu, KVM_ARM_VCPU_TIMER_CTRL, attr, &intid); 166 166 + 167 167 + return intid; 168 168 + } 169 169 + 170 170 + static inline u32 vcpu_get_ptimer_irq(struct kvm_vcpu *vcpu) 171 171 + { 172 172 + u32 intid; 173 173 + u64 attr; 174 174 + 175 175 + attr = vcpu_has_el2(vcpu) ? KVM_ARM_VCPU_TIMER_IRQ_HPTIMER : 176 176 + KVM_ARM_VCPU_TIMER_IRQ_PTIMER; 177 177 + vcpu_device_attr_get(vcpu, KVM_ARM_VCPU_TIMER_CTRL, attr, &intid); 178 178 + 179 179 + return intid; 180 180 + } 181 181 + 158 182 #endif /* SELFTEST_KVM_ARCH_TIMER_H */

+4 -1

tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h

reviewed

··· 2 2 #ifndef SELFTEST_KVM_UTIL_ARCH_H 3 3 #define SELFTEST_KVM_UTIL_ARCH_H 4 4 5 5 - struct kvm_vm_arch {}; 5 5 + struct kvm_vm_arch { 6 6 + bool has_gic; 7 7 + int gic_fd; 8 8 + }; 6 9 7 10 #endif // SELFTEST_KVM_UTIL_ARCH_H

+74

tools/testing/selftests/kvm/include/arm64/processor.h

reviewed

··· 175 175 void vm_install_sync_handler(struct kvm_vm *vm, 176 176 int vector, int ec, handler_fn handler); 177 177 178 178 + uint64_t *virt_get_pte_hva_at_level(struct kvm_vm *vm, vm_vaddr_t gva, int level); 178 179 uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva); 179 180 180 181 static inline void cpu_relax(void) ··· 300 299 301 300 /* Execute a Wait For Interrupt instruction. */ 302 301 void wfi(void); 302 302 + 303 303 + void test_wants_mte(void); 304 304 + void test_disable_default_vgic(void); 305 305 + 306 306 + bool vm_supports_el2(struct kvm_vm *vm); 307 307 + static bool vcpu_has_el2(struct kvm_vcpu *vcpu) 308 308 + { 309 309 + return vcpu->init.features[0] & BIT(KVM_ARM_VCPU_HAS_EL2); 310 310 + } 311 311 + 312 312 + #define MAPPED_EL2_SYSREG(el2, el1) \ 313 313 + case SYS_##el1: \ 314 314 + if (vcpu_has_el2(vcpu)) \ 315 315 + alias = SYS_##el2; \ 316 316 + break 317 317 + 318 318 + 319 319 + static __always_inline u64 ctxt_reg_alias(struct kvm_vcpu *vcpu, u32 encoding) 320 320 + { 321 321 + u32 alias = encoding; 322 322 + 323 323 + BUILD_BUG_ON(!__builtin_constant_p(encoding)); 324 324 + 325 325 + switch (encoding) { 326 326 + MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1); 327 327 + MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1); 328 328 + MAPPED_EL2_SYSREG(TTBR0_EL2, TTBR0_EL1); 329 329 + MAPPED_EL2_SYSREG(TTBR1_EL2, TTBR1_EL1); 330 330 + MAPPED_EL2_SYSREG(TCR_EL2, TCR_EL1); 331 331 + MAPPED_EL2_SYSREG(VBAR_EL2, VBAR_EL1); 332 332 + MAPPED_EL2_SYSREG(AFSR0_EL2, AFSR0_EL1); 333 333 + MAPPED_EL2_SYSREG(AFSR1_EL2, AFSR1_EL1); 334 334 + MAPPED_EL2_SYSREG(ESR_EL2, ESR_EL1); 335 335 + MAPPED_EL2_SYSREG(FAR_EL2, FAR_EL1); 336 336 + MAPPED_EL2_SYSREG(MAIR_EL2, MAIR_EL1); 337 337 + MAPPED_EL2_SYSREG(TCR2_EL2, TCR2_EL1); 338 338 + MAPPED_EL2_SYSREG(PIR_EL2, PIR_EL1); 339 339 + MAPPED_EL2_SYSREG(PIRE0_EL2, PIRE0_EL1); 340 340 + MAPPED_EL2_SYSREG(POR_EL2, POR_EL1); 341 341 + MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1); 342 342 + MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1); 343 343 + MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1); 344 344 + MAPPED_EL2_SYSREG(ZCR_EL2, ZCR_EL1); 345 345 + MAPPED_EL2_SYSREG(CONTEXTIDR_EL2, CONTEXTIDR_EL1); 346 346 + MAPPED_EL2_SYSREG(SCTLR2_EL2, SCTLR2_EL1); 347 347 + MAPPED_EL2_SYSREG(CNTHCTL_EL2, CNTKCTL_EL1); 348 348 + case SYS_SP_EL1: 349 349 + if (!vcpu_has_el2(vcpu)) 350 350 + return ARM64_CORE_REG(sp_el1); 351 351 + 352 352 + alias = SYS_SP_EL2; 353 353 + break; 354 354 + default: 355 355 + BUILD_BUG(); 356 356 + } 357 357 + 358 358 + return KVM_ARM64_SYS_REG(alias); 359 359 + } 360 360 + 361 361 + void kvm_get_default_vcpu_target(struct kvm_vm *vm, struct kvm_vcpu_init *init); 362 362 + 363 363 + static inline unsigned int get_current_el(void) 364 364 + { 365 365 + return (read_sysreg(CurrentEL) >> 2) & 0x3; 366 366 + } 367 367 + 368 368 + #define do_smccc(...) \ 369 369 + do { \ 370 370 + if (get_current_el() == 2) \ 371 371 + smccc_smc(__VA_ARGS__); \ 372 372 + else \ 373 373 + smccc_hvc(__VA_ARGS__); \ 374 374 + } while (0) 303 375 304 376 #endif /* SELFTEST_KVM_PROCESSOR_H */

+3

tools/testing/selftests/kvm/include/arm64/vgic.h

reviewed

··· 16 16 ((uint64_t)(flags) << 12) | \ 17 17 index) 18 18 19 19 + bool kvm_supports_vgic_v3(void); 20 20 + int __vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs); 21 21 + void __vgic_v3_init(int fd); 19 22 int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs); 20 23 21 24 #define VGIC_MAX_RESERVED 1023

+17 -7

tools/testing/selftests/kvm/include/kvm_util.h

reviewed

··· 64 64 #ifdef __x86_64__ 65 65 struct kvm_cpuid2 *cpuid; 66 66 #endif 67 67 + #ifdef __aarch64__ 68 68 + struct kvm_vcpu_init init; 69 69 + #endif 67 70 struct kvm_binary_stats stats; 68 71 struct kvm_dirty_gfn *dirty_gfns; 69 72 uint32_t fetch_index; ··· 263 260 int open_path_or_exit(const char *path, int flags); 264 261 int open_kvm_dev_path_or_exit(void); 265 262 266 266 - bool get_kvm_param_bool(const char *param); 267 267 - bool get_kvm_intel_param_bool(const char *param); 268 268 - bool get_kvm_amd_param_bool(const char *param); 263 263 + int kvm_get_module_param_integer(const char *module_name, const char *param); 264 264 + bool kvm_get_module_param_bool(const char *module_name, const char *param); 269 265 270 270 - int get_kvm_param_integer(const char *param); 271 271 - int get_kvm_intel_param_integer(const char *param); 272 272 - int get_kvm_amd_param_integer(const char *param); 266 266 + static inline bool get_kvm_param_bool(const char *param) 267 267 + { 268 268 + return kvm_get_module_param_bool("kvm", param); 269 269 + } 270 270 + 271 271 + static inline int get_kvm_param_integer(const char *param) 272 272 + { 273 273 + return kvm_get_module_param_integer("kvm", param); 274 274 + } 273 275 274 276 unsigned int kvm_check_cap(long cap); 275 277 ··· 1265 1257 */ 1266 1258 void kvm_selftest_arch_init(void); 1267 1259 1268 1268 - void kvm_arch_vm_post_create(struct kvm_vm *vm); 1260 1260 + void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus); 1261 1261 + void kvm_arch_vm_finalize_vcpus(struct kvm_vm *vm); 1262 1262 + void kvm_arch_vm_release(struct kvm_vm *vm); 1269 1263 1270 1264 bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr); 1271 1265

+1

tools/testing/selftests/kvm/include/riscv/processor.h

reviewed

··· 9 9 10 10 #include <linux/stringify.h> 11 11 #include <asm/csr.h> 12 12 + #include <asm/vdso/processor.h> 12 13 #include "kvm_util.h" 13 14 14 15 #define INSN_OPCODE_MASK 0x007c

+26

tools/testing/selftests/kvm/include/x86/pmu.h

reviewed

··· 5 5 #ifndef SELFTEST_KVM_PMU_H 6 6 #define SELFTEST_KVM_PMU_H 7 7 8 8 + #include <stdbool.h> 8 9 #include <stdint.h> 10 10 + 11 11 + #include <linux/bits.h> 9 12 10 13 #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300 11 14 ··· 64 61 #define INTEL_ARCH_BRANCHES_RETIRED RAW_EVENT(0xc4, 0x00) 65 62 #define INTEL_ARCH_BRANCHES_MISPREDICTED RAW_EVENT(0xc5, 0x00) 66 63 #define INTEL_ARCH_TOPDOWN_SLOTS RAW_EVENT(0xa4, 0x01) 64 64 + #define INTEL_ARCH_TOPDOWN_BE_BOUND RAW_EVENT(0xa4, 0x02) 65 65 + #define INTEL_ARCH_TOPDOWN_BAD_SPEC RAW_EVENT(0x73, 0x00) 66 66 + #define INTEL_ARCH_TOPDOWN_FE_BOUND RAW_EVENT(0x9c, 0x01) 67 67 + #define INTEL_ARCH_TOPDOWN_RETIRING RAW_EVENT(0xc2, 0x02) 68 68 + #define INTEL_ARCH_LBR_INSERTS RAW_EVENT(0xe4, 0x01) 67 69 68 70 #define AMD_ZEN_CORE_CYCLES RAW_EVENT(0x76, 0x00) 69 71 #define AMD_ZEN_INSTRUCTIONS_RETIRED RAW_EVENT(0xc0, 0x00) ··· 88 80 INTEL_ARCH_BRANCHES_RETIRED_INDEX, 89 81 INTEL_ARCH_BRANCHES_MISPREDICTED_INDEX, 90 82 INTEL_ARCH_TOPDOWN_SLOTS_INDEX, 83 83 + INTEL_ARCH_TOPDOWN_BE_BOUND_INDEX, 84 84 + INTEL_ARCH_TOPDOWN_BAD_SPEC_INDEX, 85 85 + INTEL_ARCH_TOPDOWN_FE_BOUND_INDEX, 86 86 + INTEL_ARCH_TOPDOWN_RETIRING_INDEX, 87 87 + INTEL_ARCH_LBR_INSERTS_INDEX, 91 88 NR_INTEL_ARCH_EVENTS, 92 89 }; 93 90 ··· 106 93 107 94 extern const uint64_t intel_pmu_arch_events[]; 108 95 extern const uint64_t amd_pmu_zen_events[]; 96 96 + 97 97 + enum pmu_errata { 98 98 + INSTRUCTIONS_RETIRED_OVERCOUNT, 99 99 + BRANCHES_RETIRED_OVERCOUNT, 100 100 + }; 101 101 + extern uint64_t pmu_errata_mask; 102 102 + 103 103 + void kvm_init_pmu_errata(void); 104 104 + 105 105 + static inline bool this_pmu_has_errata(enum pmu_errata errata) 106 106 + { 107 107 + return pmu_errata_mask & BIT_ULL(errata); 108 108 + } 109 109 110 110 #endif /* SELFTEST_KVM_PMU_H */

+34 -1

tools/testing/selftests/kvm/include/x86/processor.h

reviewed

··· 34 34 35 35 #define NMI_VECTOR 0x02 36 36 37 37 + const char *ex_str(int vector); 38 38 + 37 39 #define X86_EFLAGS_FIXED (1u << 1) 38 40 39 41 #define X86_CR4_VME (1ul << 0) ··· 267 265 #define X86_PROPERTY_PMU_NR_GP_COUNTERS KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 8, 15) 268 266 #define X86_PROPERTY_PMU_GP_COUNTERS_BIT_WIDTH KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 16, 23) 269 267 #define X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 24, 31) 270 270 - #define X86_PROPERTY_PMU_EVENTS_MASK KVM_X86_CPU_PROPERTY(0xa, 0, EBX, 0, 7) 268 268 + #define X86_PROPERTY_PMU_EVENTS_MASK KVM_X86_CPU_PROPERTY(0xa, 0, EBX, 0, 12) 271 269 #define X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK KVM_X86_CPU_PROPERTY(0xa, 0, ECX, 0, 31) 272 270 #define X86_PROPERTY_PMU_NR_FIXED_COUNTERS KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 0, 4) 273 271 #define X86_PROPERTY_PMU_FIXED_COUNTERS_BIT_WIDTH KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 5, 12) ··· 334 332 #define X86_PMU_FEATURE_BRANCH_INSNS_RETIRED KVM_X86_PMU_FEATURE(EBX, 5) 335 333 #define X86_PMU_FEATURE_BRANCHES_MISPREDICTED KVM_X86_PMU_FEATURE(EBX, 6) 336 334 #define X86_PMU_FEATURE_TOPDOWN_SLOTS KVM_X86_PMU_FEATURE(EBX, 7) 335 335 + #define X86_PMU_FEATURE_TOPDOWN_BE_BOUND KVM_X86_PMU_FEATURE(EBX, 8) 336 336 + #define X86_PMU_FEATURE_TOPDOWN_BAD_SPEC KVM_X86_PMU_FEATURE(EBX, 9) 337 337 + #define X86_PMU_FEATURE_TOPDOWN_FE_BOUND KVM_X86_PMU_FEATURE(EBX, 10) 338 338 + #define X86_PMU_FEATURE_TOPDOWN_RETIRING KVM_X86_PMU_FEATURE(EBX, 11) 339 339 + #define X86_PMU_FEATURE_LBR_INSERTS KVM_X86_PMU_FEATURE(EBX, 12) 337 340 338 341 #define X86_PMU_FEATURE_INSNS_RETIRED_FIXED KVM_X86_PMU_FEATURE(ECX, 0) 339 342 #define X86_PMU_FEATURE_CPU_CYCLES_FIXED KVM_X86_PMU_FEATURE(ECX, 1) ··· 1186 1179 void vm_install_exception_handler(struct kvm_vm *vm, int vector, 1187 1180 void (*handler)(struct ex_regs *)); 1188 1181 1182 1182 + /* 1183 1183 + * Exception fixup morphs #DE to an arbitrary magic vector so that '0' can be 1184 1184 + * used to signal "no expcetion". 1185 1185 + */ 1186 1186 + #define KVM_MAGIC_DE_VECTOR 0xff 1187 1187 + 1189 1188 /* If a toddler were to say "abracadabra". */ 1190 1189 #define KVM_EXCEPTION_MAGIC 0xabacadabaULL 1191 1190 ··· 1326 1313 } 1327 1314 1328 1315 bool kvm_is_tdp_enabled(void); 1316 1316 + 1317 1317 + static inline bool get_kvm_intel_param_bool(const char *param) 1318 1318 + { 1319 1319 + return kvm_get_module_param_bool("kvm_intel", param); 1320 1320 + } 1321 1321 + 1322 1322 + static inline bool get_kvm_amd_param_bool(const char *param) 1323 1323 + { 1324 1324 + return kvm_get_module_param_bool("kvm_amd", param); 1325 1325 + } 1326 1326 + 1327 1327 + static inline int get_kvm_intel_param_integer(const char *param) 1328 1328 + { 1329 1329 + return kvm_get_module_param_integer("kvm_intel", param); 1330 1330 + } 1331 1331 + 1332 1332 + static inline int get_kvm_amd_param_integer(const char *param) 1333 1333 + { 1334 1334 + return kvm_get_module_param_integer("kvm_amd", param); 1335 1335 + } 1329 1336 1330 1337 static inline bool kvm_is_pmu_enabled(void) 1331 1338 {

+88 -16

tools/testing/selftests/kvm/lib/arm64/processor.c

reviewed

··· 12 12 #include "kvm_util.h" 13 13 #include "processor.h" 14 14 #include "ucall_common.h" 15 15 + #include "vgic.h" 15 16 16 17 #include <linux/bitfield.h> 17 18 #include <linux/sizes.h> ··· 186 185 _virt_pg_map(vm, vaddr, paddr, attr_idx); 187 186 } 188 187 189 189 - uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva) 188 188 + uint64_t *virt_get_pte_hva_at_level(struct kvm_vm *vm, vm_vaddr_t gva, int level) 190 189 { 191 190 uint64_t *ptep; 192 191 ··· 196 195 ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, gva) * 8; 197 196 if (!ptep) 198 197 goto unmapped_gva; 198 198 + if (level == 0) 199 199 + return ptep; 199 200 200 201 switch (vm->pgtable_levels) { 201 202 case 4: 202 203 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, gva) * 8; 203 204 if (!ptep) 204 205 goto unmapped_gva; 206 206 + if (level == 1) 207 207 + break; 205 208 /* fall through */ 206 209 case 3: 207 210 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, gva) * 8; 208 211 if (!ptep) 209 212 goto unmapped_gva; 213 213 + if (level == 2) 214 214 + break; 210 215 /* fall through */ 211 216 case 2: 212 217 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, gva) * 8; ··· 228 221 unmapped_gva: 229 222 TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva); 230 223 exit(EXIT_FAILURE); 224 224 + } 225 225 + 226 226 + uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva) 227 227 + { 228 228 + return virt_get_pte_hva_at_level(vm, gva, 3); 231 229 } 232 230 233 231 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) ··· 278 266 } 279 267 } 280 268 269 269 + bool vm_supports_el2(struct kvm_vm *vm) 270 270 + { 271 271 + const char *value = getenv("NV"); 272 272 + 273 273 + if (value && *value == '0') 274 274 + return false; 275 275 + 276 276 + return vm_check_cap(vm, KVM_CAP_ARM_EL2) && vm->arch.has_gic; 277 277 + } 278 278 + 279 279 + void kvm_get_default_vcpu_target(struct kvm_vm *vm, struct kvm_vcpu_init *init) 280 280 + { 281 281 + struct kvm_vcpu_init preferred = {}; 282 282 + 283 283 + vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &preferred); 284 284 + if (vm_supports_el2(vm)) 285 285 + preferred.features[0] |= BIT(KVM_ARM_VCPU_HAS_EL2); 286 286 + 287 287 + *init = preferred; 288 288 + } 289 289 + 281 290 void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) 282 291 { 283 292 struct kvm_vcpu_init default_init = { .target = -1, }; 284 293 struct kvm_vm *vm = vcpu->vm; 285 294 uint64_t sctlr_el1, tcr_el1, ttbr0_el1; 286 295 287 287 - if (!init) 296 296 + if (!init) { 297 297 + kvm_get_default_vcpu_target(vm, &default_init); 288 298 init = &default_init; 289 289 - 290 290 - if (init->target == -1) { 291 291 - struct kvm_vcpu_init preferred; 292 292 - vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &preferred); 293 293 - init->target = preferred.target; 294 299 } 295 300 296 301 vcpu_ioctl(vcpu, KVM_ARM_VCPU_INIT, init); 302 302 + vcpu->init = *init; 297 303 298 304 /* 299 305 * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15 300 306 * registers, which the variable argument list macros do. 301 307 */ 302 302 - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPACR_EL1), 3 << 20); 308 308 + vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_CPACR_EL1), 3 << 20); 303 309 304 304 - sctlr_el1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1)); 305 305 - tcr_el1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1)); 310 310 + sctlr_el1 = vcpu_get_reg(vcpu, ctxt_reg_alias(vcpu, SYS_SCTLR_EL1)); 311 311 + tcr_el1 = vcpu_get_reg(vcpu, ctxt_reg_alias(vcpu, SYS_TCR_EL1)); 306 312 307 313 /* Configure base granule size */ 308 314 switch (vm->mode) { ··· 387 357 if (use_lpa2_pte_format(vm)) 388 358 tcr_el1 |= TCR_DS; 389 359 390 390 - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1); 391 391 - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1); 392 392 - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL1), DEFAULT_MAIR_EL1); 393 393 - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL1), ttbr0_el1); 360 360 + vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_SCTLR_EL1), sctlr_el1); 361 361 + vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_TCR_EL1), tcr_el1); 362 362 + vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_MAIR_EL1), DEFAULT_MAIR_EL1); 363 363 + vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_TTBR0_EL1), ttbr0_el1); 394 364 vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL1), vcpu->id); 365 365 + 366 366 + if (!vcpu_has_el2(vcpu)) 367 367 + return; 368 368 + 369 369 + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_HCR_EL2), 370 370 + HCR_EL2_RW | HCR_EL2_TGE | HCR_EL2_E2H); 395 371 } 396 372 397 373 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) ··· 431 395 432 396 aarch64_vcpu_setup(vcpu, init); 433 397 434 434 - vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size); 398 398 + vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_SP_EL1), stack_vaddr + stack_size); 435 399 return vcpu; 436 400 } 437 401 ··· 501 465 { 502 466 extern char vectors; 503 467 504 504 - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL1), (uint64_t)&vectors); 468 468 + vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_VBAR_EL1), (uint64_t)&vectors); 505 469 } 506 470 507 471 void route_exception(struct ex_regs *regs, int vector) ··· 688 652 void wfi(void) 689 653 { 690 654 asm volatile("wfi"); 655 655 + } 656 656 + 657 657 + static bool request_mte; 658 658 + static bool request_vgic = true; 659 659 + 660 660 + void test_wants_mte(void) 661 661 + { 662 662 + request_mte = true; 663 663 + } 664 664 + 665 665 + void test_disable_default_vgic(void) 666 666 + { 667 667 + request_vgic = false; 668 668 + } 669 669 + 670 670 + void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus) 671 671 + { 672 672 + if (request_mte && vm_check_cap(vm, KVM_CAP_ARM_MTE)) 673 673 + vm_enable_cap(vm, KVM_CAP_ARM_MTE, 0); 674 674 + 675 675 + if (request_vgic && kvm_supports_vgic_v3()) { 676 676 + vm->arch.gic_fd = __vgic_v3_setup(vm, nr_vcpus, 64); 677 677 + vm->arch.has_gic = true; 678 678 + } 679 679 + } 680 680 + 681 681 + void kvm_arch_vm_finalize_vcpus(struct kvm_vm *vm) 682 682 + { 683 683 + if (vm->arch.has_gic) 684 684 + __vgic_v3_init(vm->arch.gic_fd); 685 685 + } 686 686 + 687 687 + void kvm_arch_vm_release(struct kvm_vm *vm) 688 688 + { 689 689 + if (vm->arch.has_gic) 690 690 + close(vm->arch.gic_fd); 691 691 }

+45 -21

tools/testing/selftests/kvm/lib/arm64/vgic.c

reviewed

··· 15 15 #include "gic.h" 16 16 #include "gic_v3.h" 17 17 18 18 + bool kvm_supports_vgic_v3(void) 19 19 + { 20 20 + struct kvm_vm *vm = vm_create_barebones(); 21 21 + int r; 22 22 + 23 23 + r = __kvm_test_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V3); 24 24 + kvm_vm_free(vm); 25 25 + 26 26 + return !r; 27 27 + } 28 28 + 18 29 /* 19 30 * vGIC-v3 default host setup 20 31 * ··· 41 30 * redistributor regions of the guest. Since it depends on the number of 42 31 * vCPUs for the VM, it must be called after all the vCPUs have been created. 43 32 */ 44 44 - int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs) 33 33 + int __vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs) 45 34 { 46 35 int gic_fd; 47 36 uint64_t attr; 48 48 - struct list_head *iter; 49 49 - unsigned int nr_gic_pages, nr_vcpus_created = 0; 50 50 - 51 51 - TEST_ASSERT(nr_vcpus, "Number of vCPUs cannot be empty"); 52 52 - 53 53 - /* 54 54 - * Make sure that the caller is infact calling this 55 55 - * function after all the vCPUs are added. 56 56 - */ 57 57 - list_for_each(iter, &vm->vcpus) 58 58 - nr_vcpus_created++; 59 59 - TEST_ASSERT(nr_vcpus == nr_vcpus_created, 60 60 - "Number of vCPUs requested (%u) doesn't match with the ones created for the VM (%u)", 61 61 - nr_vcpus, nr_vcpus_created); 37 37 + unsigned int nr_gic_pages; 62 38 63 39 /* Distributor setup */ 64 40 gic_fd = __kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V3); ··· 53 55 return gic_fd; 54 56 55 57 kvm_device_attr_set(gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0, &nr_irqs); 56 56 - 57 57 - kvm_device_attr_set(gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, 58 58 - KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); 59 58 60 59 attr = GICD_BASE_GPA; 61 60 kvm_device_attr_set(gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, ··· 68 73 KVM_VGIC_V3_REDIST_SIZE * nr_vcpus); 69 74 virt_map(vm, GICR_BASE_GPA, GICR_BASE_GPA, nr_gic_pages); 70 75 71 71 - kvm_device_attr_set(gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, 72 72 - KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); 73 73 - 74 76 return gic_fd; 77 77 + } 78 78 + 79 79 + void __vgic_v3_init(int fd) 80 80 + { 81 81 + kvm_device_attr_set(fd, KVM_DEV_ARM_VGIC_GRP_CTRL, 82 82 + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); 83 83 + } 84 84 + 85 85 + int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs) 86 86 + { 87 87 + unsigned int nr_vcpus_created = 0; 88 88 + struct list_head *iter; 89 89 + int fd; 90 90 + 91 91 + TEST_ASSERT(nr_vcpus, "Number of vCPUs cannot be empty"); 92 92 + 93 93 + /* 94 94 + * Make sure that the caller is infact calling this 95 95 + * function after all the vCPUs are added. 96 96 + */ 97 97 + list_for_each(iter, &vm->vcpus) 98 98 + nr_vcpus_created++; 99 99 + TEST_ASSERT(nr_vcpus == nr_vcpus_created, 100 100 + "Number of vCPUs requested (%u) doesn't match with the ones created for the VM (%u)", 101 101 + nr_vcpus, nr_vcpus_created); 102 102 + 103 103 + fd = __vgic_v3_setup(vm, nr_vcpus, nr_irqs); 104 104 + if (fd < 0) 105 105 + return fd; 106 106 + 107 107 + __vgic_v3_init(fd); 108 108 + return fd; 75 109 } 76 110 77 111 /* should only work for level sensitive interrupts */

+19 -38

tools/testing/selftests/kvm/lib/kvm_util.c

reviewed

··· 24 24 struct guest_random_state guest_rng; 25 25 static uint32_t last_guest_seed; 26 26 27 27 - static int vcpu_mmap_sz(void); 27 27 + static size_t vcpu_mmap_sz(void); 28 28 29 29 int __open_path_or_exit(const char *path, int flags, const char *enoent_help) 30 30 { ··· 95 95 return bytes_read; 96 96 } 97 97 98 98 - static int get_module_param_integer(const char *module_name, const char *param) 98 98 + int kvm_get_module_param_integer(const char *module_name, const char *param) 99 99 { 100 100 /* 101 101 * 16 bytes to hold a 64-bit value (1 byte per char), 1 byte for the ··· 119 119 return atoi_paranoid(value); 120 120 } 121 121 122 122 - static bool get_module_param_bool(const char *module_name, const char *param) 122 122 + bool kvm_get_module_param_bool(const char *module_name, const char *param) 123 123 { 124 124 char value; 125 125 ssize_t r; ··· 133 133 return false; 134 134 135 135 TEST_FAIL("Unrecognized value '%c' for boolean module param", value); 136 136 - } 137 137 - 138 138 - bool get_kvm_param_bool(const char *param) 139 139 - { 140 140 - return get_module_param_bool("kvm", param); 141 141 - } 142 142 - 143 143 - bool get_kvm_intel_param_bool(const char *param) 144 144 - { 145 145 - return get_module_param_bool("kvm_intel", param); 146 146 - } 147 147 - 148 148 - bool get_kvm_amd_param_bool(const char *param) 149 149 - { 150 150 - return get_module_param_bool("kvm_amd", param); 151 151 - } 152 152 - 153 153 - int get_kvm_param_integer(const char *param) 154 154 - { 155 155 - return get_module_param_integer("kvm", param); 156 156 - } 157 157 - 158 158 - int get_kvm_intel_param_integer(const char *param) 159 159 - { 160 160 - return get_module_param_integer("kvm_intel", param); 161 161 - } 162 162 - 163 163 - int get_kvm_amd_param_integer(const char *param) 164 164 - { 165 165 - return get_module_param_integer("kvm_amd", param); 166 136 } 167 137 168 138 /* ··· 487 517 guest_rng = new_guest_random_state(guest_random_seed); 488 518 sync_global_to_guest(vm, guest_rng); 489 519 490 490 - kvm_arch_vm_post_create(vm); 520 520 + kvm_arch_vm_post_create(vm, nr_runnable_vcpus); 491 521 492 522 return vm; 493 523 } ··· 525 555 for (i = 0; i < nr_vcpus; ++i) 526 556 vcpus[i] = vm_vcpu_add(vm, i, guest_code); 527 557 558 558 + kvm_arch_vm_finalize_vcpus(vm); 528 559 return vm; 529 560 } 530 561 ··· 776 805 777 806 /* Free cached stats metadata and close FD */ 778 807 kvm_stats_release(&vmp->stats); 808 808 + 809 809 + kvm_arch_vm_release(vmp); 779 810 } 780 811 781 812 static void __vm_mem_region_delete(struct kvm_vm *vm, ··· 1294 1321 } 1295 1322 1296 1323 /* Returns the size of a vCPU's kvm_run structure. */ 1297 1297 - static int vcpu_mmap_sz(void) 1324 1324 + static size_t vcpu_mmap_sz(void) 1298 1325 { 1299 1326 int dev_fd, ret; 1300 1327 1301 1328 dev_fd = open_kvm_dev_path_or_exit(); 1302 1329 1303 1330 ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL); 1304 1304 - TEST_ASSERT(ret >= sizeof(struct kvm_run), 1331 1331 + TEST_ASSERT(ret >= 0 && ret >= sizeof(struct kvm_run), 1305 1332 KVM_IOCTL_ERROR(KVM_GET_VCPU_MMAP_SIZE, ret)); 1306 1333 1307 1334 close(dev_fd); ··· 1342 1369 TEST_ASSERT_VM_VCPU_IOCTL(vcpu->fd >= 0, KVM_CREATE_VCPU, vcpu->fd, vm); 1343 1370 1344 1371 TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->run), "vcpu mmap size " 1345 1345 - "smaller than expected, vcpu_mmap_sz: %i expected_min: %zi", 1372 1372 + "smaller than expected, vcpu_mmap_sz: %zi expected_min: %zi", 1346 1373 vcpu_mmap_sz(), sizeof(*vcpu->run)); 1347 1374 vcpu->run = (struct kvm_run *) mmap(NULL, vcpu_mmap_sz(), 1348 1375 PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd, 0); ··· 2303 2330 TEST_FAIL("Unable to find stat '%s'", name); 2304 2331 } 2305 2332 2306 2306 - __weak void kvm_arch_vm_post_create(struct kvm_vm *vm) 2333 2333 + __weak void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus) 2334 2334 + { 2335 2335 + } 2336 2336 + 2337 2337 + __weak void kvm_arch_vm_finalize_vcpus(struct kvm_vm *vm) 2338 2338 + { 2339 2339 + } 2340 2340 + 2341 2341 + __weak void kvm_arch_vm_release(struct kvm_vm *vm) 2307 2342 { 2308 2343 } 2309 2344

+49

tools/testing/selftests/kvm/lib/x86/pmu.c

reviewed

··· 8 8 #include <linux/kernel.h> 9 9 10 10 #include "kvm_util.h" 11 11 + #include "processor.h" 11 12 #include "pmu.h" 12 13 13 14 const uint64_t intel_pmu_arch_events[] = { ··· 20 19 INTEL_ARCH_BRANCHES_RETIRED, 21 20 INTEL_ARCH_BRANCHES_MISPREDICTED, 22 21 INTEL_ARCH_TOPDOWN_SLOTS, 22 22 + INTEL_ARCH_TOPDOWN_BE_BOUND, 23 23 + INTEL_ARCH_TOPDOWN_BAD_SPEC, 24 24 + INTEL_ARCH_TOPDOWN_FE_BOUND, 25 25 + INTEL_ARCH_TOPDOWN_RETIRING, 26 26 + INTEL_ARCH_LBR_INSERTS, 23 27 }; 24 28 kvm_static_assert(ARRAY_SIZE(intel_pmu_arch_events) == NR_INTEL_ARCH_EVENTS); 25 29 ··· 35 29 AMD_ZEN_BRANCHES_MISPREDICTED, 36 30 }; 37 31 kvm_static_assert(ARRAY_SIZE(amd_pmu_zen_events) == NR_AMD_ZEN_EVENTS); 32 32 + 33 33 + /* 34 34 + * For Intel Atom CPUs, the PMU events "Instruction Retired" or 35 35 + * "Branch Instruction Retired" may be overcounted for some certain 36 36 + * instructions, like FAR CALL/JMP, RETF, IRET, VMENTRY/VMEXIT/VMPTRLD 37 37 + * and complex SGX/SMX/CSTATE instructions/flows. 38 38 + * 39 39 + * The detailed information can be found in the errata (section SRF7): 40 40 + * https://edc.intel.com/content/www/us/en/design/products-and-solutions/processors-and-chipsets/sierra-forest/xeon-6700-series-processor-with-e-cores-specification-update/errata-details/ 41 41 + * 42 42 + * For the Atom platforms before Sierra Forest (including Sierra Forest), 43 43 + * Both 2 events "Instruction Retired" and "Branch Instruction Retired" would 44 44 + * be overcounted on these certain instructions, but for Clearwater Forest 45 45 + * only "Instruction Retired" event is overcounted on these instructions. 46 46 + */ 47 47 + static uint64_t get_pmu_errata(void) 48 48 + { 49 49 + if (!this_cpu_is_intel()) 50 50 + return 0; 51 51 + 52 52 + if (this_cpu_family() != 0x6) 53 53 + return 0; 54 54 + 55 55 + switch (this_cpu_model()) { 56 56 + case 0xDD: /* Clearwater Forest */ 57 57 + return BIT_ULL(INSTRUCTIONS_RETIRED_OVERCOUNT); 58 58 + case 0xAF: /* Sierra Forest */ 59 59 + case 0x4D: /* Avaton, Rangely */ 60 60 + case 0x5F: /* Denverton */ 61 61 + case 0x86: /* Jacobsville */ 62 62 + return BIT_ULL(INSTRUCTIONS_RETIRED_OVERCOUNT) | 63 63 + BIT_ULL(BRANCHES_RETIRED_OVERCOUNT); 64 64 + default: 65 65 + return 0; 66 66 + } 67 67 + } 68 68 + 69 69 + uint64_t pmu_errata_mask; 70 70 + 71 71 + void kvm_init_pmu_errata(void) 72 72 + { 73 73 + pmu_errata_mask = get_pmu_errata(); 74 74 + }

+39 -2

tools/testing/selftests/kvm/lib/x86/processor.c

reviewed

··· 6 6 #include "linux/bitmap.h" 7 7 #include "test_util.h" 8 8 #include "kvm_util.h" 9 9 + #include "pmu.h" 9 10 #include "processor.h" 10 11 #include "sev.h" 11 12 ··· 23 22 bool host_cpu_is_intel; 24 23 bool is_forced_emulation_enabled; 25 24 uint64_t guest_tsc_khz; 25 25 + 26 26 + const char *ex_str(int vector) 27 27 + { 28 28 + switch (vector) { 29 29 + #define VEC_STR(v) case v##_VECTOR: return "#" #v 30 30 + case DE_VECTOR: return "no exception"; 31 31 + case KVM_MAGIC_DE_VECTOR: return "#DE"; 32 32 + VEC_STR(DB); 33 33 + VEC_STR(NMI); 34 34 + VEC_STR(BP); 35 35 + VEC_STR(OF); 36 36 + VEC_STR(BR); 37 37 + VEC_STR(UD); 38 38 + VEC_STR(NM); 39 39 + VEC_STR(DF); 40 40 + VEC_STR(TS); 41 41 + VEC_STR(NP); 42 42 + VEC_STR(SS); 43 43 + VEC_STR(GP); 44 44 + VEC_STR(PF); 45 45 + VEC_STR(MF); 46 46 + VEC_STR(AC); 47 47 + VEC_STR(MC); 48 48 + VEC_STR(XM); 49 49 + VEC_STR(VE); 50 50 + VEC_STR(CP); 51 51 + VEC_STR(HV); 52 52 + VEC_STR(VC); 53 53 + VEC_STR(SX); 54 54 + default: return "#??"; 55 55 + #undef VEC_STR 56 56 + } 57 57 + } 26 58 27 59 static void regs_dump(FILE *stream, struct kvm_regs *regs, uint8_t indent) 28 60 { ··· 591 557 return false; 592 558 593 559 if (regs->vector == DE_VECTOR) 594 594 - return false; 560 560 + regs->vector = KVM_MAGIC_DE_VECTOR; 595 561 596 562 regs->rip = regs->r11; 597 563 regs->r9 = regs->vector; ··· 659 625 REPORT_GUEST_ASSERT(uc); 660 626 } 661 627 662 662 - void kvm_arch_vm_post_create(struct kvm_vm *vm) 628 628 + void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus) 663 629 { 664 630 int r; 665 631 ··· 672 638 sync_global_to_guest(vm, host_cpu_is_intel); 673 639 sync_global_to_guest(vm, host_cpu_is_amd); 674 640 sync_global_to_guest(vm, is_forced_emulation_enabled); 641 641 + sync_global_to_guest(vm, pmu_errata_mask); 675 642 676 643 if (is_sev_vm(vm)) { 677 644 struct kvm_sev_init init = { 0 }; ··· 1304 1269 host_cpu_is_intel = this_cpu_is_intel(); 1305 1270 host_cpu_is_amd = this_cpu_is_amd(); 1306 1271 is_forced_emulation_enabled = kvm_is_forced_emulation_enabled(); 1272 1272 + 1273 1273 + kvm_init_pmu_errata(); 1307 1274 } 1308 1275 1309 1276 bool sys_clocksource_is_based_on_tsc(void)

+1

tools/testing/selftests/kvm/memslot_modification_stress_test.c

reviewed

··· 22 22 #include "processor.h" 23 23 #include "test_util.h" 24 24 #include "guest_modes.h" 25 25 + #include "ucall_common.h" 25 26 26 27 #define DUMMY_MEMSLOT_INDEX 7 27 28

+1

tools/testing/selftests/kvm/memslot_perf_test.c

reviewed

··· 25 25 #include <test_util.h> 26 26 #include <kvm_util.h> 27 27 #include <processor.h> 28 28 + #include <ucall_common.h> 28 29 29 30 #define MEM_EXTRA_SIZE SZ_64K 30 31

+60

tools/testing/selftests/kvm/riscv/get-reg-list.c

reviewed

··· 80 80 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZCF: 81 81 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZCMOP: 82 82 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZFA: 83 83 + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZFBFMIN: 83 84 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZFH: 84 85 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZFHMIN: 85 86 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOM: 87 87 + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOP: 86 88 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOZ: 87 89 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICCRSE: 88 90 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICNTR: ··· 105 103 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZTSO: 106 104 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZVBB: 107 105 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZVBC: 106 106 + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZVFBFMIN: 107 107 + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZVFBFWMA: 108 108 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZVFH: 109 109 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZVFHMIN: 110 110 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZVKB: ··· 132 128 case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_DBCN: 133 129 case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_SUSP: 134 130 case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_STA: 131 131 + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_FWFT: 135 132 case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_EXPERIMENTAL: 136 133 case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_VENDOR: 137 134 return true; ··· 260 255 return "KVM_REG_RISCV_CONFIG_REG(zicbom_block_size)"; 261 256 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 262 257 return "KVM_REG_RISCV_CONFIG_REG(zicboz_block_size)"; 258 258 + case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): 259 259 + return "KVM_REG_RISCV_CONFIG_REG(zicbop_block_size)"; 263 260 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 264 261 return "KVM_REG_RISCV_CONFIG_REG(mvendorid)"; 265 262 case KVM_REG_RISCV_CONFIG_REG(marchid): ··· 539 532 KVM_ISA_EXT_ARR(ZCF), 540 533 KVM_ISA_EXT_ARR(ZCMOP), 541 534 KVM_ISA_EXT_ARR(ZFA), 535 535 + KVM_ISA_EXT_ARR(ZFBFMIN), 542 536 KVM_ISA_EXT_ARR(ZFH), 543 537 KVM_ISA_EXT_ARR(ZFHMIN), 544 538 KVM_ISA_EXT_ARR(ZICBOM), 539 539 + KVM_ISA_EXT_ARR(ZICBOP), 545 540 KVM_ISA_EXT_ARR(ZICBOZ), 546 541 KVM_ISA_EXT_ARR(ZICCRSE), 547 542 KVM_ISA_EXT_ARR(ZICNTR), ··· 564 555 KVM_ISA_EXT_ARR(ZTSO), 565 556 KVM_ISA_EXT_ARR(ZVBB), 566 557 KVM_ISA_EXT_ARR(ZVBC), 558 558 + KVM_ISA_EXT_ARR(ZVFBFMIN), 559 559 + KVM_ISA_EXT_ARR(ZVFBFWMA), 567 560 KVM_ISA_EXT_ARR(ZVFH), 568 561 KVM_ISA_EXT_ARR(ZVFHMIN), 569 562 KVM_ISA_EXT_ARR(ZVKB), ··· 638 627 KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_DBCN), 639 628 KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_SUSP), 640 629 KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_STA), 630 630 + KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_FWFT), 641 631 KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_EXPERIMENTAL), 642 632 KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_VENDOR), 643 633 }; ··· 695 683 return strdup_printf("KVM_REG_RISCV_SBI_STA | %lld /* UNKNOWN */", reg_off); 696 684 } 697 685 686 686 + static const char *sbi_fwft_id_to_str(__u64 reg_off) 687 687 + { 688 688 + switch (reg_off) { 689 689 + case 0: return "KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(misaligned_deleg.enable)"; 690 690 + case 1: return "KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(misaligned_deleg.flags)"; 691 691 + case 2: return "KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(misaligned_deleg.value)"; 692 692 + case 3: return "KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(pointer_masking.enable)"; 693 693 + case 4: return "KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(pointer_masking.flags)"; 694 694 + case 5: return "KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(pointer_masking.value)"; 695 695 + } 696 696 + return strdup_printf("KVM_REG_RISCV_SBI_FWFT | %lld /* UNKNOWN */", reg_off); 697 697 + } 698 698 + 698 699 static const char *sbi_id_to_str(const char *prefix, __u64 id) 699 700 { 700 701 __u64 reg_off = id & ~(REG_MASK | KVM_REG_RISCV_SBI_STATE); ··· 720 695 switch (reg_subtype) { 721 696 case KVM_REG_RISCV_SBI_STA: 722 697 return sbi_sta_id_to_str(reg_off); 698 698 + case KVM_REG_RISCV_SBI_FWFT: 699 699 + return sbi_fwft_id_to_str(reg_off); 723 700 } 724 701 725 702 return strdup_printf("%lld | %lld /* UNKNOWN */", reg_subtype, reg_off); ··· 807 780 */ 808 781 static __u64 base_regs[] = { 809 782 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(isa), 783 783 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(zicbom_block_size), 810 784 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(mvendorid), 811 785 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(marchid), 812 786 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(mimpid), 787 787 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(zicboz_block_size), 813 788 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(satp_mode), 789 789 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(zicbop_block_size), 814 790 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CORE | KVM_REG_RISCV_CORE_REG(regs.pc), 815 791 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CORE | KVM_REG_RISCV_CORE_REG(regs.ra), 816 792 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CORE | KVM_REG_RISCV_CORE_REG(regs.sp), ··· 889 859 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_STATE | KVM_REG_RISCV_SBI_STA | KVM_REG_RISCV_SBI_STA_REG(shmem_hi), 890 860 }; 891 861 862 862 + static __u64 sbi_fwft_regs[] = { 863 863 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_FWFT, 864 864 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_STATE | KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(misaligned_deleg.enable), 865 865 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_STATE | KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(misaligned_deleg.flags), 866 866 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_STATE | KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(misaligned_deleg.value), 867 867 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_STATE | KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(pointer_masking.enable), 868 868 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_STATE | KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(pointer_masking.flags), 869 869 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_STATE | KVM_REG_RISCV_SBI_FWFT | KVM_REG_RISCV_SBI_FWFT_REG(pointer_masking.value), 870 870 + }; 871 871 + 892 872 static __u64 zicbom_regs[] = { 893 873 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(zicbom_block_size), 894 874 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOM, 875 875 + }; 876 876 + 877 877 + static __u64 zicbop_regs[] = { 878 878 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(zicbop_block_size), 879 879 + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOP, 895 880 }; 896 881 897 882 static __u64 zicboz_regs[] = { ··· 1055 1010 #define SUBLIST_SBI_STA \ 1056 1011 {"sbi-sta", .feature_type = VCPU_FEATURE_SBI_EXT, .feature = KVM_RISCV_SBI_EXT_STA, \ 1057 1012 .regs = sbi_sta_regs, .regs_n = ARRAY_SIZE(sbi_sta_regs),} 1013 1013 + #define SUBLIST_SBI_FWFT \ 1014 1014 + {"sbi-fwft", .feature_type = VCPU_FEATURE_SBI_EXT, .feature = KVM_RISCV_SBI_EXT_FWFT, \ 1015 1015 + .regs = sbi_fwft_regs, .regs_n = ARRAY_SIZE(sbi_fwft_regs),} 1058 1016 #define SUBLIST_ZICBOM \ 1059 1017 {"zicbom", .feature = KVM_RISCV_ISA_EXT_ZICBOM, .regs = zicbom_regs, .regs_n = ARRAY_SIZE(zicbom_regs),} 1018 1018 + #define SUBLIST_ZICBOP \ 1019 1019 + {"zicbop", .feature = KVM_RISCV_ISA_EXT_ZICBOP, .regs = zicbop_regs, .regs_n = ARRAY_SIZE(zicbop_regs),} 1060 1020 #define SUBLIST_ZICBOZ \ 1061 1021 {"zicboz", .feature = KVM_RISCV_ISA_EXT_ZICBOZ, .regs = zicboz_regs, .regs_n = ARRAY_SIZE(zicboz_regs),} 1062 1022 #define SUBLIST_AIA \ ··· 1142 1092 KVM_SBI_EXT_SIMPLE_CONFIG(pmu, PMU); 1143 1093 KVM_SBI_EXT_SIMPLE_CONFIG(dbcn, DBCN); 1144 1094 KVM_SBI_EXT_SIMPLE_CONFIG(susp, SUSP); 1095 1095 + KVM_SBI_EXT_SUBLIST_CONFIG(fwft, FWFT); 1145 1096 1146 1097 KVM_ISA_EXT_SUBLIST_CONFIG(aia, AIA); 1147 1098 KVM_ISA_EXT_SUBLIST_CONFIG(fp_f, FP_F); ··· 1178 1127 KVM_ISA_EXT_SIMPLE_CONFIG(zcf, ZCF); 1179 1128 KVM_ISA_EXT_SIMPLE_CONFIG(zcmop, ZCMOP); 1180 1129 KVM_ISA_EXT_SIMPLE_CONFIG(zfa, ZFA); 1130 1130 + KVM_ISA_EXT_SIMPLE_CONFIG(zfbfmin, ZFBFMIN); 1181 1131 KVM_ISA_EXT_SIMPLE_CONFIG(zfh, ZFH); 1182 1132 KVM_ISA_EXT_SIMPLE_CONFIG(zfhmin, ZFHMIN); 1183 1133 KVM_ISA_EXT_SUBLIST_CONFIG(zicbom, ZICBOM); 1134 1134 + KVM_ISA_EXT_SUBLIST_CONFIG(zicbop, ZICBOP); 1184 1135 KVM_ISA_EXT_SUBLIST_CONFIG(zicboz, ZICBOZ); 1185 1136 KVM_ISA_EXT_SIMPLE_CONFIG(ziccrse, ZICCRSE); 1186 1137 KVM_ISA_EXT_SIMPLE_CONFIG(zicntr, ZICNTR); ··· 1203 1150 KVM_ISA_EXT_SIMPLE_CONFIG(ztso, ZTSO); 1204 1151 KVM_ISA_EXT_SIMPLE_CONFIG(zvbb, ZVBB); 1205 1152 KVM_ISA_EXT_SIMPLE_CONFIG(zvbc, ZVBC); 1153 1153 + KVM_ISA_EXT_SIMPLE_CONFIG(zvfbfmin, ZVFBFMIN); 1154 1154 + KVM_ISA_EXT_SIMPLE_CONFIG(zvfbfwma, ZVFBFWMA); 1206 1155 KVM_ISA_EXT_SIMPLE_CONFIG(zvfh, ZVFH); 1207 1156 KVM_ISA_EXT_SIMPLE_CONFIG(zvfhmin, ZVFHMIN); 1208 1157 KVM_ISA_EXT_SIMPLE_CONFIG(zvkb, ZVKB); ··· 1222 1167 &config_sbi_pmu, 1223 1168 &config_sbi_dbcn, 1224 1169 &config_sbi_susp, 1170 1170 + &config_sbi_fwft, 1225 1171 &config_aia, 1226 1172 &config_fp_f, 1227 1173 &config_fp_d, ··· 1257 1201 &config_zcf, 1258 1202 &config_zcmop, 1259 1203 &config_zfa, 1204 1204 + &config_zfbfmin, 1260 1205 &config_zfh, 1261 1206 &config_zfhmin, 1262 1207 &config_zicbom, 1208 1208 + &config_zicbop, 1263 1209 &config_zicboz, 1264 1210 &config_ziccrse, 1265 1211 &config_zicntr, ··· 1282 1224 &config_ztso, 1283 1225 &config_zvbb, 1284 1226 &config_zvbc, 1227 1227 + &config_zvfbfmin, 1228 1228 + &config_zvfbfwma, 1285 1229 &config_zvfh, 1286 1230 &config_zvfhmin, 1287 1231 &config_zvkb,

+1 -1

tools/testing/selftests/kvm/s390/cmma_test.c

reviewed

··· 145 145 slot0 = memslot2region(vm, 0); 146 146 ucall_init(vm, slot0->region.guest_phys_addr + slot0->region.memory_size); 147 147 148 148 - kvm_arch_vm_post_create(vm); 148 148 + kvm_arch_vm_post_create(vm, 0); 149 149 } 150 150 151 151 static struct kvm_vm *create_vm_two_memslots(void)

+1 -1

tools/testing/selftests/kvm/s390/cpumodel_subfuncs_test.c

reviewed

··· 291 291 ksft_test_result_pass("%s\n", testlist[idx].subfunc_name); 292 292 free(array); 293 293 } else { 294 294 - ksft_test_result_skip("%s feature is not avaialable\n", 294 294 + ksft_test_result_skip("%s feature is not available\n", 295 295 testlist[idx].subfunc_name); 296 296 } 297 297 }

+1 -1

tools/testing/selftests/kvm/steal_time.c

reviewed

··· 118 118 { 119 119 struct arm_smccc_res res; 120 120 121 121 - smccc_hvc(func, arg, 0, 0, 0, 0, 0, 0, &res); 121 121 + do_smccc(func, arg, 0, 0, 0, 0, 0, 0, &res); 122 122 return res.a0; 123 123 } 124 124

+63 -19

tools/testing/selftests/kvm/x86/fastops_test.c

reviewed

··· 8 8 * to set RFLAGS.CF based on whether or not the input is even or odd, so that 9 9 * instructions like ADC and SBB are deterministic. 10 10 */ 11 11 + #define fastop(__insn) \ 12 12 + "bt $0, %[bt_val]\n\t" \ 13 13 + __insn "\n\t" \ 14 14 + "pushfq\n\t" \ 15 15 + "pop %[flags]\n\t" 16 16 + 17 17 + #define flags_constraint(flags_val) [flags]"=r"(flags_val) 18 18 + #define bt_constraint(__bt_val) [bt_val]"rm"((uint32_t)__bt_val) 19 19 + 11 20 #define guest_execute_fastop_1(FEP, insn, __val, __flags) \ 12 21 ({ \ 13 13 - __asm__ __volatile__("bt $0, %[val]\n\t" \ 14 14 - FEP insn " %[val]\n\t" \ 15 15 - "pushfq\n\t" \ 16 16 - "pop %[flags]\n\t" \ 17 17 - : [val]"+r"(__val), [flags]"=r"(__flags) \ 18 18 - : : "cc", "memory"); \ 22 22 + __asm__ __volatile__(fastop(FEP insn " %[val]") \ 23 23 + : [val]"+r"(__val), flags_constraint(__flags) \ 24 24 + : bt_constraint(__val) \ 25 25 + : "cc", "memory"); \ 19 26 }) 20 27 21 28 #define guest_test_fastop_1(insn, type_t, __val) \ ··· 43 36 44 37 #define guest_execute_fastop_2(FEP, insn, __input, __output, __flags) \ 45 38 ({ \ 46 46 - __asm__ __volatile__("bt $0, %[output]\n\t" \ 47 47 - FEP insn " %[input], %[output]\n\t" \ 48 48 - "pushfq\n\t" \ 49 49 - "pop %[flags]\n\t" \ 50 50 - : [output]"+r"(__output), [flags]"=r"(__flags) \ 51 51 - : [input]"r"(__input) : "cc", "memory"); \ 39 39 + __asm__ __volatile__(fastop(FEP insn " %[input], %[output]") \ 40 40 + : [output]"+r"(__output), flags_constraint(__flags) \ 41 41 + : [input]"r"(__input), bt_constraint(__output) \ 42 42 + : "cc", "memory"); \ 52 43 }) 53 44 54 45 #define guest_test_fastop_2(insn, type_t, __val1, __val2) \ ··· 68 63 69 64 #define guest_execute_fastop_cl(FEP, insn, __shift, __output, __flags) \ 70 65 ({ \ 71 71 - __asm__ __volatile__("bt $0, %[output]\n\t" \ 72 72 - FEP insn " %%cl, %[output]\n\t" \ 73 73 - "pushfq\n\t" \ 74 74 - "pop %[flags]\n\t" \ 75 75 - : [output]"+r"(__output), [flags]"=r"(__flags) \ 76 76 - : "c"(__shift) : "cc", "memory"); \ 66 66 + __asm__ __volatile__(fastop(FEP insn " %%cl, %[output]") \ 67 67 + : [output]"+r"(__output), flags_constraint(__flags) \ 68 68 + : "c"(__shift), bt_constraint(__output) \ 69 69 + : "cc", "memory"); \ 77 70 }) 78 71 79 72 #define guest_test_fastop_cl(insn, type_t, __val1, __val2) \ ··· 90 87 __GUEST_ASSERT(flags == ex_flags, \ 91 88 "Wanted flags 0x%lx for '%s 0x%x, 0x%lx', got 0x%lx", \ 92 89 ex_flags, insn, shift, (uint64_t)input, flags); \ 90 90 + }) 91 91 + 92 92 + #define guest_execute_fastop_div(__KVM_ASM_SAFE, insn, __a, __d, __rm, __flags) \ 93 93 + ({ \ 94 94 + uint64_t ign_error_code; \ 95 95 + uint8_t vector; \ 96 96 + \ 97 97 + __asm__ __volatile__(fastop(__KVM_ASM_SAFE(insn " %[denom]")) \ 98 98 + : "+a"(__a), "+d"(__d), flags_constraint(__flags), \ 99 99 + KVM_ASM_SAFE_OUTPUTS(vector, ign_error_code) \ 100 100 + : [denom]"rm"(__rm), bt_constraint(__rm) \ 101 101 + : "cc", "memory", KVM_ASM_SAFE_CLOBBERS); \ 102 102 + vector; \ 103 103 + }) 104 104 + 105 105 + #define guest_test_fastop_div(insn, type_t, __val1, __val2) \ 106 106 + ({ \ 107 107 + type_t _a = __val1, _d = __val1, rm = __val2; \ 108 108 + type_t a = _a, d = _d, ex_a = _a, ex_d = _d; \ 109 109 + uint64_t flags, ex_flags; \ 110 110 + uint8_t v, ex_v; \ 111 111 + \ 112 112 + ex_v = guest_execute_fastop_div(KVM_ASM_SAFE, insn, ex_a, ex_d, rm, ex_flags); \ 113 113 + v = guest_execute_fastop_div(KVM_ASM_SAFE_FEP, insn, a, d, rm, flags); \ 114 114 + \ 115 115 + GUEST_ASSERT_EQ(v, ex_v); \ 116 116 + __GUEST_ASSERT(v == ex_v, \ 117 117 + "Wanted vector 0x%x for '%s 0x%lx:0x%lx/0x%lx', got 0x%x", \ 118 118 + ex_v, insn, (uint64_t)_a, (uint64_t)_d, (uint64_t)rm, v); \ 119 119 + __GUEST_ASSERT(a == ex_a && d == ex_d, \ 120 120 + "Wanted 0x%lx:0x%lx for '%s 0x%lx:0x%lx/0x%lx', got 0x%lx:0x%lx",\ 121 121 + (uint64_t)ex_a, (uint64_t)ex_d, insn, (uint64_t)_a, \ 122 122 + (uint64_t)_d, (uint64_t)rm, (uint64_t)a, (uint64_t)d); \ 123 123 + __GUEST_ASSERT(v || ex_v || (flags == ex_flags), \ 124 124 + "Wanted flags 0x%lx for '%s 0x%lx:0x%lx/0x%lx', got 0x%lx", \ 125 125 + ex_flags, insn, (uint64_t)_a, (uint64_t)_d, (uint64_t)rm, flags);\ 93 126 }) 94 127 95 128 static const uint64_t vals[] = { ··· 154 115 guest_test_fastop_2("add" suffix, type_t, vals[i], vals[j]); \ 155 116 guest_test_fastop_2("adc" suffix, type_t, vals[i], vals[j]); \ 156 117 guest_test_fastop_2("and" suffix, type_t, vals[i], vals[j]); \ 118 118 + if (sizeof(type_t) != 1) { \ 157 119 guest_test_fastop_2("bsf" suffix, type_t, vals[i], vals[j]); \ 158 120 guest_test_fastop_2("bsr" suffix, type_t, vals[i], vals[j]); \ 159 121 guest_test_fastop_2("bt" suffix, type_t, vals[i], vals[j]); \ 160 122 guest_test_fastop_2("btc" suffix, type_t, vals[i], vals[j]); \ 161 123 guest_test_fastop_2("btr" suffix, type_t, vals[i], vals[j]); \ 162 124 guest_test_fastop_2("bts" suffix, type_t, vals[i], vals[j]); \ 163 163 - guest_test_fastop_2("cmp" suffix, type_t, vals[i], vals[j]); \ 164 125 guest_test_fastop_2("imul" suffix, type_t, vals[i], vals[j]); \ 126 126 + } \ 127 127 + guest_test_fastop_2("cmp" suffix, type_t, vals[i], vals[j]); \ 165 128 guest_test_fastop_2("or" suffix, type_t, vals[i], vals[j]); \ 166 129 guest_test_fastop_2("sbb" suffix, type_t, vals[i], vals[j]); \ 167 130 guest_test_fastop_2("sub" suffix, type_t, vals[i], vals[j]); \ ··· 177 136 guest_test_fastop_cl("sar" suffix, type_t, vals[i], vals[j]); \ 178 137 guest_test_fastop_cl("shl" suffix, type_t, vals[i], vals[j]); \ 179 138 guest_test_fastop_cl("shr" suffix, type_t, vals[i], vals[j]); \ 139 139 + \ 140 140 + guest_test_fastop_div("div" suffix, type_t, vals[i], vals[j]); \ 180 141 } \ 181 142 } \ 182 143 } while (0) 183 144 184 145 static void guest_code(void) 185 146 { 147 147 + guest_test_fastops(uint8_t, "b"); 186 148 guest_test_fastops(uint16_t, "w"); 187 149 guest_test_fastops(uint32_t, "l"); 188 150 guest_test_fastops(uint64_t, "q");

+1 -1

tools/testing/selftests/kvm/x86/hyperv_cpuid.c

reviewed

··· 45 45 46 46 TEST_ASSERT((entry->function >= 0x40000000) && 47 47 (entry->function <= 0x40000082), 48 48 - "function %x is our of supported range", 48 48 + "function %x is out of supported range", 49 49 entry->function); 50 50 51 51 TEST_ASSERT(entry->index == 0,

+8 -8

tools/testing/selftests/kvm/x86/hyperv_features.c

reviewed

··· 54 54 55 55 if (msr->fault_expected) 56 56 __GUEST_ASSERT(vector == GP_VECTOR, 57 57 - "Expected #GP on %sMSR(0x%x), got vector '0x%x'", 58 58 - msr->write ? "WR" : "RD", msr->idx, vector); 57 57 + "Expected #GP on %sMSR(0x%x), got %s", 58 58 + msr->write ? "WR" : "RD", msr->idx, ex_str(vector)); 59 59 else 60 60 __GUEST_ASSERT(!vector, 61 61 - "Expected success on %sMSR(0x%x), got vector '0x%x'", 62 62 - msr->write ? "WR" : "RD", msr->idx, vector); 61 61 + "Expected success on %sMSR(0x%x), got %s", 62 62 + msr->write ? "WR" : "RD", msr->idx, ex_str(vector)); 63 63 64 64 if (vector || is_write_only_msr(msr->idx)) 65 65 goto done; ··· 102 102 vector = __hyperv_hypercall(hcall->control, input, output, &res); 103 103 if (hcall->ud_expected) { 104 104 __GUEST_ASSERT(vector == UD_VECTOR, 105 105 - "Expected #UD for control '%lu', got vector '0x%x'", 106 106 - hcall->control, vector); 105 105 + "Expected #UD for control '%lu', got %s", 106 106 + hcall->control, ex_str(vector)); 107 107 } else { 108 108 __GUEST_ASSERT(!vector, 109 109 - "Expected no exception for control '%lu', got vector '0x%x'", 110 110 - hcall->control, vector); 109 109 + "Expected no exception for control '%lu', got %s", 110 110 + hcall->control, ex_str(vector)); 111 111 GUEST_ASSERT_EQ(res, hcall->expect); 112 112 } 113 113

+4 -4

tools/testing/selftests/kvm/x86/monitor_mwait_test.c

reviewed

··· 30 30 \ 31 31 if (fault_wanted) \ 32 32 __GUEST_ASSERT((vector) == UD_VECTOR, \ 33 33 - "Expected #UD on " insn " for testcase '0x%x', got '0x%x'", \ 34 34 - testcase, vector); \ 33 33 + "Expected #UD on " insn " for testcase '0x%x', got %s", \ 34 34 + testcase, ex_str(vector)); \ 35 35 else \ 36 36 __GUEST_ASSERT(!(vector), \ 37 37 - "Expected success on " insn " for testcase '0x%x', got '0x%x'", \ 38 38 - testcase, vector); \ 37 37 + "Expected success on " insn " for testcase '0x%x', got %s", \ 38 38 + testcase, ex_str(vector)); \ 39 39 } while (0) 40 40 41 41 static void guest_monitor_wait(void *arg)

+47 -20

tools/testing/selftests/kvm/x86/pmu_counters_test.c

reviewed

··· 75 75 [INTEL_ARCH_BRANCHES_RETIRED_INDEX] = { X86_PMU_FEATURE_BRANCH_INSNS_RETIRED, X86_PMU_FEATURE_NULL }, 76 76 [INTEL_ARCH_BRANCHES_MISPREDICTED_INDEX] = { X86_PMU_FEATURE_BRANCHES_MISPREDICTED, X86_PMU_FEATURE_NULL }, 77 77 [INTEL_ARCH_TOPDOWN_SLOTS_INDEX] = { X86_PMU_FEATURE_TOPDOWN_SLOTS, X86_PMU_FEATURE_TOPDOWN_SLOTS_FIXED }, 78 78 + [INTEL_ARCH_TOPDOWN_BE_BOUND_INDEX] = { X86_PMU_FEATURE_TOPDOWN_BE_BOUND, X86_PMU_FEATURE_NULL }, 79 79 + [INTEL_ARCH_TOPDOWN_BAD_SPEC_INDEX] = { X86_PMU_FEATURE_TOPDOWN_BAD_SPEC, X86_PMU_FEATURE_NULL }, 80 80 + [INTEL_ARCH_TOPDOWN_FE_BOUND_INDEX] = { X86_PMU_FEATURE_TOPDOWN_FE_BOUND, X86_PMU_FEATURE_NULL }, 81 81 + [INTEL_ARCH_TOPDOWN_RETIRING_INDEX] = { X86_PMU_FEATURE_TOPDOWN_RETIRING, X86_PMU_FEATURE_NULL }, 82 82 + [INTEL_ARCH_LBR_INSERTS_INDEX] = { X86_PMU_FEATURE_LBR_INSERTS, X86_PMU_FEATURE_NULL }, 78 83 }; 79 84 80 85 kvm_static_assert(ARRAY_SIZE(__intel_event_to_feature) == NR_INTEL_ARCH_EVENTS); ··· 163 158 164 159 switch (idx) { 165 160 case INTEL_ARCH_INSTRUCTIONS_RETIRED_INDEX: 166 166 - GUEST_ASSERT_EQ(count, NUM_INSNS_RETIRED); 161 161 + /* Relax precise count check due to VM-EXIT/VM-ENTRY overcount issue */ 162 162 + if (this_pmu_has_errata(INSTRUCTIONS_RETIRED_OVERCOUNT)) 163 163 + GUEST_ASSERT(count >= NUM_INSNS_RETIRED); 164 164 + else 165 165 + GUEST_ASSERT_EQ(count, NUM_INSNS_RETIRED); 167 166 break; 168 167 case INTEL_ARCH_BRANCHES_RETIRED_INDEX: 169 169 - GUEST_ASSERT_EQ(count, NUM_BRANCH_INSNS_RETIRED); 168 168 + /* Relax precise count check due to VM-EXIT/VM-ENTRY overcount issue */ 169 169 + if (this_pmu_has_errata(BRANCHES_RETIRED_OVERCOUNT)) 170 170 + GUEST_ASSERT(count >= NUM_BRANCH_INSNS_RETIRED); 171 171 + else 172 172 + GUEST_ASSERT_EQ(count, NUM_BRANCH_INSNS_RETIRED); 170 173 break; 171 174 case INTEL_ARCH_LLC_REFERENCES_INDEX: 172 175 case INTEL_ARCH_LLC_MISSES_INDEX: ··· 184 171 fallthrough; 185 172 case INTEL_ARCH_CPU_CYCLES_INDEX: 186 173 case INTEL_ARCH_REFERENCE_CYCLES_INDEX: 174 174 + case INTEL_ARCH_TOPDOWN_BE_BOUND_INDEX: 175 175 + case INTEL_ARCH_TOPDOWN_FE_BOUND_INDEX: 187 176 GUEST_ASSERT_NE(count, 0); 188 177 break; 189 178 case INTEL_ARCH_TOPDOWN_SLOTS_INDEX: 179 179 + case INTEL_ARCH_TOPDOWN_RETIRING_INDEX: 190 180 __GUEST_ASSERT(count >= NUM_INSNS_RETIRED, 191 181 "Expected top-down slots >= %u, got count = %lu", 192 182 NUM_INSNS_RETIRED, count); ··· 327 311 } 328 312 329 313 static void test_arch_events(uint8_t pmu_version, uint64_t perf_capabilities, 330 330 - uint8_t length, uint8_t unavailable_mask) 314 314 + uint8_t length, uint32_t unavailable_mask) 331 315 { 332 316 struct kvm_vcpu *vcpu; 333 317 struct kvm_vm *vm; ··· 335 319 /* Testing arch events requires a vPMU (there are no negative tests). */ 336 320 if (!pmu_version) 337 321 return; 322 322 + 323 323 + unavailable_mask &= GENMASK(X86_PROPERTY_PMU_EVENTS_MASK.hi_bit, 324 324 + X86_PROPERTY_PMU_EVENTS_MASK.lo_bit); 338 325 339 326 vm = pmu_vm_create_with_one_vcpu(&vcpu, guest_test_arch_events, 340 327 pmu_version, perf_capabilities); ··· 363 344 364 345 #define GUEST_ASSERT_PMC_MSR_ACCESS(insn, msr, expect_gp, vector) \ 365 346 __GUEST_ASSERT(expect_gp ? vector == GP_VECTOR : !vector, \ 366 366 - "Expected %s on " #insn "(0x%x), got vector %u", \ 367 367 - expect_gp ? "#GP" : "no fault", msr, vector) \ 347 347 + "Expected %s on " #insn "(0x%x), got %s", \ 348 348 + expect_gp ? "#GP" : "no fault", msr, ex_str(vector)) \ 368 349 369 350 #define GUEST_ASSERT_PMC_VALUE(insn, msr, val, expected) \ 370 351 __GUEST_ASSERT(val == expected, \ ··· 594 575 }; 595 576 596 577 /* 578 578 + * To keep the total runtime reasonable, test only a handful of select, 579 579 + * semi-arbitrary values for the mask of unavailable PMU events. Test 580 580 + * 0 (all events available) and all ones (no events available) as well 581 581 + * as alternating bit sequencues, e.g. to detect if KVM is checking the 582 582 + * wrong bit(s). 583 583 + */ 584 584 + const uint32_t unavailable_masks[] = { 585 585 + 0x0, 586 586 + 0xffffffffu, 587 587 + 0xaaaaaaaau, 588 588 + 0x55555555u, 589 589 + 0xf0f0f0f0u, 590 590 + 0x0f0f0f0fu, 591 591 + 0xa0a0a0a0u, 592 592 + 0x0a0a0a0au, 593 593 + 0x50505050u, 594 594 + 0x05050505u, 595 595 + }; 596 596 + 597 597 + /* 597 598 * Test up to PMU v5, which is the current maximum version defined by 598 599 * Intel, i.e. is the last version that is guaranteed to be backwards 599 600 * compatible with KVM's existing behavior. ··· 650 611 651 612 pr_info("Testing arch events, PMU version %u, perf_caps = %lx\n", 652 613 v, perf_caps[i]); 653 653 - /* 654 654 - * To keep the total runtime reasonable, test every 655 655 - * possible non-zero, non-reserved bitmap combination 656 656 - * only with the native PMU version and the full bit 657 657 - * vector length. 658 658 - */ 659 659 - if (v == pmu_version) { 660 660 - for (k = 1; k < (BIT(NR_INTEL_ARCH_EVENTS) - 1); k++) 661 661 - test_arch_events(v, perf_caps[i], NR_INTEL_ARCH_EVENTS, k); 662 662 - } 614 614 + 663 615 /* 664 616 * Test single bits for all PMU version and lengths up 665 617 * the number of events +1 (to verify KVM doesn't do ··· 659 629 * ones i.e. all events being available and unavailable. 660 630 */ 661 631 for (j = 0; j <= NR_INTEL_ARCH_EVENTS + 1; j++) { 662 662 - test_arch_events(v, perf_caps[i], j, 0); 663 663 - test_arch_events(v, perf_caps[i], j, 0xff); 664 664 - 665 665 - for (k = 0; k < NR_INTEL_ARCH_EVENTS; k++) 666 666 - test_arch_events(v, perf_caps[i], j, BIT(k)); 632 632 + for (k = 1; k < ARRAY_SIZE(unavailable_masks); k++) 633 633 + test_arch_events(v, perf_caps[i], j, unavailable_masks[k]); 667 634 } 668 635 669 636 pr_info("Testing GP counters, PMU version %u, perf_caps = %lx\n",

+3 -1

tools/testing/selftests/kvm/x86/pmu_event_filter_test.c

reviewed

··· 214 214 do { \ 215 215 uint64_t br = pmc_results.branches_retired; \ 216 216 uint64_t ir = pmc_results.instructions_retired; \ 217 217 + bool br_matched = this_pmu_has_errata(BRANCHES_RETIRED_OVERCOUNT) ? \ 218 218 + br >= NUM_BRANCHES : br == NUM_BRANCHES; \ 217 219 \ 218 218 - if (br && br != NUM_BRANCHES) \ 220 220 + if (br && !br_matched) \ 219 221 pr_info("%s: Branch instructions retired = %lu (expected %u)\n", \ 220 222 __func__, br, NUM_BRANCHES); \ 221 223 TEST_ASSERT(br, "%s: Branch instructions retired = %lu (expected > 0)", \

+4 -3

tools/testing/selftests/kvm/x86/vmx_pmu_caps_test.c

reviewed

··· 29 29 u64 pebs_baseline:1; 30 30 u64 perf_metrics:1; 31 31 u64 pebs_output_pt_available:1; 32 32 - u64 anythread_deprecated:1; 32 32 + u64 pebs_timing_info:1; 33 33 }; 34 34 u64 capabilities; 35 35 } host_cap; ··· 44 44 .pebs_arch_reg = 1, 45 45 .pebs_format = -1, 46 46 .pebs_baseline = 1, 47 47 + .pebs_timing_info = 1, 47 48 }; 48 49 49 50 static const union perf_capabilities format_caps = { ··· 57 56 uint8_t vector = wrmsr_safe(MSR_IA32_PERF_CAPABILITIES, val); 58 57 59 58 __GUEST_ASSERT(vector == GP_VECTOR, 60 60 - "Expected #GP for value '0x%lx', got vector '0x%x'", 61 61 - val, vector); 59 59 + "Expected #GP for value '0x%lx', got %s", 60 60 + val, ex_str(vector)); 62 61 } 63 62 64 63 static void guest_code(uint64_t current_val)

+2 -2

tools/testing/selftests/kvm/x86/xapic_state_test.c

reviewed

··· 120 120 __test_icr(x, icr | i); 121 121 122 122 /* 123 123 - * Send all flavors of IPIs to non-existent vCPUs. TODO: use number of 124 124 - * vCPUs, not vcpu.id + 1. Arbitrarily use vector 0xff. 123 123 + * Send all flavors of IPIs to non-existent vCPUs. Arbitrarily use 124 124 + * vector 0xff. 125 125 */ 126 126 icr = APIC_INT_ASSERT | 0xff; 127 127 for (i = 0; i < 0xff; i++) {

+6 -6

tools/testing/selftests/kvm/x86/xcr0_cpuid_test.c

reviewed

··· 81 81 82 82 vector = xsetbv_safe(0, XFEATURE_MASK_FP); 83 83 __GUEST_ASSERT(!vector, 84 84 - "Expected success on XSETBV(FP), got vector '0x%x'", 85 85 - vector); 84 84 + "Expected success on XSETBV(FP), got %s", 85 85 + ex_str(vector)); 86 86 87 87 vector = xsetbv_safe(0, supported_xcr0); 88 88 __GUEST_ASSERT(!vector, 89 89 - "Expected success on XSETBV(0x%lx), got vector '0x%x'", 90 90 - supported_xcr0, vector); 89 89 + "Expected success on XSETBV(0x%lx), got %s", 90 90 + supported_xcr0, ex_str(vector)); 91 91 92 92 for (i = 0; i < 64; i++) { 93 93 if (supported_xcr0 & BIT_ULL(i)) ··· 95 95 96 96 vector = xsetbv_safe(0, supported_xcr0 | BIT_ULL(i)); 97 97 __GUEST_ASSERT(vector == GP_VECTOR, 98 98 - "Expected #GP on XSETBV(0x%llx), supported XCR0 = %lx, got vector '0x%x'", 99 99 - BIT_ULL(i), supported_xcr0, vector); 98 98 + "Expected #GP on XSETBV(0x%llx), supported XCR0 = %lx, got %s", 99 99 + BIT_ULL(i), supported_xcr0, ex_str(vector)); 100 100 } 101 101 102 102 GUEST_DONE();

+1 -2

tools/testing/selftests/rseq/rseq-riscv.h

reviewed

··· 8 8 * exception when executed in all modes. 9 9 */ 10 10 #include <endian.h> 11 11 + #include <asm/fence.h> 11 12 12 13 #if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN) 13 14 #define RSEQ_SIG 0xf1401073 /* csrr mhartid, x0 */ ··· 25 24 #define REG_L __REG_SEL("ld ", "lw ") 26 25 #define REG_S __REG_SEL("sd ", "sw ") 27 26 28 28 - #define RISCV_FENCE(p, s) \ 29 29 - __asm__ __volatile__ ("fence " #p "," #s : : : "memory") 30 27 #define rseq_smp_mb() RISCV_FENCE(rw, rw) 31 28 #define rseq_smp_rmb() RISCV_FENCE(r, r) 32 29 #define rseq_smp_wmb() RISCV_FENCE(w, w)

+7 -8

virt/kvm/Kconfig

reviewed

··· 112 112 depends on KVM_GENERIC_MMU_NOTIFIER 113 113 bool 114 114 115 115 - config KVM_PRIVATE_MEM 115 115 + config KVM_GUEST_MEMFD 116 116 select XARRAY_MULTI 117 117 - bool 118 118 - 119 119 - config KVM_GENERIC_PRIVATE_MEM 120 120 - select KVM_GENERIC_MEMORY_ATTRIBUTES 121 121 - select KVM_PRIVATE_MEM 122 117 bool 123 118 124 119 config HAVE_KVM_ARCH_GMEM_PREPARE 125 120 bool 126 126 - depends on KVM_PRIVATE_MEM 121 121 + depends on KVM_GUEST_MEMFD 127 122 128 123 config HAVE_KVM_ARCH_GMEM_INVALIDATE 129 124 bool 130 130 - depends on KVM_PRIVATE_MEM 125 125 + depends on KVM_GUEST_MEMFD 126 126 + 127 127 + config HAVE_KVM_ARCH_GMEM_POPULATE 128 128 + bool 129 129 + depends on KVM_GUEST_MEMFD

+1 -1

virt/kvm/Makefile.kvm

reviewed

··· 12 12 kvm-$(CONFIG_HAVE_KVM_IRQ_ROUTING) += $(KVM)/irqchip.o 13 13 kvm-$(CONFIG_HAVE_KVM_DIRTY_RING) += $(KVM)/dirty_ring.o 14 14 kvm-$(CONFIG_HAVE_KVM_PFNCACHE) += $(KVM)/pfncache.o 15 15 - kvm-$(CONFIG_KVM_PRIVATE_MEM) += $(KVM)/guest_memfd.o 15 15 + kvm-$(CONFIG_KVM_GUEST_MEMFD) += $(KVM)/guest_memfd.o

+1 -1

virt/kvm/async_pf.c

reviewed

··· 192 192 * do alloc nowait since if we are going to sleep anyway we 193 193 * may as well sleep faulting in page 194 194 */ 195 195 - work = kmem_cache_zalloc(async_pf_cache, GFP_NOWAIT | __GFP_NOWARN); 195 195 + work = kmem_cache_zalloc(async_pf_cache, GFP_NOWAIT); 196 196 if (!work) 197 197 return false; 198 198

+79 -2

virt/kvm/guest_memfd.c

reviewed

··· 312 312 return gfn - slot->base_gfn + slot->gmem.pgoff; 313 313 } 314 314 315 315 + static bool kvm_gmem_supports_mmap(struct inode *inode) 316 316 + { 317 317 + const u64 flags = (u64)inode->i_private; 318 318 + 319 319 + return flags & GUEST_MEMFD_FLAG_MMAP; 320 320 + } 321 321 + 322 322 + static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf) 323 323 + { 324 324 + struct inode *inode = file_inode(vmf->vma->vm_file); 325 325 + struct folio *folio; 326 326 + vm_fault_t ret = VM_FAULT_LOCKED; 327 327 + 328 328 + if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode)) 329 329 + return VM_FAULT_SIGBUS; 330 330 + 331 331 + folio = kvm_gmem_get_folio(inode, vmf->pgoff); 332 332 + if (IS_ERR(folio)) { 333 333 + int err = PTR_ERR(folio); 334 334 + 335 335 + if (err == -EAGAIN) 336 336 + return VM_FAULT_RETRY; 337 337 + 338 338 + return vmf_error(err); 339 339 + } 340 340 + 341 341 + if (WARN_ON_ONCE(folio_test_large(folio))) { 342 342 + ret = VM_FAULT_SIGBUS; 343 343 + goto out_folio; 344 344 + } 345 345 + 346 346 + if (!folio_test_uptodate(folio)) { 347 347 + clear_highpage(folio_page(folio, 0)); 348 348 + kvm_gmem_mark_prepared(folio); 349 349 + } 350 350 + 351 351 + vmf->page = folio_file_page(folio, vmf->pgoff); 352 352 + 353 353 + out_folio: 354 354 + if (ret != VM_FAULT_LOCKED) { 355 355 + folio_unlock(folio); 356 356 + folio_put(folio); 357 357 + } 358 358 + 359 359 + return ret; 360 360 + } 361 361 + 362 362 + static const struct vm_operations_struct kvm_gmem_vm_ops = { 363 363 + .fault = kvm_gmem_fault_user_mapping, 364 364 + }; 365 365 + 366 366 + static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma) 367 367 + { 368 368 + if (!kvm_gmem_supports_mmap(file_inode(file))) 369 369 + return -ENODEV; 370 370 + 371 371 + if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) != 372 372 + (VM_SHARED | VM_MAYSHARE)) { 373 373 + return -EINVAL; 374 374 + } 375 375 + 376 376 + vma->vm_ops = &kvm_gmem_vm_ops; 377 377 + 378 378 + return 0; 379 379 + } 380 380 + 315 381 static struct file_operations kvm_gmem_fops = { 382 382 + .mmap = kvm_gmem_mmap, 316 383 .open = generic_file_open, 317 384 .release = kvm_gmem_release, 318 385 .fallocate = kvm_gmem_fallocate, ··· 458 391 .setattr = kvm_gmem_setattr, 459 392 }; 460 393 394 394 + bool __weak kvm_arch_supports_gmem_mmap(struct kvm *kvm) 395 395 + { 396 396 + return true; 397 397 + } 398 398 + 461 399 static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) 462 400 { 463 401 const char *anon_name = "[kvm-gmem]"; ··· 524 452 u64 flags = args->flags; 525 453 u64 valid_flags = 0; 526 454 455 455 + if (kvm_arch_supports_gmem_mmap(kvm)) 456 456 + valid_flags |= GUEST_MEMFD_FLAG_MMAP; 457 457 + 527 458 if (flags & ~valid_flags) 528 459 return -EINVAL; 529 460 ··· 583 508 */ 584 509 WRITE_ONCE(slot->gmem.file, file); 585 510 slot->gmem.pgoff = start; 511 511 + if (kvm_gmem_supports_mmap(inode)) 512 512 + slot->flags |= KVM_MEMSLOT_GMEM_ONLY; 586 513 587 514 xa_store_range(&gmem->bindings, start, end - 1, slot, GFP_KERNEL); 588 515 filemap_invalidate_unlock(inode->i_mapping); ··· 704 627 } 705 628 EXPORT_SYMBOL_GPL(kvm_gmem_get_pfn); 706 629 707 707 - #ifdef CONFIG_KVM_GENERIC_PRIVATE_MEM 630 630 + #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE 708 631 long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long npages, 709 632 kvm_gmem_populate_cb post_populate, void *opaque) 710 633 { ··· 720 643 return -EINVAL; 721 644 722 645 slot = gfn_to_memslot(kvm, start_gfn); 723 723 - if (!kvm_slot_can_be_private(slot)) 646 646 + if (!kvm_slot_has_gmem(slot)) 724 647 return -EINVAL; 725 648 726 649 file = kvm_gmem_get_file(slot);

+42 -13

virt/kvm/kvm_main.c

reviewed

··· 331 331 * All current use cases for flushing the TLBs for a specific memslot 332 332 * are related to dirty logging, and many do the TLB flush out of 333 333 * mmu_lock. The interaction between the various operations on memslot 334 334 - * must be serialized by slots_locks to ensure the TLB flush from one 334 334 + * must be serialized by slots_lock to ensure the TLB flush from one 335 335 * operation is observed by any other operation on the same memslot. 336 336 */ 337 337 lockdep_assert_held(&kvm->slots_lock); ··· 1103 1103 { 1104 1104 } 1105 1105 1106 1106 + /* Called only on cleanup and destruction paths when there are no users. */ 1107 1107 + static inline struct kvm_io_bus *kvm_get_bus_for_destruction(struct kvm *kvm, 1108 1108 + enum kvm_bus idx) 1109 1109 + { 1110 1110 + return rcu_dereference_protected(kvm->buses[idx], 1111 1111 + !refcount_read(&kvm->users_count)); 1112 1112 + } 1113 1113 + 1106 1114 static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) 1107 1115 { 1108 1116 struct kvm *kvm = kvm_arch_alloc_vm(); ··· 1236 1228 out_err_no_arch_destroy_vm: 1237 1229 WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count)); 1238 1230 for (i = 0; i < KVM_NR_BUSES; i++) 1239 1239 - kfree(kvm_get_bus(kvm, i)); 1231 1231 + kfree(kvm_get_bus_for_destruction(kvm, i)); 1240 1232 kvm_free_irq_routing(kvm); 1241 1233 out_err_no_irq_routing: 1242 1234 cleanup_srcu_struct(&kvm->irq_srcu); ··· 1284 1276 1285 1277 kvm_free_irq_routing(kvm); 1286 1278 for (i = 0; i < KVM_NR_BUSES; i++) { 1287 1287 - struct kvm_io_bus *bus = kvm_get_bus(kvm, i); 1279 1279 + struct kvm_io_bus *bus = kvm_get_bus_for_destruction(kvm, i); 1288 1280 1289 1281 if (bus) 1290 1282 kvm_io_bus_destroy(bus); ··· 1320 1312 kvm_free_memslots(kvm, &kvm->__memslots[i][1]); 1321 1313 } 1322 1314 cleanup_srcu_struct(&kvm->irq_srcu); 1315 1315 + srcu_barrier(&kvm->srcu); 1323 1316 cleanup_srcu_struct(&kvm->srcu); 1324 1317 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES 1325 1318 xa_destroy(&kvm->mem_attr_array); ··· 1597 1588 { 1598 1589 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 1599 1590 1600 1600 - if (kvm_arch_has_private_mem(kvm)) 1591 1591 + if (IS_ENABLED(CONFIG_KVM_GUEST_MEMFD)) 1601 1592 valid_flags |= KVM_MEM_GUEST_MEMFD; 1602 1593 1603 1594 /* Dirty logging private memory is not currently supported. */ ··· 4924 4915 case KVM_CAP_MEMORY_ATTRIBUTES: 4925 4916 return kvm_supported_mem_attributes(kvm); 4926 4917 #endif 4927 4927 - #ifdef CONFIG_KVM_PRIVATE_MEM 4918 4918 + #ifdef CONFIG_KVM_GUEST_MEMFD 4928 4919 case KVM_CAP_GUEST_MEMFD: 4929 4929 - return !kvm || kvm_arch_has_private_mem(kvm); 4920 4920 + return 1; 4921 4921 + case KVM_CAP_GUEST_MEMFD_MMAP: 4922 4922 + return !kvm || kvm_arch_supports_gmem_mmap(kvm); 4930 4923 #endif 4931 4924 default: 4932 4925 break; ··· 5363 5352 case KVM_GET_STATS_FD: 5364 5353 r = kvm_vm_ioctl_get_stats_fd(kvm); 5365 5354 break; 5366 5366 - #ifdef CONFIG_KVM_PRIVATE_MEM 5355 5355 + #ifdef CONFIG_KVM_GUEST_MEMFD 5367 5356 case KVM_CREATE_GUEST_MEMFD: { 5368 5357 struct kvm_create_guest_memfd guest_memfd; 5369 5358 ··· 5854 5843 return -EOPNOTSUPP; 5855 5844 } 5856 5845 5846 5846 + static struct kvm_io_bus *kvm_get_bus_srcu(struct kvm *kvm, enum kvm_bus idx) 5847 5847 + { 5848 5848 + /* 5849 5849 + * Ensure that any updates to kvm_buses[] observed by the previous vCPU 5850 5850 + * machine instruction are also visible to the vCPU machine instruction 5851 5851 + * that triggered this call. 5852 5852 + */ 5853 5853 + smp_mb__after_srcu_read_lock(); 5854 5854 + 5855 5855 + return srcu_dereference(kvm->buses[idx], &kvm->srcu); 5856 5856 + } 5857 5857 + 5857 5858 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 5858 5859 int len, const void *val) 5859 5860 { ··· 5878 5855 .len = len, 5879 5856 }; 5880 5857 5881 5881 - bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 5858 5858 + bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx); 5882 5859 if (!bus) 5883 5860 return -ENOMEM; 5884 5861 r = __kvm_io_bus_write(vcpu, bus, &range, val); ··· 5897 5874 .len = len, 5898 5875 }; 5899 5876 5900 5900 - bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 5877 5877 + bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx); 5901 5878 if (!bus) 5902 5879 return -ENOMEM; 5903 5880 ··· 5947 5924 .len = len, 5948 5925 }; 5949 5926 5950 5950 - bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 5927 5927 + bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx); 5951 5928 if (!bus) 5952 5929 return -ENOMEM; 5953 5930 r = __kvm_io_bus_read(vcpu, bus, &range, val); 5954 5931 return r < 0 ? r : 0; 5955 5932 } 5956 5933 EXPORT_SYMBOL_GPL(kvm_io_bus_read); 5934 5934 + 5935 5935 + static void __free_bus(struct rcu_head *rcu) 5936 5936 + { 5937 5937 + struct kvm_io_bus *bus = container_of(rcu, struct kvm_io_bus, rcu); 5938 5938 + 5939 5939 + kfree(bus); 5940 5940 + } 5957 5941 5958 5942 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 5959 5943 int len, struct kvm_io_device *dev) ··· 6000 5970 memcpy(new_bus->range + i + 1, bus->range + i, 6001 5971 (bus->dev_count - i) * sizeof(struct kvm_io_range)); 6002 5972 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 6003 6003 - synchronize_srcu_expedited(&kvm->srcu); 6004 6004 - kfree(bus); 5973 5973 + call_srcu(&kvm->srcu, &bus->rcu, __free_bus); 6005 5974 6006 5975 return 0; 6007 5976 } ··· 6062 6033 6063 6034 srcu_idx = srcu_read_lock(&kvm->srcu); 6064 6035 6065 6065 - bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 6036 6036 + bus = kvm_get_bus_srcu(kvm, bus_idx); 6066 6037 if (!bus) 6067 6038 goto out_unlock; 6068 6039

+2 -2

virt/kvm/kvm_mm.h

reviewed

··· 67 67 } 68 68 #endif /* HAVE_KVM_PFNCACHE */ 69 69 70 70 - #ifdef CONFIG_KVM_PRIVATE_MEM 70 70 + #ifdef CONFIG_KVM_GUEST_MEMFD 71 71 void kvm_gmem_init(struct module *module); 72 72 int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args); 73 73 int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, ··· 91 91 { 92 92 WARN_ON_ONCE(1); 93 93 } 94 94 - #endif /* CONFIG_KVM_PRIVATE_MEM */ 94 94 + #endif /* CONFIG_KVM_GUEST_MEMFD */ 95 95 96 96 #endif /* __KVM_MM_H__ */