+1 -1

Documentation/virt/kvm/mmu.rst

reviewed

··· 455 455 number, it will ignore the cached MMIO information and handle the page 456 456 fault through the slow path. 457 457 458 458 - Since only 19 bits are used to store generation-number on mmio spte, all 458 458 + Since only 18 bits are used to store generation-number on mmio spte, all 459 459 pages are zapped when there is an overflow. 460 460 461 461 Unfortunately, a single memory access might access kvm_memslots(kvm) multiple

+1

arch/arm64/include/asm/esr.h

reviewed

··· 104 104 /* Shared ISS fault status code(IFSC/DFSC) for Data/Instruction aborts */ 105 105 #define ESR_ELx_FSC (0x3F) 106 106 #define ESR_ELx_FSC_TYPE (0x3C) 107 107 + #define ESR_ELx_FSC_LEVEL (0x03) 107 108 #define ESR_ELx_FSC_EXTABT (0x10) 108 109 #define ESR_ELx_FSC_SERROR (0x11) 109 110 #define ESR_ELx_FSC_ACCESS (0x08)

+5

arch/arm64/include/asm/kvm_emulate.h

reviewed

··· 350 350 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC_TYPE; 351 351 } 352 352 353 353 + static __always_inline u8 kvm_vcpu_trap_get_fault_level(const struct kvm_vcpu *vcpu) 354 354 + { 355 355 + return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC_LEVEL; 356 356 + } 357 357 + 353 358 static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu) 354 359 { 355 360 switch (kvm_vcpu_trap_get_fault(vcpu)) {

+16 -1

arch/arm64/kvm/hyp/pgtable.c

reviewed

··· 470 470 if (!kvm_block_mapping_supported(addr, end, phys, level)) 471 471 return false; 472 472 473 473 + /* 474 474 + * If the PTE was already valid, drop the refcount on the table 475 475 + * early, as it will be bumped-up again in stage2_map_walk_leaf(). 476 476 + * This ensures that the refcount stays constant across a valid to 477 477 + * valid PTE update. 478 478 + */ 479 479 + if (kvm_pte_valid(*ptep)) 480 480 + put_page(virt_to_page(ptep)); 481 481 + 473 482 if (kvm_set_valid_leaf_pte(ptep, phys, data->attr, level)) 474 483 goto out; 475 484 ··· 502 493 return 0; 503 494 504 495 kvm_set_invalid_pte(ptep); 505 505 - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, 0); 496 496 + 497 497 + /* 498 498 + * Invalidate the whole stage-2, as we may have numerous leaf 499 499 + * entries below us which would otherwise need invalidating 500 500 + * individually. 501 501 + */ 502 502 + kvm_call_hyp(__kvm_tlb_flush_vmid, data->mmu); 506 503 data->anchor = ptep; 507 504 return 0; 508 505 }

+9 -2

arch/arm64/kvm/mmu.c

reviewed

··· 754 754 gfn_t gfn; 755 755 kvm_pfn_t pfn; 756 756 bool logging_active = memslot_is_logging(memslot); 757 757 - unsigned long vma_pagesize; 757 757 + unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu); 758 758 + unsigned long vma_pagesize, fault_granule; 758 759 enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; 759 760 struct kvm_pgtable *pgt; 760 761 762 762 + fault_granule = 1UL << ARM64_HW_PGTABLE_LEVEL_SHIFT(fault_level); 761 763 write_fault = kvm_is_write_fault(vcpu); 762 764 exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu); 763 765 VM_BUG_ON(write_fault && exec_fault); ··· 898 896 else if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC)) 899 897 prot |= KVM_PGTABLE_PROT_X; 900 898 901 901 - if (fault_status == FSC_PERM && !(logging_active && writable)) { 899 899 + /* 900 900 + * Under the premise of getting a FSC_PERM fault, we just need to relax 901 901 + * permissions only if vma_pagesize equals fault_granule. Otherwise, 902 902 + * kvm_pgtable_stage2_map() should be called to change block size. 903 903 + */ 904 904 + if (fault_status == FSC_PERM && vma_pagesize == fault_granule) { 902 905 ret = kvm_pgtable_stage2_relax_perms(pgt, fault_ipa, prot); 903 906 } else { 904 907 ret = kvm_pgtable_stage2_map(pgt, fault_ipa, vma_pagesize,

+2 -2

arch/x86/kvm/mmu/spte.c

reviewed

··· 40 40 WARN_ON(gen & ~MMIO_SPTE_GEN_MASK); 41 41 BUILD_BUG_ON((MMIO_SPTE_GEN_HIGH_MASK | MMIO_SPTE_GEN_LOW_MASK) & SPTE_SPECIAL_MASK); 42 42 43 43 - mask = (gen << MMIO_SPTE_GEN_LOW_START) & MMIO_SPTE_GEN_LOW_MASK; 44 44 - mask |= (gen << MMIO_SPTE_GEN_HIGH_START) & MMIO_SPTE_GEN_HIGH_MASK; 43 43 + mask = (gen << MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_SPTE_GEN_LOW_MASK; 44 44 + mask |= (gen << MMIO_SPTE_GEN_HIGH_SHIFT) & MMIO_SPTE_GEN_HIGH_MASK; 45 45 return mask; 46 46 } 47 47

+18 -7

arch/x86/kvm/mmu/spte.h

reviewed

··· 56 56 #define SPTE_MMU_WRITEABLE (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1)) 57 57 58 58 /* 59 59 - * Due to limited space in PTEs, the MMIO generation is a 19 bit subset of 59 59 + * Due to limited space in PTEs, the MMIO generation is a 18 bit subset of 60 60 * the memslots generation and is derived as follows: 61 61 * 62 62 * Bits 0-8 of the MMIO generation are propagated to spte bits 3-11 63 63 - * Bits 9-18 of the MMIO generation are propagated to spte bits 52-61 63 63 + * Bits 9-17 of the MMIO generation are propagated to spte bits 54-62 64 64 * 65 65 * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in 66 66 * the MMIO generation number, as doing so would require stealing a bit from ··· 69 69 * requires a full MMU zap). The flag is instead explicitly queried when 70 70 * checking for MMIO spte cache hits. 71 71 */ 72 72 - #define MMIO_SPTE_GEN_MASK GENMASK_ULL(17, 0) 73 72 74 73 #define MMIO_SPTE_GEN_LOW_START 3 75 74 #define MMIO_SPTE_GEN_LOW_END 11 76 76 - #define MMIO_SPTE_GEN_LOW_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \ 77 77 - MMIO_SPTE_GEN_LOW_START) 78 75 79 76 #define MMIO_SPTE_GEN_HIGH_START PT64_SECOND_AVAIL_BITS_SHIFT 80 77 #define MMIO_SPTE_GEN_HIGH_END 62 78 78 + 79 79 + #define MMIO_SPTE_GEN_LOW_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \ 80 80 + MMIO_SPTE_GEN_LOW_START) 81 81 #define MMIO_SPTE_GEN_HIGH_MASK GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \ 82 82 MMIO_SPTE_GEN_HIGH_START) 83 83 + 84 84 + #define MMIO_SPTE_GEN_LOW_BITS (MMIO_SPTE_GEN_LOW_END - MMIO_SPTE_GEN_LOW_START + 1) 85 85 + #define MMIO_SPTE_GEN_HIGH_BITS (MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1) 86 86 + 87 87 + /* remember to adjust the comment above as well if you change these */ 88 88 + static_assert(MMIO_SPTE_GEN_LOW_BITS == 9 && MMIO_SPTE_GEN_HIGH_BITS == 9); 89 89 + 90 90 + #define MMIO_SPTE_GEN_LOW_SHIFT (MMIO_SPTE_GEN_LOW_START - 0) 91 91 + #define MMIO_SPTE_GEN_HIGH_SHIFT (MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS) 92 92 + 93 93 + #define MMIO_SPTE_GEN_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_BITS + MMIO_SPTE_GEN_HIGH_BITS - 1, 0) 83 94 84 95 extern u64 __read_mostly shadow_nx_mask; 85 96 extern u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */ ··· 239 228 { 240 229 u64 gen; 241 230 242 242 - gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_START; 243 243 - gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_START; 231 231 + gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_SHIFT; 232 232 + gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_SHIFT; 244 233 return gen; 245 234 } 246 235

+2 -2

arch/x86/kvm/mmu/tdp_mmu.c

reviewed

··· 66 66 67 67 void kvm_tdp_mmu_free_root(struct kvm *kvm, struct kvm_mmu_page *root) 68 68 { 69 69 - gfn_t max_gfn = 1ULL << (boot_cpu_data.x86_phys_bits - PAGE_SHIFT); 69 69 + gfn_t max_gfn = 1ULL << (shadow_phys_bits - PAGE_SHIFT); 70 70 71 71 lockdep_assert_held(&kvm->mmu_lock); 72 72 ··· 456 456 457 457 void kvm_tdp_mmu_zap_all(struct kvm *kvm) 458 458 { 459 459 - gfn_t max_gfn = 1ULL << (boot_cpu_data.x86_phys_bits - PAGE_SHIFT); 459 459 + gfn_t max_gfn = 1ULL << (shadow_phys_bits - PAGE_SHIFT); 460 460 bool flush; 461 461 462 462 flush = kvm_tdp_mmu_zap_gfn_range(kvm, 0, max_gfn);

+2 -2

arch/x86/kvm/svm/svm.c

reviewed

··· 530 530 531 531 static void svm_cpu_uninit(int cpu) 532 532 { 533 533 - struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id()); 533 533 + struct svm_cpu_data *sd = per_cpu(svm_data, cpu); 534 534 535 535 if (!sd) 536 536 return; 537 537 538 538 - per_cpu(svm_data, raw_smp_processor_id()) = NULL; 538 538 + per_cpu(svm_data, cpu) = NULL; 539 539 kfree(sd->sev_vmcbs); 540 540 __free_page(sd->save_area); 541 541 kfree(sd);

+5 -1

tools/kvm/kvm_stat/kvm_stat

reviewed

··· 742 742 The fields are all available KVM debugfs files 743 743 744 744 """ 745 745 - return self.walkdir(PATH_DEBUGFS_KVM)[2] 745 745 + exempt_list = ['halt_poll_fail_ns', 'halt_poll_success_ns'] 746 746 + fields = [field for field in self.walkdir(PATH_DEBUGFS_KVM)[2] 747 747 + if field not in exempt_list] 748 748 + 749 749 + return fields 746 750 747 751 def update_fields(self, fields_filter): 748 752 """Refresh fields, applying fields_filter"""

+13 -4

tools/testing/selftests/kvm/set_memory_region_test.c

reviewed

··· 156 156 GUEST_SYNC(0); 157 157 158 158 /* 159 159 - * Spin until the memory region is moved to a misaligned address. This 160 160 - * may or may not trigger MMIO, as the window where the memslot is 161 161 - * invalid is quite small. 159 159 + * Spin until the memory region starts getting moved to a 160 160 + * misaligned address. 161 161 + * Every region move may or may not trigger MMIO, as the 162 162 + * window where the memslot is invalid is usually quite small. 162 163 */ 163 164 val = guest_spin_on_val(0); 164 165 GUEST_ASSERT_1(val == 1 || val == MMIO_VAL, val); 165 166 166 166 - /* Spin until the memory region is realigned. */ 167 167 + /* Spin until the misaligning memory region move completes. */ 168 168 + val = guest_spin_on_val(MMIO_VAL); 169 169 + GUEST_ASSERT_1(val == 1 || val == 0, val); 170 170 + 171 171 + /* Spin until the memory region starts to get re-aligned. */ 172 172 + val = guest_spin_on_val(0); 173 173 + GUEST_ASSERT_1(val == 1 || val == MMIO_VAL, val); 174 174 + 175 175 + /* Spin until the re-aligning memory region move completes. */ 167 176 val = guest_spin_on_val(MMIO_VAL); 168 177 GUEST_ASSERT_1(val == 1, val); 169 178