Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

+10

arch/arm/include/asm/kvm_host.h

··· 48 48 #define KVM_REQ_SLEEP \ 49 49 KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 50 50 #define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) 51 + #define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(2) 51 52 52 53 DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use); 53 54 ··· 148 147 149 148 typedef struct kvm_cpu_context kvm_cpu_context_t; 150 149 150 + struct vcpu_reset_state { 151 + unsigned long pc; 152 + unsigned long r0; 153 + bool be; 154 + bool reset; 155 + }; 156 + 151 157 struct kvm_vcpu_arch { 152 158 struct kvm_cpu_context ctxt; 153 159 ··· 193 185 194 186 /* Cache some mmu pages needed inside spinlock regions */ 195 187 struct kvm_mmu_memory_cache mmu_page_cache; 188 + 189 + struct vcpu_reset_state reset_state; 196 190 197 191 /* Detect first run of a vcpu */ 198 192 bool has_run_once;

+5

arch/arm/include/asm/stage2_pgtable.h

··· 76 76 #define S2_PMD_MASK PMD_MASK 77 77 #define S2_PMD_SIZE PMD_SIZE 78 78 79 + static inline bool kvm_stage2_has_pmd(struct kvm *kvm) 80 + { 81 + return true; 82 + } 83 + 79 84 #endif /* __ARM_S2_PGTABLE_H_ */

+2 -2

arch/arm/kvm/coproc.c

··· 1450 1450 reset_coproc_regs(vcpu, table, num); 1451 1451 1452 1452 for (num = 1; num < NR_CP15_REGS; num++) 1453 - if (vcpu_cp15(vcpu, num) == 0x42424242) 1454 - panic("Didn't reset vcpu_cp15(vcpu, %zi)", num); 1453 + WARN(vcpu_cp15(vcpu, num) == 0x42424242, 1454 + "Didn't reset vcpu_cp15(vcpu, %zi)", num); 1455 1455 }

+24

arch/arm/kvm/reset.c

··· 26 26 #include <asm/cputype.h> 27 27 #include <asm/kvm_arm.h> 28 28 #include <asm/kvm_coproc.h> 29 + #include <asm/kvm_emulate.h> 29 30 30 31 #include <kvm/arm_arch_timer.h> 31 32 ··· 69 68 70 69 /* Reset CP15 registers */ 71 70 kvm_reset_coprocs(vcpu); 71 + 72 + /* 73 + * Additional reset state handling that PSCI may have imposed on us. 74 + * Must be done after all the sys_reg reset. 75 + */ 76 + if (READ_ONCE(vcpu->arch.reset_state.reset)) { 77 + unsigned long target_pc = vcpu->arch.reset_state.pc; 78 + 79 + /* Gracefully handle Thumb2 entry point */ 80 + if (target_pc & 1) { 81 + target_pc &= ~1UL; 82 + vcpu_set_thumb(vcpu); 83 + } 84 + 85 + /* Propagate caller endianness */ 86 + if (vcpu->arch.reset_state.be) 87 + kvm_vcpu_set_be(vcpu); 88 + 89 + *vcpu_pc(vcpu) = target_pc; 90 + vcpu_set_reg(vcpu, 0, vcpu->arch.reset_state.r0); 91 + 92 + vcpu->arch.reset_state.reset = false; 93 + } 72 94 73 95 /* Reset arch_timer context */ 74 96 return kvm_timer_vcpu_reset(vcpu);

+11

arch/arm64/include/asm/kvm_host.h

··· 48 48 #define KVM_REQ_SLEEP \ 49 49 KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 50 50 #define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) 51 + #define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(2) 51 52 52 53 DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use); 53 54 ··· 209 208 210 209 typedef struct kvm_cpu_context kvm_cpu_context_t; 211 210 211 + struct vcpu_reset_state { 212 + unsigned long pc; 213 + unsigned long r0; 214 + bool be; 215 + bool reset; 216 + }; 217 + 212 218 struct kvm_vcpu_arch { 213 219 struct kvm_cpu_context ctxt; 214 220 ··· 304 296 305 297 /* Virtual SError ESR to restore when HCR_EL2.VSE is set */ 306 298 u64 vsesr_el2; 299 + 300 + /* Additional reset state */ 301 + struct vcpu_reset_state reset_state; 307 302 308 303 /* True when deferrable sysregs are loaded on the physical CPU, 309 304 * see kvm_vcpu_load_sysregs and kvm_vcpu_put_sysregs. */

+5

arch/arm64/kvm/hyp/switch.c

··· 23 23 #include <kvm/arm_psci.h> 24 24 25 25 #include <asm/cpufeature.h> 26 + #include <asm/kprobes.h> 26 27 #include <asm/kvm_asm.h> 27 28 #include <asm/kvm_emulate.h> 28 29 #include <asm/kvm_host.h> ··· 108 107 109 108 write_sysreg(kvm_get_hyp_vector(), vbar_el1); 110 109 } 110 + NOKPROBE_SYMBOL(activate_traps_vhe); 111 111 112 112 static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu) 113 113 { ··· 156 154 write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1); 157 155 write_sysreg(vectors, vbar_el1); 158 156 } 157 + NOKPROBE_SYMBOL(deactivate_traps_vhe); 159 158 160 159 static void __hyp_text __deactivate_traps_nvhe(void) 161 160 { ··· 516 513 517 514 return exit_code; 518 515 } 516 + NOKPROBE_SYMBOL(kvm_vcpu_run_vhe); 519 517 520 518 /* Switch to the guest for legacy non-VHE systems */ 521 519 int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu) ··· 624 620 read_sysreg_el2(esr), read_sysreg_el2(far), 625 621 read_sysreg(hpfar_el2), par, vcpu); 626 622 } 623 + NOKPROBE_SYMBOL(__hyp_call_panic_vhe); 627 624 628 625 void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt) 629 626 {

+5

arch/arm64/kvm/hyp/sysreg-sr.c

··· 18 18 #include <linux/compiler.h> 19 19 #include <linux/kvm_host.h> 20 20 21 + #include <asm/kprobes.h> 21 22 #include <asm/kvm_asm.h> 22 23 #include <asm/kvm_emulate.h> 23 24 #include <asm/kvm_hyp.h> ··· 99 98 { 100 99 __sysreg_save_common_state(ctxt); 101 100 } 101 + NOKPROBE_SYMBOL(sysreg_save_host_state_vhe); 102 102 103 103 void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt) 104 104 { 105 105 __sysreg_save_common_state(ctxt); 106 106 __sysreg_save_el2_return_state(ctxt); 107 107 } 108 + NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe); 108 109 109 110 static void __hyp_text __sysreg_restore_common_state(struct kvm_cpu_context *ctxt) 110 111 { ··· 191 188 { 192 189 __sysreg_restore_common_state(ctxt); 193 190 } 191 + NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe); 194 192 195 193 void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt) 196 194 { 197 195 __sysreg_restore_common_state(ctxt); 198 196 __sysreg_restore_el2_return_state(ctxt); 199 197 } 198 + NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe); 200 199 201 200 void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu) 202 201 {

+48 -2

arch/arm64/kvm/reset.c

··· 32 32 #include <asm/kvm_arm.h> 33 33 #include <asm/kvm_asm.h> 34 34 #include <asm/kvm_coproc.h> 35 + #include <asm/kvm_emulate.h> 35 36 #include <asm/kvm_mmu.h> 36 37 37 38 /* Maximum phys_shift supported for any VM on this host */ ··· 106 105 * This function finds the right table above and sets the registers on 107 106 * the virtual CPU struct to their architecturally defined reset 108 107 * values. 108 + * 109 + * Note: This function can be called from two paths: The KVM_ARM_VCPU_INIT 110 + * ioctl or as part of handling a request issued by another VCPU in the PSCI 111 + * handling code. In the first case, the VCPU will not be loaded, and in the 112 + * second case the VCPU will be loaded. Because this function operates purely 113 + * on the memory-backed valus of system registers, we want to do a full put if 114 + * we were loaded (handling a request) and load the values back at the end of 115 + * the function. Otherwise we leave the state alone. In both cases, we 116 + * disable preemption around the vcpu reset as we would otherwise race with 117 + * preempt notifiers which also call put/load. 109 118 */ 110 119 int kvm_reset_vcpu(struct kvm_vcpu *vcpu) 111 120 { 112 121 const struct kvm_regs *cpu_reset; 122 + int ret = -EINVAL; 123 + bool loaded; 124 + 125 + preempt_disable(); 126 + loaded = (vcpu->cpu != -1); 127 + if (loaded) 128 + kvm_arch_vcpu_put(vcpu); 113 129 114 130 switch (vcpu->arch.target) { 115 131 default: 116 132 if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features)) { 117 133 if (!cpu_has_32bit_el1()) 118 - return -EINVAL; 134 + goto out; 119 135 cpu_reset = &default_regs_reset32; 120 136 } else { 121 137 cpu_reset = &default_regs_reset; ··· 147 129 /* Reset system registers */ 148 130 kvm_reset_sys_regs(vcpu); 149 131 132 + /* 133 + * Additional reset state handling that PSCI may have imposed on us. 134 + * Must be done after all the sys_reg reset. 135 + */ 136 + if (vcpu->arch.reset_state.reset) { 137 + unsigned long target_pc = vcpu->arch.reset_state.pc; 138 + 139 + /* Gracefully handle Thumb2 entry point */ 140 + if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) { 141 + target_pc &= ~1UL; 142 + vcpu_set_thumb(vcpu); 143 + } 144 + 145 + /* Propagate caller endianness */ 146 + if (vcpu->arch.reset_state.be) 147 + kvm_vcpu_set_be(vcpu); 148 + 149 + *vcpu_pc(vcpu) = target_pc; 150 + vcpu_set_reg(vcpu, 0, vcpu->arch.reset_state.r0); 151 + 152 + vcpu->arch.reset_state.reset = false; 153 + } 154 + 150 155 /* Reset PMU */ 151 156 kvm_pmu_vcpu_reset(vcpu); 152 157 ··· 178 137 vcpu->arch.workaround_flags |= VCPU_WORKAROUND_2_FLAG; 179 138 180 139 /* Reset timer */ 181 - return kvm_timer_vcpu_reset(vcpu); 140 + ret = kvm_timer_vcpu_reset(vcpu); 141 + out: 142 + if (loaded) 143 + kvm_arch_vcpu_load(vcpu, smp_processor_id()); 144 + preempt_enable(); 145 + return ret; 182 146 } 183 147 184 148 void kvm_set_ipa_limit(void)

+32 -18

arch/arm64/kvm/sys_regs.c

··· 314 314 return read_zero(vcpu, p); 315 315 } 316 316 317 - static bool trap_undef(struct kvm_vcpu *vcpu, 318 - struct sys_reg_params *p, 319 - const struct sys_reg_desc *r) 317 + /* 318 + * ARMv8.1 mandates at least a trivial LORegion implementation, where all the 319 + * RW registers are RES0 (which we can implement as RAZ/WI). On an ARMv8.0 320 + * system, these registers should UNDEF. LORID_EL1 being a RO register, we 321 + * treat it separately. 322 + */ 323 + static bool trap_loregion(struct kvm_vcpu *vcpu, 324 + struct sys_reg_params *p, 325 + const struct sys_reg_desc *r) 320 326 { 321 - kvm_inject_undefined(vcpu); 322 - return false; 327 + u64 val = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); 328 + u32 sr = sys_reg((u32)r->Op0, (u32)r->Op1, 329 + (u32)r->CRn, (u32)r->CRm, (u32)r->Op2); 330 + 331 + if (!(val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))) { 332 + kvm_inject_undefined(vcpu); 333 + return false; 334 + } 335 + 336 + if (p->is_write && sr == SYS_LORID_EL1) 337 + return write_to_read_only(vcpu, p, r); 338 + 339 + return trap_raz_wi(vcpu, p, r); 323 340 } 324 341 325 342 static bool trap_oslsr_el1(struct kvm_vcpu *vcpu, ··· 1065 1048 if (val & ptrauth_mask) 1066 1049 kvm_debug("ptrauth unsupported for guests, suppressing\n"); 1067 1050 val &= ~ptrauth_mask; 1068 - } else if (id == SYS_ID_AA64MMFR1_EL1) { 1069 - if (val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT)) 1070 - kvm_debug("LORegions unsupported for guests, suppressing\n"); 1071 - 1072 - val &= ~(0xfUL << ID_AA64MMFR1_LOR_SHIFT); 1073 1051 } 1074 1052 1075 1053 return val; ··· 1350 1338 { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 }, 1351 1339 { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 }, 1352 1340 1353 - { SYS_DESC(SYS_LORSA_EL1), trap_undef }, 1354 - { SYS_DESC(SYS_LOREA_EL1), trap_undef }, 1355 - { SYS_DESC(SYS_LORN_EL1), trap_undef }, 1356 - { SYS_DESC(SYS_LORC_EL1), trap_undef }, 1357 - { SYS_DESC(SYS_LORID_EL1), trap_undef }, 1341 + { SYS_DESC(SYS_LORSA_EL1), trap_loregion }, 1342 + { SYS_DESC(SYS_LOREA_EL1), trap_loregion }, 1343 + { SYS_DESC(SYS_LORN_EL1), trap_loregion }, 1344 + { SYS_DESC(SYS_LORC_EL1), trap_loregion }, 1345 + { SYS_DESC(SYS_LORID_EL1), trap_loregion }, 1358 1346 1359 1347 { SYS_DESC(SYS_VBAR_EL1), NULL, reset_val, VBAR_EL1, 0 }, 1360 1348 { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 }, ··· 2608 2596 table = get_target_table(vcpu->arch.target, true, &num); 2609 2597 reset_sys_reg_descs(vcpu, table, num); 2610 2598 2611 - for (num = 1; num < NR_SYS_REGS; num++) 2612 - if (__vcpu_sys_reg(vcpu, num) == 0x4242424242424242) 2613 - panic("Didn't reset __vcpu_sys_reg(%zi)", num); 2599 + for (num = 1; num < NR_SYS_REGS; num++) { 2600 + if (WARN(__vcpu_sys_reg(vcpu, num) == 0x4242424242424242, 2601 + "Didn't reset __vcpu_sys_reg(%zi)\n", num)) 2602 + break; 2603 + } 2614 2604 }

+9 -3

arch/x86/kvm/vmx/nested.c

··· 2473 2473 (nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 2474 2474 return -EINVAL; 2475 2475 2476 + if (!nested_cpu_has_preemption_timer(vmcs12) && 2477 + nested_cpu_has_save_preemption_timer(vmcs12)) 2478 + return -EINVAL; 2479 + 2476 2480 if (nested_cpu_has_ept(vmcs12) && 2477 2481 !valid_ept_address(vcpu, vmcs12->ept_pointer)) 2478 2482 return -EINVAL; ··· 5561 5557 * secondary cpu-based controls. Do not include those that 5562 5558 * depend on CPUID bits, they are added later by vmx_cpuid_update. 5563 5559 */ 5564 - rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2, 5565 - msrs->secondary_ctls_low, 5566 - msrs->secondary_ctls_high); 5560 + if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) 5561 + rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2, 5562 + msrs->secondary_ctls_low, 5563 + msrs->secondary_ctls_high); 5564 + 5567 5565 msrs->secondary_ctls_low = 0; 5568 5566 msrs->secondary_ctls_high &= 5569 5567 SECONDARY_EXEC_DESC |

+13 -16

arch/x86/kvm/vmx/vmx.c

··· 863 863 if (!entry_only) 864 864 j = find_msr(&m->host, msr); 865 865 866 - if (i == NR_AUTOLOAD_MSRS || j == NR_AUTOLOAD_MSRS) { 866 + if ((i < 0 && m->guest.nr == NR_AUTOLOAD_MSRS) || 867 + (j < 0 && m->host.nr == NR_AUTOLOAD_MSRS)) { 867 868 printk_once(KERN_WARNING "Not enough msr switch entries. " 868 869 "Can't add msr %x\n", msr); 869 870 return; ··· 1194 1193 if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu) 1195 1194 return; 1196 1195 1197 - /* 1198 - * First handle the simple case where no cmpxchg is necessary; just 1199 - * allow posting non-urgent interrupts. 1200 - * 1201 - * If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change 1202 - * PI.NDST: pi_post_block will do it for us and the wakeup_handler 1203 - * expects the VCPU to be on the blocked_vcpu_list that matches 1204 - * PI.NDST. 1205 - */ 1206 - if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR || 1207 - vcpu->cpu == cpu) { 1208 - pi_clear_sn(pi_desc); 1209 - return; 1210 - } 1211 - 1212 1196 /* The full case. */ 1213 1197 do { 1214 1198 old.control = new.control = pi_desc->control; ··· 1208 1222 new.sn = 0; 1209 1223 } while (cmpxchg64(&pi_desc->control, old.control, 1210 1224 new.control) != old.control); 1225 + 1226 + /* 1227 + * Clear SN before reading the bitmap. The VT-d firmware 1228 + * writes the bitmap and reads SN atomically (5.2.3 in the 1229 + * spec), so it doesn't really have a memory barrier that 1230 + * pairs with this, but we cannot do that and we need one. 1231 + */ 1232 + smp_mb__after_atomic(); 1233 + 1234 + if (!bitmap_empty((unsigned long *)pi_desc->pir, NR_VECTORS)) 1235 + pi_set_on(pi_desc); 1211 1236 } 1212 1237 1213 1238 /*

+6 -6

arch/x86/kvm/vmx/vmx.h

··· 337 337 return test_and_set_bit(vector, (unsigned long *)pi_desc->pir); 338 338 } 339 339 340 - static inline void pi_clear_sn(struct pi_desc *pi_desc) 341 - { 342 - return clear_bit(POSTED_INTR_SN, 343 - (unsigned long *)&pi_desc->control); 344 - } 345 - 346 340 static inline void pi_set_sn(struct pi_desc *pi_desc) 347 341 { 348 342 return set_bit(POSTED_INTR_SN, 349 343 (unsigned long *)&pi_desc->control); 344 + } 345 + 346 + static inline void pi_set_on(struct pi_desc *pi_desc) 347 + { 348 + set_bit(POSTED_INTR_ON, 349 + (unsigned long *)&pi_desc->control); 350 350 } 351 351 352 352 static inline void pi_clear_on(struct pi_desc *pi_desc)

+1 -1

arch/x86/kvm/x86.c

··· 7801 7801 * 1) We should set ->mode before checking ->requests. Please see 7802 7802 * the comment in kvm_vcpu_exiting_guest_mode(). 7803 7803 * 7804 - * 2) For APICv, we should set ->mode before checking PIR.ON. This 7804 + * 2) For APICv, we should set ->mode before checking PID.ON. This 7805 7805 * pairs with the memory barrier implicit in pi_test_and_set_on 7806 7806 * (see vmx_deliver_posted_interrupt). 7807 7807 *

+3 -3

include/kvm/arm_vgic.h

··· 100 100 }; 101 101 102 102 struct vgic_irq { 103 - spinlock_t irq_lock; /* Protects the content of the struct */ 103 + raw_spinlock_t irq_lock; /* Protects the content of the struct */ 104 104 struct list_head lpi_list; /* Used to link all LPIs together */ 105 105 struct list_head ap_list; 106 106 ··· 256 256 u64 propbaser; 257 257 258 258 /* Protects the lpi_list and the count value below. */ 259 - spinlock_t lpi_list_lock; 259 + raw_spinlock_t lpi_list_lock; 260 260 struct list_head lpi_list_head; 261 261 int lpi_list_count; 262 262 ··· 307 307 unsigned int used_lrs; 308 308 struct vgic_irq private_irqs[VGIC_NR_PRIVATE_IRQS]; 309 309 310 - spinlock_t ap_list_lock; /* Protects the ap_list */ 310 + raw_spinlock_t ap_list_lock; /* Protects the ap_list */ 311 311 312 312 /* 313 313 * List of IRQs that this VCPU should consider because they are either

+10

virt/kvm/arm/arm.c

··· 626 626 /* Awaken to handle a signal, request we sleep again later. */ 627 627 kvm_make_request(KVM_REQ_SLEEP, vcpu); 628 628 } 629 + 630 + /* 631 + * Make sure we will observe a potential reset request if we've 632 + * observed a change to the power state. Pairs with the smp_wmb() in 633 + * kvm_psci_vcpu_on(). 634 + */ 635 + smp_rmb(); 629 636 } 630 637 631 638 static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu) ··· 645 638 if (kvm_request_pending(vcpu)) { 646 639 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) 647 640 vcpu_req_sleep(vcpu); 641 + 642 + if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) 643 + kvm_reset_vcpu(vcpu); 648 644 649 645 /* 650 646 * Clear IRQ_PENDING requests that were made to guarantee

+6 -3

virt/kvm/arm/mmu.c

··· 1695 1695 1696 1696 vma_pagesize = vma_kernel_pagesize(vma); 1697 1697 /* 1698 - * PUD level may not exist for a VM but PMD is guaranteed to 1699 - * exist. 1698 + * The stage2 has a minimum of 2 level table (For arm64 see 1699 + * kvm_arm_setup_stage2()). Hence, we are guaranteed that we can 1700 + * use PMD_SIZE huge mappings (even when the PMD is folded into PGD). 1701 + * As for PUD huge maps, we must make sure that we have at least 1702 + * 3 levels, i.e, PMD is not folded. 1700 1703 */ 1701 1704 if ((vma_pagesize == PMD_SIZE || 1702 - (vma_pagesize == PUD_SIZE && kvm_stage2_has_pud(kvm))) && 1705 + (vma_pagesize == PUD_SIZE && kvm_stage2_has_pmd(kvm))) && 1703 1706 !force_pte) { 1704 1707 gfn = (fault_ipa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT; 1705 1708 }

+16 -20

virt/kvm/arm/psci.c

··· 104 104 105 105 static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) 106 106 { 107 + struct vcpu_reset_state *reset_state; 107 108 struct kvm *kvm = source_vcpu->kvm; 108 109 struct kvm_vcpu *vcpu = NULL; 109 - struct swait_queue_head *wq; 110 110 unsigned long cpu_id; 111 - unsigned long context_id; 112 - phys_addr_t target_pc; 113 111 114 112 cpu_id = smccc_get_arg1(source_vcpu) & MPIDR_HWID_BITMASK; 115 113 if (vcpu_mode_is_32bit(source_vcpu)) ··· 128 130 return PSCI_RET_INVALID_PARAMS; 129 131 } 130 132 131 - target_pc = smccc_get_arg2(source_vcpu); 132 - context_id = smccc_get_arg3(source_vcpu); 133 + reset_state = &vcpu->arch.reset_state; 133 134 134 - kvm_reset_vcpu(vcpu); 135 - 136 - /* Gracefully handle Thumb2 entry point */ 137 - if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) { 138 - target_pc &= ~((phys_addr_t) 1); 139 - vcpu_set_thumb(vcpu); 140 - } 135 + reset_state->pc = smccc_get_arg2(source_vcpu); 141 136 142 137 /* Propagate caller endianness */ 143 - if (kvm_vcpu_is_be(source_vcpu)) 144 - kvm_vcpu_set_be(vcpu); 138 + reset_state->be = kvm_vcpu_is_be(source_vcpu); 145 139 146 - *vcpu_pc(vcpu) = target_pc; 147 140 /* 148 141 * NOTE: We always update r0 (or x0) because for PSCI v0.1 149 142 * the general puspose registers are undefined upon CPU_ON. 150 143 */ 151 - smccc_set_retval(vcpu, context_id, 0, 0, 0); 152 - vcpu->arch.power_off = false; 153 - smp_mb(); /* Make sure the above is visible */ 144 + reset_state->r0 = smccc_get_arg3(source_vcpu); 154 145 155 - wq = kvm_arch_vcpu_wq(vcpu); 156 - swake_up_one(wq); 146 + WRITE_ONCE(reset_state->reset, true); 147 + kvm_make_request(KVM_REQ_VCPU_RESET, vcpu); 148 + 149 + /* 150 + * Make sure the reset request is observed if the change to 151 + * power_state is observed. 152 + */ 153 + smp_wmb(); 154 + 155 + vcpu->arch.power_off = false; 156 + kvm_vcpu_wake_up(vcpu); 157 157 158 158 return PSCI_RET_SUCCESS; 159 159 }

+2 -2

virt/kvm/arm/vgic/vgic-debug.c

··· 251 251 return 0; 252 252 } 253 253 254 - spin_lock_irqsave(&irq->irq_lock, flags); 254 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 255 255 print_irq_state(s, irq, vcpu); 256 - spin_unlock_irqrestore(&irq->irq_lock, flags); 256 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 257 257 258 258 vgic_put_irq(kvm, irq); 259 259 return 0;

+18 -12

virt/kvm/arm/vgic/vgic-init.c

··· 64 64 struct vgic_dist *dist = &kvm->arch.vgic; 65 65 66 66 INIT_LIST_HEAD(&dist->lpi_list_head); 67 - spin_lock_init(&dist->lpi_list_lock); 67 + raw_spin_lock_init(&dist->lpi_list_lock); 68 68 } 69 69 70 70 /* CREATION */ ··· 171 171 172 172 irq->intid = i + VGIC_NR_PRIVATE_IRQS; 173 173 INIT_LIST_HEAD(&irq->ap_list); 174 - spin_lock_init(&irq->irq_lock); 174 + raw_spin_lock_init(&irq->irq_lock); 175 175 irq->vcpu = NULL; 176 176 irq->target_vcpu = vcpu0; 177 177 kref_init(&irq->refcount); ··· 206 206 vgic_cpu->sgi_iodev.base_addr = VGIC_ADDR_UNDEF; 207 207 208 208 INIT_LIST_HEAD(&vgic_cpu->ap_list_head); 209 - spin_lock_init(&vgic_cpu->ap_list_lock); 209 + raw_spin_lock_init(&vgic_cpu->ap_list_lock); 210 210 211 211 /* 212 212 * Enable and configure all SGIs to be edge-triggered and ··· 216 216 struct vgic_irq *irq = &vgic_cpu->private_irqs[i]; 217 217 218 218 INIT_LIST_HEAD(&irq->ap_list); 219 - spin_lock_init(&irq->irq_lock); 219 + raw_spin_lock_init(&irq->irq_lock); 220 220 irq->intid = i; 221 221 irq->vcpu = NULL; 222 222 irq->target_vcpu = vcpu; ··· 231 231 irq->config = VGIC_CONFIG_LEVEL; 232 232 } 233 233 234 - /* 235 - * GICv3 can only be created via the KVM_DEVICE_CREATE API and 236 - * so we always know the emulation type at this point as it's 237 - * either explicitly configured as GICv3, or explicitly 238 - * configured as GICv2, or not configured yet which also 239 - * implies GICv2. 240 - */ 241 234 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) 242 235 irq->group = 1; 243 236 else ··· 274 281 { 275 282 struct vgic_dist *dist = &kvm->arch.vgic; 276 283 struct kvm_vcpu *vcpu; 277 - int ret = 0, i; 284 + int ret = 0, i, idx; 278 285 279 286 if (vgic_initialized(kvm)) 280 287 return 0; ··· 290 297 ret = kvm_vgic_dist_init(kvm, dist->nr_spis); 291 298 if (ret) 292 299 goto out; 300 + 301 + /* Initialize groups on CPUs created before the VGIC type was known */ 302 + kvm_for_each_vcpu(idx, vcpu, kvm) { 303 + struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 304 + 305 + for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) { 306 + struct vgic_irq *irq = &vgic_cpu->private_irqs[i]; 307 + if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) 308 + irq->group = 1; 309 + else 310 + irq->group = 0; 311 + } 312 + } 293 313 294 314 if (vgic_has_its(kvm)) { 295 315 ret = vgic_v4_init(kvm);

+11 -11

virt/kvm/arm/vgic/vgic-its.c

··· 65 65 66 66 INIT_LIST_HEAD(&irq->lpi_list); 67 67 INIT_LIST_HEAD(&irq->ap_list); 68 - spin_lock_init(&irq->irq_lock); 68 + raw_spin_lock_init(&irq->irq_lock); 69 69 70 70 irq->config = VGIC_CONFIG_EDGE; 71 71 kref_init(&irq->refcount); ··· 73 73 irq->target_vcpu = vcpu; 74 74 irq->group = 1; 75 75 76 - spin_lock_irqsave(&dist->lpi_list_lock, flags); 76 + raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); 77 77 78 78 /* 79 79 * There could be a race with another vgic_add_lpi(), so we need to ··· 101 101 dist->lpi_list_count++; 102 102 103 103 out_unlock: 104 - spin_unlock_irqrestore(&dist->lpi_list_lock, flags); 104 + raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); 105 105 106 106 /* 107 107 * We "cache" the configuration table entries in our struct vgic_irq's. ··· 287 287 if (ret) 288 288 return ret; 289 289 290 - spin_lock_irqsave(&irq->irq_lock, flags); 290 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 291 291 292 292 if (!filter_vcpu || filter_vcpu == irq->target_vcpu) { 293 293 irq->priority = LPI_PROP_PRIORITY(prop); ··· 299 299 } 300 300 } 301 301 302 - spin_unlock_irqrestore(&irq->irq_lock, flags); 302 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 303 303 304 304 if (irq->hw) 305 305 return its_prop_update_vlpi(irq->host_irq, prop, needs_inv); ··· 332 332 if (!intids) 333 333 return -ENOMEM; 334 334 335 - spin_lock_irqsave(&dist->lpi_list_lock, flags); 335 + raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); 336 336 list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) { 337 337 if (i == irq_count) 338 338 break; ··· 341 341 continue; 342 342 intids[i++] = irq->intid; 343 343 } 344 - spin_unlock_irqrestore(&dist->lpi_list_lock, flags); 344 + raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); 345 345 346 346 *intid_ptr = intids; 347 347 return i; ··· 352 352 int ret = 0; 353 353 unsigned long flags; 354 354 355 - spin_lock_irqsave(&irq->irq_lock, flags); 355 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 356 356 irq->target_vcpu = vcpu; 357 - spin_unlock_irqrestore(&irq->irq_lock, flags); 357 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 358 358 359 359 if (irq->hw) { 360 360 struct its_vlpi_map map; ··· 455 455 } 456 456 457 457 irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]); 458 - spin_lock_irqsave(&irq->irq_lock, flags); 458 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 459 459 irq->pending_latch = pendmask & (1U << bit_nr); 460 460 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 461 461 vgic_put_irq(vcpu->kvm, irq); ··· 612 612 return irq_set_irqchip_state(irq->host_irq, 613 613 IRQCHIP_STATE_PENDING, true); 614 614 615 - spin_lock_irqsave(&irq->irq_lock, flags); 615 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 616 616 irq->pending_latch = true; 617 617 vgic_queue_irq_unlock(kvm, irq, flags); 618 618

+7 -7

virt/kvm/arm/vgic/vgic-mmio-v2.c

··· 147 147 148 148 irq = vgic_get_irq(source_vcpu->kvm, vcpu, intid); 149 149 150 - spin_lock_irqsave(&irq->irq_lock, flags); 150 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 151 151 irq->pending_latch = true; 152 152 irq->source |= 1U << source_vcpu->vcpu_id; 153 153 ··· 191 191 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid + i); 192 192 int target; 193 193 194 - spin_lock_irqsave(&irq->irq_lock, flags); 194 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 195 195 196 196 irq->targets = (val >> (i * 8)) & cpu_mask; 197 197 target = irq->targets ? __ffs(irq->targets) : 0; 198 198 irq->target_vcpu = kvm_get_vcpu(vcpu->kvm, target); 199 199 200 - spin_unlock_irqrestore(&irq->irq_lock, flags); 200 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 201 201 vgic_put_irq(vcpu->kvm, irq); 202 202 } 203 203 } ··· 230 230 for (i = 0; i < len; i++) { 231 231 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 232 232 233 - spin_lock_irqsave(&irq->irq_lock, flags); 233 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 234 234 235 235 irq->source &= ~((val >> (i * 8)) & 0xff); 236 236 if (!irq->source) 237 237 irq->pending_latch = false; 238 238 239 - spin_unlock_irqrestore(&irq->irq_lock, flags); 239 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 240 240 vgic_put_irq(vcpu->kvm, irq); 241 241 } 242 242 } ··· 252 252 for (i = 0; i < len; i++) { 253 253 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 254 254 255 - spin_lock_irqsave(&irq->irq_lock, flags); 255 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 256 256 257 257 irq->source |= (val >> (i * 8)) & 0xff; 258 258 ··· 260 260 irq->pending_latch = true; 261 261 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 262 262 } else { 263 - spin_unlock_irqrestore(&irq->irq_lock, flags); 263 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 264 264 } 265 265 vgic_put_irq(vcpu->kvm, irq); 266 266 }

+6 -6

virt/kvm/arm/vgic/vgic-mmio-v3.c

··· 169 169 if (!irq) 170 170 return; 171 171 172 - spin_lock_irqsave(&irq->irq_lock, flags); 172 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 173 173 174 174 /* We only care about and preserve Aff0, Aff1 and Aff2. */ 175 175 irq->mpidr = val & GENMASK(23, 0); 176 176 irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr); 177 177 178 - spin_unlock_irqrestore(&irq->irq_lock, flags); 178 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 179 179 vgic_put_irq(vcpu->kvm, irq); 180 180 } 181 181 ··· 281 281 for (i = 0; i < len * 8; i++) { 282 282 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 283 283 284 - spin_lock_irqsave(&irq->irq_lock, flags); 284 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 285 285 if (test_bit(i, &val)) { 286 286 /* 287 287 * pending_latch is set irrespective of irq type ··· 292 292 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 293 293 } else { 294 294 irq->pending_latch = false; 295 - spin_unlock_irqrestore(&irq->irq_lock, flags); 295 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 296 296 } 297 297 298 298 vgic_put_irq(vcpu->kvm, irq); ··· 957 957 958 958 irq = vgic_get_irq(vcpu->kvm, c_vcpu, sgi); 959 959 960 - spin_lock_irqsave(&irq->irq_lock, flags); 960 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 961 961 962 962 /* 963 963 * An access targetting Group0 SGIs can only generate ··· 968 968 irq->pending_latch = true; 969 969 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 970 970 } else { 971 - spin_unlock_irqrestore(&irq->irq_lock, flags); 971 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 972 972 } 973 973 974 974 vgic_put_irq(vcpu->kvm, irq);

+17 -17

virt/kvm/arm/vgic/vgic-mmio.c

··· 77 77 for (i = 0; i < len * 8; i++) { 78 78 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 79 79 80 - spin_lock_irqsave(&irq->irq_lock, flags); 80 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 81 81 irq->group = !!(val & BIT(i)); 82 82 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 83 83 ··· 120 120 for_each_set_bit(i, &val, len * 8) { 121 121 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 122 122 123 - spin_lock_irqsave(&irq->irq_lock, flags); 123 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 124 124 irq->enabled = true; 125 125 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 126 126 ··· 139 139 for_each_set_bit(i, &val, len * 8) { 140 140 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 141 141 142 - spin_lock_irqsave(&irq->irq_lock, flags); 142 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 143 143 144 144 irq->enabled = false; 145 145 146 - spin_unlock_irqrestore(&irq->irq_lock, flags); 146 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 147 147 vgic_put_irq(vcpu->kvm, irq); 148 148 } 149 149 } ··· 160 160 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 161 161 unsigned long flags; 162 162 163 - spin_lock_irqsave(&irq->irq_lock, flags); 163 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 164 164 if (irq_is_pending(irq)) 165 165 value |= (1U << i); 166 - spin_unlock_irqrestore(&irq->irq_lock, flags); 166 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 167 167 168 168 vgic_put_irq(vcpu->kvm, irq); 169 169 } ··· 215 215 for_each_set_bit(i, &val, len * 8) { 216 216 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 217 217 218 - spin_lock_irqsave(&irq->irq_lock, flags); 218 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 219 219 if (irq->hw) 220 220 vgic_hw_irq_spending(vcpu, irq, is_uaccess); 221 221 else ··· 262 262 for_each_set_bit(i, &val, len * 8) { 263 263 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 264 264 265 - spin_lock_irqsave(&irq->irq_lock, flags); 265 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 266 266 267 267 if (irq->hw) 268 268 vgic_hw_irq_cpending(vcpu, irq, is_uaccess); 269 269 else 270 270 irq->pending_latch = false; 271 271 272 - spin_unlock_irqrestore(&irq->irq_lock, flags); 272 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 273 273 vgic_put_irq(vcpu->kvm, irq); 274 274 } 275 275 } ··· 311 311 unsigned long flags; 312 312 struct kvm_vcpu *requester_vcpu = vgic_get_mmio_requester_vcpu(); 313 313 314 - spin_lock_irqsave(&irq->irq_lock, flags); 314 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 315 315 316 316 if (irq->hw) { 317 317 vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu); ··· 342 342 if (irq->active) 343 343 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 344 344 else 345 - spin_unlock_irqrestore(&irq->irq_lock, flags); 345 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 346 346 } 347 347 348 348 /* ··· 485 485 for (i = 0; i < len; i++) { 486 486 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 487 487 488 - spin_lock_irqsave(&irq->irq_lock, flags); 488 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 489 489 /* Narrow the priority range to what we actually support */ 490 490 irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS); 491 - spin_unlock_irqrestore(&irq->irq_lock, flags); 491 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 492 492 493 493 vgic_put_irq(vcpu->kvm, irq); 494 494 } ··· 534 534 continue; 535 535 536 536 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 537 - spin_lock_irqsave(&irq->irq_lock, flags); 537 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 538 538 539 539 if (test_bit(i * 2 + 1, &val)) 540 540 irq->config = VGIC_CONFIG_EDGE; 541 541 else 542 542 irq->config = VGIC_CONFIG_LEVEL; 543 543 544 - spin_unlock_irqrestore(&irq->irq_lock, flags); 544 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 545 545 vgic_put_irq(vcpu->kvm, irq); 546 546 } 547 547 } ··· 590 590 * restore irq config before line level. 591 591 */ 592 592 new_level = !!(val & (1U << i)); 593 - spin_lock_irqsave(&irq->irq_lock, flags); 593 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 594 594 irq->line_level = new_level; 595 595 if (new_level) 596 596 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 597 597 else 598 - spin_unlock_irqrestore(&irq->irq_lock, flags); 598 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 599 599 600 600 vgic_put_irq(vcpu->kvm, irq); 601 601 }

+2 -2

virt/kvm/arm/vgic/vgic-v2.c

··· 84 84 85 85 irq = vgic_get_irq(vcpu->kvm, vcpu, intid); 86 86 87 - spin_lock(&irq->irq_lock); 87 + raw_spin_lock(&irq->irq_lock); 88 88 89 89 /* Always preserve the active bit */ 90 90 irq->active = !!(val & GICH_LR_ACTIVE_BIT); ··· 127 127 vgic_irq_set_phys_active(irq, false); 128 128 } 129 129 130 - spin_unlock(&irq->irq_lock); 130 + raw_spin_unlock(&irq->irq_lock); 131 131 vgic_put_irq(vcpu->kvm, irq); 132 132 } 133 133

+4 -4

virt/kvm/arm/vgic/vgic-v3.c

··· 76 76 if (!irq) /* An LPI could have been unmapped. */ 77 77 continue; 78 78 79 - spin_lock(&irq->irq_lock); 79 + raw_spin_lock(&irq->irq_lock); 80 80 81 81 /* Always preserve the active bit */ 82 82 irq->active = !!(val & ICH_LR_ACTIVE_BIT); ··· 119 119 vgic_irq_set_phys_active(irq, false); 120 120 } 121 121 122 - spin_unlock(&irq->irq_lock); 122 + raw_spin_unlock(&irq->irq_lock); 123 123 vgic_put_irq(vcpu->kvm, irq); 124 124 } 125 125 ··· 347 347 348 348 status = val & (1 << bit_nr); 349 349 350 - spin_lock_irqsave(&irq->irq_lock, flags); 350 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 351 351 if (irq->target_vcpu != vcpu) { 352 - spin_unlock_irqrestore(&irq->irq_lock, flags); 352 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 353 353 goto retry; 354 354 } 355 355 irq->pending_latch = status;

+59 -59

virt/kvm/arm/vgic/vgic.c

··· 54 54 * When taking more than one ap_list_lock at the same time, always take the 55 55 * lowest numbered VCPU's ap_list_lock first, so: 56 56 * vcpuX->vcpu_id < vcpuY->vcpu_id: 57 - * spin_lock(vcpuX->arch.vgic_cpu.ap_list_lock); 58 - * spin_lock(vcpuY->arch.vgic_cpu.ap_list_lock); 57 + * raw_spin_lock(vcpuX->arch.vgic_cpu.ap_list_lock); 58 + * raw_spin_lock(vcpuY->arch.vgic_cpu.ap_list_lock); 59 59 * 60 60 * Since the VGIC must support injecting virtual interrupts from ISRs, we have 61 - * to use the spin_lock_irqsave/spin_unlock_irqrestore versions of outer 61 + * to use the raw_spin_lock_irqsave/raw_spin_unlock_irqrestore versions of outer 62 62 * spinlocks for any lock that may be taken while injecting an interrupt. 63 63 */ 64 64 ··· 72 72 struct vgic_irq *irq = NULL; 73 73 unsigned long flags; 74 74 75 - spin_lock_irqsave(&dist->lpi_list_lock, flags); 75 + raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); 76 76 77 77 list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) { 78 78 if (irq->intid != intid) ··· 88 88 irq = NULL; 89 89 90 90 out_unlock: 91 - spin_unlock_irqrestore(&dist->lpi_list_lock, flags); 91 + raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); 92 92 93 93 return irq; 94 94 } ··· 138 138 if (irq->intid < VGIC_MIN_LPI) 139 139 return; 140 140 141 - spin_lock_irqsave(&dist->lpi_list_lock, flags); 141 + raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); 142 142 if (!kref_put(&irq->refcount, vgic_irq_release)) { 143 - spin_unlock_irqrestore(&dist->lpi_list_lock, flags); 143 + raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); 144 144 return; 145 145 }; 146 146 147 147 list_del(&irq->lpi_list); 148 148 dist->lpi_list_count--; 149 - spin_unlock_irqrestore(&dist->lpi_list_lock, flags); 149 + raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); 150 150 151 151 kfree(irq); 152 152 } ··· 244 244 bool penda, pendb; 245 245 int ret; 246 246 247 - spin_lock(&irqa->irq_lock); 248 - spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING); 247 + raw_spin_lock(&irqa->irq_lock); 248 + raw_spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING); 249 249 250 250 if (irqa->active || irqb->active) { 251 251 ret = (int)irqb->active - (int)irqa->active; ··· 263 263 /* Both pending and enabled, sort by priority */ 264 264 ret = irqa->priority - irqb->priority; 265 265 out: 266 - spin_unlock(&irqb->irq_lock); 267 - spin_unlock(&irqa->irq_lock); 266 + raw_spin_unlock(&irqb->irq_lock); 267 + raw_spin_unlock(&irqa->irq_lock); 268 268 return ret; 269 269 } 270 270 ··· 325 325 * not need to be inserted into an ap_list and there is also 326 326 * no more work for us to do. 327 327 */ 328 - spin_unlock_irqrestore(&irq->irq_lock, flags); 328 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 329 329 330 330 /* 331 331 * We have to kick the VCPU here, because we could be ··· 347 347 * We must unlock the irq lock to take the ap_list_lock where 348 348 * we are going to insert this new pending interrupt. 349 349 */ 350 - spin_unlock_irqrestore(&irq->irq_lock, flags); 350 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 351 351 352 352 /* someone can do stuff here, which we re-check below */ 353 353 354 - spin_lock_irqsave(&vcpu->arch.vgic_cpu.ap_list_lock, flags); 355 - spin_lock(&irq->irq_lock); 354 + raw_spin_lock_irqsave(&vcpu->arch.vgic_cpu.ap_list_lock, flags); 355 + raw_spin_lock(&irq->irq_lock); 356 356 357 357 /* 358 358 * Did something change behind our backs? ··· 367 367 */ 368 368 369 369 if (unlikely(irq->vcpu || vcpu != vgic_target_oracle(irq))) { 370 - spin_unlock(&irq->irq_lock); 371 - spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, flags); 370 + raw_spin_unlock(&irq->irq_lock); 371 + raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, 372 + flags); 372 373 373 - spin_lock_irqsave(&irq->irq_lock, flags); 374 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 374 375 goto retry; 375 376 } 376 377 ··· 383 382 list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head); 384 383 irq->vcpu = vcpu; 385 384 386 - spin_unlock(&irq->irq_lock); 387 - spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, flags); 385 + raw_spin_unlock(&irq->irq_lock); 386 + raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, flags); 388 387 389 388 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); 390 389 kvm_vcpu_kick(vcpu); ··· 431 430 if (!irq) 432 431 return -EINVAL; 433 432 434 - spin_lock_irqsave(&irq->irq_lock, flags); 433 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 435 434 436 435 if (!vgic_validate_injection(irq, level, owner)) { 437 436 /* Nothing to see here, move along... */ 438 - spin_unlock_irqrestore(&irq->irq_lock, flags); 437 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 439 438 vgic_put_irq(kvm, irq); 440 439 return 0; 441 440 } ··· 495 494 496 495 BUG_ON(!irq); 497 496 498 - spin_lock_irqsave(&irq->irq_lock, flags); 497 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 499 498 ret = kvm_vgic_map_irq(vcpu, irq, host_irq, get_input_level); 500 - spin_unlock_irqrestore(&irq->irq_lock, flags); 499 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 501 500 vgic_put_irq(vcpu->kvm, irq); 502 501 503 502 return ret; ··· 520 519 if (!irq->hw) 521 520 goto out; 522 521 523 - spin_lock_irqsave(&irq->irq_lock, flags); 522 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 524 523 irq->active = false; 525 524 irq->pending_latch = false; 526 525 irq->line_level = false; 527 - spin_unlock_irqrestore(&irq->irq_lock, flags); 526 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 528 527 out: 529 528 vgic_put_irq(vcpu->kvm, irq); 530 529 } ··· 540 539 irq = vgic_get_irq(vcpu->kvm, vcpu, vintid); 541 540 BUG_ON(!irq); 542 541 543 - spin_lock_irqsave(&irq->irq_lock, flags); 542 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 544 543 kvm_vgic_unmap_irq(irq); 545 - spin_unlock_irqrestore(&irq->irq_lock, flags); 544 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 546 545 vgic_put_irq(vcpu->kvm, irq); 547 546 548 547 return 0; ··· 572 571 return -EINVAL; 573 572 574 573 irq = vgic_get_irq(vcpu->kvm, vcpu, intid); 575 - spin_lock_irqsave(&irq->irq_lock, flags); 574 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 576 575 if (irq->owner && irq->owner != owner) 577 576 ret = -EEXIST; 578 577 else 579 578 irq->owner = owner; 580 - spin_unlock_irqrestore(&irq->irq_lock, flags); 579 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 581 580 582 581 return ret; 583 582 } ··· 598 597 DEBUG_SPINLOCK_BUG_ON(!irqs_disabled()); 599 598 600 599 retry: 601 - spin_lock(&vgic_cpu->ap_list_lock); 600 + raw_spin_lock(&vgic_cpu->ap_list_lock); 602 601 603 602 list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) { 604 603 struct kvm_vcpu *target_vcpu, *vcpuA, *vcpuB; 605 604 bool target_vcpu_needs_kick = false; 606 605 607 - spin_lock(&irq->irq_lock); 606 + raw_spin_lock(&irq->irq_lock); 608 607 609 608 BUG_ON(vcpu != irq->vcpu); 610 609 ··· 617 616 */ 618 617 list_del(&irq->ap_list); 619 618 irq->vcpu = NULL; 620 - spin_unlock(&irq->irq_lock); 619 + raw_spin_unlock(&irq->irq_lock); 621 620 622 621 /* 623 622 * This vgic_put_irq call matches the ··· 632 631 633 632 if (target_vcpu == vcpu) { 634 633 /* We're on the right CPU */ 635 - spin_unlock(&irq->irq_lock); 634 + raw_spin_unlock(&irq->irq_lock); 636 635 continue; 637 636 } 638 637 639 638 /* This interrupt looks like it has to be migrated. */ 640 639 641 - spin_unlock(&irq->irq_lock); 642 - spin_unlock(&vgic_cpu->ap_list_lock); 640 + raw_spin_unlock(&irq->irq_lock); 641 + raw_spin_unlock(&vgic_cpu->ap_list_lock); 643 642 644 643 /* 645 644 * Ensure locking order by always locking the smallest ··· 653 652 vcpuB = vcpu; 654 653 } 655 654 656 - spin_lock(&vcpuA->arch.vgic_cpu.ap_list_lock); 657 - spin_lock_nested(&vcpuB->arch.vgic_cpu.ap_list_lock, 658 - SINGLE_DEPTH_NESTING); 659 - spin_lock(&irq->irq_lock); 655 + raw_spin_lock(&vcpuA->arch.vgic_cpu.ap_list_lock); 656 + raw_spin_lock_nested(&vcpuB->arch.vgic_cpu.ap_list_lock, 657 + SINGLE_DEPTH_NESTING); 658 + raw_spin_lock(&irq->irq_lock); 660 659 661 660 /* 662 661 * If the affinity has been preserved, move the ··· 676 675 target_vcpu_needs_kick = true; 677 676 } 678 677 679 - spin_unlock(&irq->irq_lock); 680 - spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock); 681 - spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock); 678 + raw_spin_unlock(&irq->irq_lock); 679 + raw_spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock); 680 + raw_spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock); 682 681 683 682 if (target_vcpu_needs_kick) { 684 683 kvm_make_request(KVM_REQ_IRQ_PENDING, target_vcpu); ··· 688 687 goto retry; 689 688 } 690 689 691 - spin_unlock(&vgic_cpu->ap_list_lock); 690 + raw_spin_unlock(&vgic_cpu->ap_list_lock); 692 691 } 693 692 694 693 static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu) ··· 742 741 list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { 743 742 int w; 744 743 745 - spin_lock(&irq->irq_lock); 744 + raw_spin_lock(&irq->irq_lock); 746 745 /* GICv2 SGIs can count for more than one... */ 747 746 w = vgic_irq_get_lr_count(irq); 748 - spin_unlock(&irq->irq_lock); 747 + raw_spin_unlock(&irq->irq_lock); 749 748 750 749 count += w; 751 750 *multi_sgi |= (w > 1); ··· 771 770 count = 0; 772 771 773 772 list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { 774 - spin_lock(&irq->irq_lock); 773 + raw_spin_lock(&irq->irq_lock); 775 774 776 775 /* 777 776 * If we have multi-SGIs in the pipeline, we need to ··· 781 780 * the AP list has been sorted already. 782 781 */ 783 782 if (multi_sgi && irq->priority > prio) { 784 - spin_unlock(&irq->irq_lock); 783 + _raw_spin_unlock(&irq->irq_lock); 785 784 break; 786 785 } 787 786 ··· 792 791 prio = irq->priority; 793 792 } 794 793 795 - spin_unlock(&irq->irq_lock); 794 + raw_spin_unlock(&irq->irq_lock); 796 795 797 796 if (count == kvm_vgic_global_state.nr_lr) { 798 797 if (!list_is_last(&irq->ap_list, ··· 873 872 874 873 DEBUG_SPINLOCK_BUG_ON(!irqs_disabled()); 875 874 876 - spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock); 875 + raw_spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock); 877 876 vgic_flush_lr_state(vcpu); 878 - spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock); 877 + raw_spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock); 879 878 880 879 if (can_access_vgic_from_kernel()) 881 880 vgic_restore_state(vcpu); ··· 919 918 920 919 vgic_get_vmcr(vcpu, &vmcr); 921 920 922 - spin_lock_irqsave(&vgic_cpu->ap_list_lock, flags); 921 + raw_spin_lock_irqsave(&vgic_cpu->ap_list_lock, flags); 923 922 924 923 list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { 925 - spin_lock(&irq->irq_lock); 924 + raw_spin_lock(&irq->irq_lock); 926 925 pending = irq_is_pending(irq) && irq->enabled && 927 926 !irq->active && 928 927 irq->priority < vmcr.pmr; 929 - spin_unlock(&irq->irq_lock); 928 + raw_spin_unlock(&irq->irq_lock); 930 929 931 930 if (pending) 932 931 break; 933 932 } 934 933 935 - spin_unlock_irqrestore(&vgic_cpu->ap_list_lock, flags); 934 + raw_spin_unlock_irqrestore(&vgic_cpu->ap_list_lock, flags); 936 935 937 936 return pending; 938 937 } ··· 964 963 return false; 965 964 966 965 irq = vgic_get_irq(vcpu->kvm, vcpu, vintid); 967 - spin_lock_irqsave(&irq->irq_lock, flags); 966 + raw_spin_lock_irqsave(&irq->irq_lock, flags); 968 967 map_is_active = irq->hw && irq->active; 969 - spin_unlock_irqrestore(&irq->irq_lock, flags); 968 + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 970 969 vgic_put_irq(vcpu->kvm, irq); 971 970 972 971 return map_is_active; 973 972 } 974 -

Configure Feed

Configure Feed