Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
KVM: arm64: gic-v5: Implement GICv5 load/put and save/restore
This change introduces GICv5 load/put. Additionally, it plumbs in
save/restore for:
* PPIs (ICH_PPI_x_EL2 regs)
* ICH_VMCR_EL2
* ICH_APR_EL2
* ICC_ICSR_EL1
A GICv5-specific enable bit is added to struct vgic_vmcr as this
differs from previous GICs. On GICv5-native systems, the VMCR only
contains the enable bit (driven by the guest via ICC_CR0_EL1.EN) and
the priority mask (PCR).
A struct gicv5_vpe is also introduced. This currently only contains a
single field - bool resident - which is used to track if a VPE is
currently running or not, and is used to avoid a case of double load
or double put on the WFI path for a vCPU. This struct will be extended
as additional GICv5 support is merged, specifically for VPE doorbells.
Co-authored-by: Timothy Hayes <timothy.hayes@arm.com>
Signed-off-by: Timothy Hayes <timothy.hayes@arm.com>
Signed-off-by: Sascha Bischoff <sascha.bischoff@arm.com>
Reviewed-by: Jonathan Cameron <jonathan.cameron@huawei.com>
Link: https://patch.msgid.link/20260319154937.3619520-18-sascha.bischoff@arm.com
Signed-off-by: Marc Zyngier <maz@kernel.org>
authored by
Sascha Bischoff
Timothy Hayes
and committed by
Marc Zyngier
9b8e3d4c
af325e87
+193
-21
+12
arch/arm64/kvm/hyp/nvhe/switch.c
+12
arch/arm64/kvm/hyp/nvhe/switch.c
···
113
113
/* Save VGICv3 state on non-VHE systems */
114
114
static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu)
115
115
{
116
+
if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) {
117
+
__vgic_v5_save_state(&vcpu->arch.vgic_cpu.vgic_v5);
118
+
__vgic_v5_save_ppi_state(&vcpu->arch.vgic_cpu.vgic_v5);
119
+
return;
120
+
}
121
+
116
122
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
117
123
__vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3);
118
124
__vgic_v3_deactivate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
···
128
122
/* Restore VGICv3 state on non-VHE systems */
129
123
static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu)
130
124
{
125
+
if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) {
126
+
__vgic_v5_restore_state(&vcpu->arch.vgic_cpu.vgic_v5);
127
+
__vgic_v5_restore_ppi_state(&vcpu->arch.vgic_cpu.vgic_v5);
128
+
return;
129
+
}
130
+
131
131
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
132
132
__vgic_v3_activate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
133
133
__vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3);
+34
-6
arch/arm64/kvm/vgic/vgic-mmio.c
+34
-6
arch/arm64/kvm/vgic/vgic-mmio.c
···
842
842
843
843
void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
844
844
{
845
-
if (kvm_vgic_global_state.type == VGIC_V2)
846
-
vgic_v2_set_vmcr(vcpu, vmcr);
847
-
else
845
+
const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
846
+
847
+
switch (dist->vgic_model) {
848
+
case KVM_DEV_TYPE_ARM_VGIC_V5:
849
+
vgic_v5_set_vmcr(vcpu, vmcr);
850
+
break;
851
+
case KVM_DEV_TYPE_ARM_VGIC_V3:
848
852
vgic_v3_set_vmcr(vcpu, vmcr);
853
+
break;
854
+
case KVM_DEV_TYPE_ARM_VGIC_V2:
855
+
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
856
+
vgic_v3_set_vmcr(vcpu, vmcr);
857
+
else
858
+
vgic_v2_set_vmcr(vcpu, vmcr);
859
+
break;
860
+
default:
861
+
BUG();
862
+
}
849
863
}
850
864
851
865
void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
852
866
{
853
-
if (kvm_vgic_global_state.type == VGIC_V2)
854
-
vgic_v2_get_vmcr(vcpu, vmcr);
855
-
else
867
+
const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
868
+
869
+
switch (dist->vgic_model) {
870
+
case KVM_DEV_TYPE_ARM_VGIC_V5:
871
+
vgic_v5_get_vmcr(vcpu, vmcr);
872
+
break;
873
+
case KVM_DEV_TYPE_ARM_VGIC_V3:
856
874
vgic_v3_get_vmcr(vcpu, vmcr);
875
+
break;
876
+
case KVM_DEV_TYPE_ARM_VGIC_V2:
877
+
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
878
+
vgic_v3_get_vmcr(vcpu, vmcr);
879
+
else
880
+
vgic_v2_get_vmcr(vcpu, vmcr);
881
+
break;
882
+
default:
883
+
BUG();
884
+
}
857
885
}
858
886
859
887
/*
+74
arch/arm64/kvm/vgic/vgic-v5.c
+74
arch/arm64/kvm/vgic/vgic-v5.c
···
86
86
87
87
return 0;
88
88
}
89
+
90
+
void vgic_v5_load(struct kvm_vcpu *vcpu)
91
+
{
92
+
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
93
+
94
+
/*
95
+
* On the WFI path, vgic_load is called a second time. The first is when
96
+
* scheduling in the vcpu thread again, and the second is when leaving
97
+
* WFI. Skip the second instance as it serves no purpose and just
98
+
* restores the same state again.
99
+
*/
100
+
if (cpu_if->gicv5_vpe.resident)
101
+
return;
102
+
103
+
kvm_call_hyp(__vgic_v5_restore_vmcr_apr, cpu_if);
104
+
105
+
cpu_if->gicv5_vpe.resident = true;
106
+
}
107
+
108
+
void vgic_v5_put(struct kvm_vcpu *vcpu)
109
+
{
110
+
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
111
+
112
+
/*
113
+
* Do nothing if we're not resident. This can happen in the WFI path
114
+
* where we do a vgic_put in the WFI path and again later when
115
+
* descheduling the thread. We risk losing VMCR state if we sync it
116
+
* twice, so instead return early in this case.
117
+
*/
118
+
if (!cpu_if->gicv5_vpe.resident)
119
+
return;
120
+
121
+
kvm_call_hyp(__vgic_v5_save_apr, cpu_if);
122
+
123
+
cpu_if->gicv5_vpe.resident = false;
124
+
}
125
+
126
+
void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
127
+
{
128
+
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
129
+
u64 vmcr = cpu_if->vgic_vmcr;
130
+
131
+
vmcrp->en = FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_EN, vmcr);
132
+
vmcrp->pmr = FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_VPMR, vmcr);
133
+
}
134
+
135
+
void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
136
+
{
137
+
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
138
+
u64 vmcr;
139
+
140
+
vmcr = FIELD_PREP(FEAT_GCIE_ICH_VMCR_EL2_VPMR, vmcrp->pmr) |
141
+
FIELD_PREP(FEAT_GCIE_ICH_VMCR_EL2_EN, vmcrp->en);
142
+
143
+
cpu_if->vgic_vmcr = vmcr;
144
+
}
145
+
146
+
void vgic_v5_restore_state(struct kvm_vcpu *vcpu)
147
+
{
148
+
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
149
+
150
+
__vgic_v5_restore_state(cpu_if);
151
+
__vgic_v5_restore_ppi_state(cpu_if);
152
+
dsb(sy);
153
+
}
154
+
155
+
void vgic_v5_save_state(struct kvm_vcpu *vcpu)
156
+
{
157
+
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
158
+
159
+
__vgic_v5_save_state(cpu_if);
160
+
__vgic_v5_save_ppi_state(cpu_if);
161
+
dsb(sy);
162
+
}
+59
-15
arch/arm64/kvm/vgic/vgic.c
+59
-15
arch/arm64/kvm/vgic/vgic.c
···
1017
1017
1018
1018
static inline void vgic_save_state(struct kvm_vcpu *vcpu)
1019
1019
{
1020
-
if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
1020
+
/* No switch statement here. See comment in vgic_restore_state() */
1021
+
if (vgic_is_v5(vcpu->kvm))
1022
+
vgic_v5_save_state(vcpu);
1023
+
else if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
1021
1024
vgic_v2_save_state(vcpu);
1022
1025
else
1023
1026
__vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3);
···
1029
1026
/* Sync back the hardware VGIC state into our emulation after a guest's run. */
1030
1027
void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1031
1028
{
1032
-
/* If nesting, emulate the HW effect from L0 to L1 */
1033
-
if (vgic_state_is_nested(vcpu)) {
1034
-
vgic_v3_sync_nested(vcpu);
1035
-
return;
1036
-
}
1029
+
if (vgic_is_v3(vcpu->kvm)) {
1030
+
/* If nesting, emulate the HW effect from L0 to L1 */
1031
+
if (vgic_state_is_nested(vcpu)) {
1032
+
vgic_v3_sync_nested(vcpu);
1033
+
return;
1034
+
}
1037
1035
1038
-
if (vcpu_has_nv(vcpu))
1039
-
vgic_v3_nested_update_mi(vcpu);
1036
+
if (vcpu_has_nv(vcpu))
1037
+
vgic_v3_nested_update_mi(vcpu);
1038
+
}
1040
1039
1041
1040
if (can_access_vgic_from_kernel())
1042
1041
vgic_save_state(vcpu);
···
1060
1055
1061
1056
static inline void vgic_restore_state(struct kvm_vcpu *vcpu)
1062
1057
{
1063
-
if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
1058
+
/*
1059
+
* As nice as it would be to restructure this code into a switch
1060
+
* statement as can be found elsewhere, the logic quickly gets ugly.
1061
+
*
1062
+
* __vgic_v3_restore_state() is doing a lot of heavy lifting here. It is
1063
+
* required for GICv3-on-GICv3, GICv2-on-GICv3, GICv3-on-GICv5, and the
1064
+
* no-in-kernel-irqchip case on GICv3 hardware. Hence, adding a switch
1065
+
* here results in much more complex code.
1066
+
*/
1067
+
if (vgic_is_v5(vcpu->kvm))
1068
+
vgic_v5_restore_state(vcpu);
1069
+
else if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
1064
1070
vgic_v2_restore_state(vcpu);
1065
1071
else
1066
1072
__vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3);
···
1125
1109
1126
1110
void kvm_vgic_load(struct kvm_vcpu *vcpu)
1127
1111
{
1112
+
const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1113
+
1128
1114
if (unlikely(!irqchip_in_kernel(vcpu->kvm) || !vgic_initialized(vcpu->kvm))) {
1129
1115
if (has_vhe() && static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
1130
1116
__vgic_v3_activate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
1131
1117
return;
1132
1118
}
1133
1119
1134
-
if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
1135
-
vgic_v2_load(vcpu);
1136
-
else
1120
+
switch (dist->vgic_model) {
1121
+
case KVM_DEV_TYPE_ARM_VGIC_V5:
1122
+
vgic_v5_load(vcpu);
1123
+
break;
1124
+
case KVM_DEV_TYPE_ARM_VGIC_V3:
1137
1125
vgic_v3_load(vcpu);
1126
+
break;
1127
+
case KVM_DEV_TYPE_ARM_VGIC_V2:
1128
+
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
1129
+
vgic_v3_load(vcpu);
1130
+
else
1131
+
vgic_v2_load(vcpu);
1132
+
break;
1133
+
default:
1134
+
BUG();
1135
+
}
1138
1136
}
1139
1137
1140
1138
void kvm_vgic_put(struct kvm_vcpu *vcpu)
1141
1139
{
1140
+
const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1141
+
1142
1142
if (unlikely(!irqchip_in_kernel(vcpu->kvm) || !vgic_initialized(vcpu->kvm))) {
1143
1143
if (has_vhe() && static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
1144
1144
__vgic_v3_deactivate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
1145
1145
return;
1146
1146
}
1147
1147
1148
-
if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
1149
-
vgic_v2_put(vcpu);
1150
-
else
1148
+
switch (dist->vgic_model) {
1149
+
case KVM_DEV_TYPE_ARM_VGIC_V5:
1150
+
vgic_v5_put(vcpu);
1151
+
break;
1152
+
case KVM_DEV_TYPE_ARM_VGIC_V3:
1151
1153
vgic_v3_put(vcpu);
1154
+
break;
1155
+
case KVM_DEV_TYPE_ARM_VGIC_V2:
1156
+
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
1157
+
vgic_v3_put(vcpu);
1158
+
else
1159
+
vgic_v2_put(vcpu);
1160
+
break;
1161
+
default:
1162
+
BUG();
1163
+
}
1152
1164
}
1153
1165
1154
1166
int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
+7
arch/arm64/kvm/vgic/vgic.h
+7
arch/arm64/kvm/vgic/vgic.h
···
187
187
* registers regardless of the hardware backed GIC used.
188
188
*/
189
189
struct vgic_vmcr {
190
+
u32 en; /* GICv5-specific */
190
191
u32 grpen0;
191
192
u32 grpen1;
192
193
···
364
363
void vgic_debug_destroy(struct kvm *kvm);
365
364
366
365
int vgic_v5_probe(const struct gic_kvm_info *info);
366
+
void vgic_v5_load(struct kvm_vcpu *vcpu);
367
+
void vgic_v5_put(struct kvm_vcpu *vcpu);
368
+
void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
369
+
void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
370
+
void vgic_v5_restore_state(struct kvm_vcpu *vcpu);
371
+
void vgic_v5_save_state(struct kvm_vcpu *vcpu);
367
372
368
373
static inline int vgic_v3_max_apr_idx(struct kvm_vcpu *vcpu)
369
374
{
+2
include/kvm/arm_vgic.h
+2
include/kvm/arm_vgic.h