Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch kvm-arm64/vtcr into kvmarm-master/next
* kvm-arm64/vtcr:
: .
: VTCR_EL2 conversion to the configuration-driven RESx framework,
: fixing a couple of UXN/PXN/XN bugs in the process.
: .
KVM: arm64: nv: Return correct RES0 bits for FGT registers
KVM: arm64: Always populate FGT masks at boot time
KVM: arm64: Honor UX/PX attributes for EL2 S1 mappings
KVM: arm64: Convert VTCR_EL2 to config-driven sanitisation
KVM: arm64: Account for RES1 bits in DECLARE_FEAT_MAP() and co
arm64: Convert VTCR_EL2 to sysreg infratructure
arm64: Convert ID_AA64MMFR0_EL1.TGRAN{4,16,64}_2 to UnsignedEnum
KVM: arm64: Invert KVM_PGTABLE_WALK_HANDLE_FAULT to fix pKVM walkers
KVM: arm64: Don't blindly set set PSTATE.PAN on guest exit
KVM: arm64: nv: Respect stage-2 write permssion when setting stage-1 AF
KVM: arm64: Remove unused vcpu_{clear,set}_wfx_traps()
KVM: arm64: Remove unused parameter in synchronize_vcpu_pstate()
KVM: arm64: Remove extra argument for __pvkm_host_{share,unshare}_hyp()
KVM: arm64: Inject UNDEF for a register trap without accessor
KVM: arm64: Copy FGT traps to unprotected pKVM VCPU on VCPU load
KVM: arm64: Fix EL2 S1 XN handling for hVHE setups
KVM: arm64: gic: Check for vGICv3 when clearing TWI
Signed-off-by: Marc Zyngier <maz@kernel.org>
+256
-139
+12
-40
arch/arm64/include/asm/kvm_arm.h
+12
-40
arch/arm64/include/asm/kvm_arm.h
···
124
124
#define TCR_EL2_MASK (TCR_EL2_TG0_MASK | TCR_EL2_SH0_MASK | \
125
125
TCR_EL2_ORGN0_MASK | TCR_EL2_IRGN0_MASK)
126
126
127
-
/* VTCR_EL2 Registers bits */
128
-
#define VTCR_EL2_DS TCR_EL2_DS
129
-
#define VTCR_EL2_RES1 (1U << 31)
130
-
#define VTCR_EL2_HD (1 << 22)
131
-
#define VTCR_EL2_HA (1 << 21)
132
-
#define VTCR_EL2_PS_SHIFT TCR_EL2_PS_SHIFT
133
-
#define VTCR_EL2_PS_MASK TCR_EL2_PS_MASK
134
-
#define VTCR_EL2_TG0_MASK TCR_TG0_MASK
135
-
#define VTCR_EL2_TG0_4K TCR_TG0_4K
136
-
#define VTCR_EL2_TG0_16K TCR_TG0_16K
137
-
#define VTCR_EL2_TG0_64K TCR_TG0_64K
138
-
#define VTCR_EL2_SH0_MASK TCR_SH0_MASK
139
-
#define VTCR_EL2_SH0_INNER TCR_SH0_INNER
140
-
#define VTCR_EL2_ORGN0_MASK TCR_ORGN0_MASK
141
-
#define VTCR_EL2_ORGN0_WBWA TCR_ORGN0_WBWA
142
-
#define VTCR_EL2_IRGN0_MASK TCR_IRGN0_MASK
143
-
#define VTCR_EL2_IRGN0_WBWA TCR_IRGN0_WBWA
144
-
#define VTCR_EL2_SL0_SHIFT 6
145
-
#define VTCR_EL2_SL0_MASK (3 << VTCR_EL2_SL0_SHIFT)
146
-
#define VTCR_EL2_T0SZ_MASK 0x3f
147
-
#define VTCR_EL2_VS_SHIFT 19
148
-
#define VTCR_EL2_VS_8BIT (0 << VTCR_EL2_VS_SHIFT)
149
-
#define VTCR_EL2_VS_16BIT (1 << VTCR_EL2_VS_SHIFT)
150
-
151
-
#define VTCR_EL2_T0SZ(x) TCR_T0SZ(x)
152
-
153
127
/*
154
-
* We configure the Stage-2 page tables to always restrict the IPA space to be
155
-
* 40 bits wide (T0SZ = 24). Systems with a PARange smaller than 40 bits are
156
-
* not known to exist and will break with this configuration.
157
-
*
158
128
* The VTCR_EL2 is configured per VM and is initialised in kvm_init_stage2_mmu.
159
129
*
160
130
* Note that when using 4K pages, we concatenate two first level page tables
161
131
* together. With 16K pages, we concatenate 16 first level page tables.
162
132
*
163
133
*/
164
-
165
-
#define VTCR_EL2_COMMON_BITS (VTCR_EL2_SH0_INNER | VTCR_EL2_ORGN0_WBWA | \
166
-
VTCR_EL2_IRGN0_WBWA | VTCR_EL2_RES1)
167
134
168
135
/*
169
136
* VTCR_EL2:SL0 indicates the entry level for Stage2 translation.
···
163
196
*/
164
197
#ifdef CONFIG_ARM64_64K_PAGES
165
198
166
-
#define VTCR_EL2_TGRAN VTCR_EL2_TG0_64K
199
+
#define VTCR_EL2_TGRAN 64K
167
200
#define VTCR_EL2_TGRAN_SL0_BASE 3UL
168
201
169
202
#elif defined(CONFIG_ARM64_16K_PAGES)
170
203
171
-
#define VTCR_EL2_TGRAN VTCR_EL2_TG0_16K
204
+
#define VTCR_EL2_TGRAN 16K
172
205
#define VTCR_EL2_TGRAN_SL0_BASE 3UL
173
206
174
207
#else /* 4K */
175
208
176
-
#define VTCR_EL2_TGRAN VTCR_EL2_TG0_4K
209
+
#define VTCR_EL2_TGRAN 4K
177
210
#define VTCR_EL2_TGRAN_SL0_BASE 2UL
178
211
179
212
#endif
180
213
181
214
#define VTCR_EL2_LVLS_TO_SL0(levels) \
182
-
((VTCR_EL2_TGRAN_SL0_BASE - (4 - (levels))) << VTCR_EL2_SL0_SHIFT)
215
+
FIELD_PREP(VTCR_EL2_SL0, (VTCR_EL2_TGRAN_SL0_BASE - (4 - (levels))))
183
216
#define VTCR_EL2_SL0_TO_LVLS(sl0) \
184
217
((sl0) + 4 - VTCR_EL2_TGRAN_SL0_BASE)
185
218
#define VTCR_EL2_LVLS(vtcr) \
186
-
VTCR_EL2_SL0_TO_LVLS(((vtcr) & VTCR_EL2_SL0_MASK) >> VTCR_EL2_SL0_SHIFT)
219
+
VTCR_EL2_SL0_TO_LVLS(FIELD_GET(VTCR_EL2_SL0, (vtcr)))
187
220
188
-
#define VTCR_EL2_FLAGS (VTCR_EL2_COMMON_BITS | VTCR_EL2_TGRAN)
189
-
#define VTCR_EL2_IPA(vtcr) (64 - ((vtcr) & VTCR_EL2_T0SZ_MASK))
221
+
#define VTCR_EL2_FLAGS (SYS_FIELD_PREP_ENUM(VTCR_EL2, SH0, INNER) | \
222
+
SYS_FIELD_PREP_ENUM(VTCR_EL2, ORGN0, WBWA) | \
223
+
SYS_FIELD_PREP_ENUM(VTCR_EL2, IRGN0, WBWA) | \
224
+
SYS_FIELD_PREP_ENUM(VTCR_EL2, TG0, VTCR_EL2_TGRAN) | \
225
+
VTCR_EL2_RES1)
226
+
227
+
#define VTCR_EL2_IPA(vtcr) (64 - FIELD_GET(VTCR_EL2_T0SZ, (vtcr)))
190
228
191
229
/*
192
230
* ARM VMSAv8-64 defines an algorithm for finding the translation table
-16
arch/arm64/include/asm/kvm_emulate.h
-16
arch/arm64/include/asm/kvm_emulate.h
···
120
120
return (unsigned long *)&vcpu->arch.hcr_el2;
121
121
}
122
122
123
-
static inline void vcpu_clear_wfx_traps(struct kvm_vcpu *vcpu)
124
-
{
125
-
vcpu->arch.hcr_el2 &= ~HCR_TWE;
126
-
if (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) ||
127
-
vcpu->kvm->arch.vgic.nassgireq)
128
-
vcpu->arch.hcr_el2 &= ~HCR_TWI;
129
-
else
130
-
vcpu->arch.hcr_el2 |= HCR_TWI;
131
-
}
132
-
133
-
static inline void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu)
134
-
{
135
-
vcpu->arch.hcr_el2 |= HCR_TWE;
136
-
vcpu->arch.hcr_el2 |= HCR_TWI;
137
-
}
138
-
139
123
static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
140
124
{
141
125
return vcpu->arch.vsesr_el2;
+1
arch/arm64/include/asm/kvm_host.h
+1
arch/arm64/include/asm/kvm_host.h
+5
-3
arch/arm64/include/asm/kvm_pgtable.h
+5
-3
arch/arm64/include/asm/kvm_pgtable.h
···
88
88
#define KVM_PTE_LEAF_ATTR_HI_SW GENMASK(58, 55)
89
89
90
90
#define KVM_PTE_LEAF_ATTR_HI_S1_XN BIT(54)
91
+
#define KVM_PTE_LEAF_ATTR_HI_S1_UXN BIT(54)
92
+
#define KVM_PTE_LEAF_ATTR_HI_S1_PXN BIT(53)
91
93
92
94
#define KVM_PTE_LEAF_ATTR_HI_S2_XN GENMASK(54, 53)
93
95
···
295
293
* children.
296
294
* @KVM_PGTABLE_WALK_SHARED: Indicates the page-tables may be shared
297
295
* with other software walkers.
298
-
* @KVM_PGTABLE_WALK_HANDLE_FAULT: Indicates the page-table walk was
299
-
* invoked from a fault handler.
296
+
* @KVM_PGTABLE_WALK_IGNORE_EAGAIN: Don't terminate the walk early if
297
+
* the walker returns -EAGAIN.
300
298
* @KVM_PGTABLE_WALK_SKIP_BBM_TLBI: Visit and update table entries
301
299
* without Break-before-make's
302
300
* TLB invalidation.
···
309
307
KVM_PGTABLE_WALK_TABLE_PRE = BIT(1),
310
308
KVM_PGTABLE_WALK_TABLE_POST = BIT(2),
311
309
KVM_PGTABLE_WALK_SHARED = BIT(3),
312
-
KVM_PGTABLE_WALK_HANDLE_FAULT = BIT(4),
310
+
KVM_PGTABLE_WALK_IGNORE_EAGAIN = BIT(4),
313
311
KVM_PGTABLE_WALK_SKIP_BBM_TLBI = BIT(5),
314
312
KVM_PGTABLE_WALK_SKIP_CMO = BIT(6),
315
313
};
+2
-2
arch/arm64/include/asm/sysreg.h
+2
-2
arch/arm64/include/asm/sysreg.h
···
91
91
*/
92
92
#define pstate_field(op1, op2) ((op1) << Op1_shift | (op2) << Op2_shift)
93
93
#define PSTATE_Imm_shift CRm_shift
94
-
#define SET_PSTATE(x, r) __emit_inst(0xd500401f | PSTATE_ ## r | ((!!x) << PSTATE_Imm_shift))
94
+
#define ENCODE_PSTATE(x, r) (0xd500401f | PSTATE_ ## r | ((!!x) << PSTATE_Imm_shift))
95
+
#define SET_PSTATE(x, r) __emit_inst(ENCODE_PSTATE(x, r))
95
96
96
97
#define PSTATE_PAN pstate_field(0, 4)
97
98
#define PSTATE_UAO pstate_field(0, 3)
···
517
516
#define SYS_TTBR1_EL2 sys_reg(3, 4, 2, 0, 1)
518
517
#define SYS_TCR_EL2 sys_reg(3, 4, 2, 0, 2)
519
518
#define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0)
520
-
#define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2)
521
519
522
520
#define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6)
523
521
#define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0)
+1
arch/arm64/kvm/arm.c
+1
arch/arm64/kvm/arm.c
+6
-2
arch/arm64/kvm/at.c
+6
-2
arch/arm64/kvm/at.c
···
403
403
struct s1_walk_result *wr, u64 va)
404
404
{
405
405
u64 va_top, va_bottom, baddr, desc, new_desc, ipa;
406
+
struct kvm_s2_trans s2_trans = {};
406
407
int level, stride, ret;
407
408
408
409
level = wi->sl;
···
421
420
ipa = baddr | index;
422
421
423
422
if (wi->s2) {
424
-
struct kvm_s2_trans s2_trans = {};
425
-
426
423
ret = kvm_walk_nested_s2(vcpu, ipa, &s2_trans);
427
424
if (ret) {
428
425
fail_s1_walk(wr,
···
514
515
new_desc |= PTE_AF;
515
516
516
517
if (new_desc != desc) {
518
+
if (wi->s2 && !kvm_s2_trans_writable(&s2_trans)) {
519
+
fail_s1_walk(wr, ESR_ELx_FSC_PERM_L(level), true);
520
+
return -EPERM;
521
+
}
522
+
517
523
ret = kvm_swap_s1_desc(vcpu, ipa, desc, new_desc, wi);
518
524
if (ret)
519
525
return ret;
+82
-12
arch/arm64/kvm/config.c
+82
-12
arch/arm64/kvm/config.c
···
16
16
*/
17
17
struct reg_bits_to_feat_map {
18
18
union {
19
-
u64 bits;
20
-
u64 *res0p;
19
+
u64 bits;
20
+
struct fgt_masks *masks;
21
21
};
22
22
23
23
#define NEVER_FGU BIT(0) /* Can trap, but never UNDEF */
24
24
#define CALL_FUNC BIT(1) /* Needs to evaluate tons of crap */
25
25
#define FIXED_VALUE BIT(2) /* RAZ/WI or RAO/WI in KVM */
26
-
#define RES0_POINTER BIT(3) /* Pointer to RES0 value instead of bits */
26
+
#define MASKS_POINTER BIT(3) /* Pointer to fgt_masks struct instead of bits */
27
27
28
28
unsigned long flags;
29
29
···
92
92
#define NEEDS_FEAT_FIXED(m, ...) \
93
93
__NEEDS_FEAT_FLAG(m, FIXED_VALUE, bits, __VA_ARGS__, 0)
94
94
95
-
#define NEEDS_FEAT_RES0(p, ...) \
96
-
__NEEDS_FEAT_FLAG(p, RES0_POINTER, res0p, __VA_ARGS__)
95
+
#define NEEDS_FEAT_MASKS(p, ...) \
96
+
__NEEDS_FEAT_FLAG(p, MASKS_POINTER, masks, __VA_ARGS__)
97
97
98
98
/*
99
99
* Declare the dependency between a set of bits and a set of features,
···
109
109
#define DECLARE_FEAT_MAP(n, r, m, f) \
110
110
struct reg_feat_map_desc n = { \
111
111
.name = #r, \
112
-
.feat_map = NEEDS_FEAT(~r##_RES0, f), \
112
+
.feat_map = NEEDS_FEAT(~(r##_RES0 | \
113
+
r##_RES1), f), \
113
114
.bit_feat_map = m, \
114
115
.bit_feat_map_sz = ARRAY_SIZE(m), \
115
116
}
116
117
117
118
/*
118
119
* Specialised version of the above for FGT registers that have their
119
-
* RES0 masks described as struct fgt_masks.
120
+
* RESx masks described as struct fgt_masks.
120
121
*/
121
122
#define DECLARE_FEAT_MAP_FGT(n, msk, m, f) \
122
123
struct reg_feat_map_desc n = { \
123
124
.name = #msk, \
124
-
.feat_map = NEEDS_FEAT_RES0(&msk.res0, f),\
125
+
.feat_map = NEEDS_FEAT_MASKS(&msk, f), \
125
126
.bit_feat_map = m, \
126
127
.bit_feat_map_sz = ARRAY_SIZE(m), \
127
128
}
···
141
140
#define FEAT_AA64EL1 ID_AA64PFR0_EL1, EL1, IMP
142
141
#define FEAT_AA64EL2 ID_AA64PFR0_EL1, EL2, IMP
143
142
#define FEAT_AA64EL3 ID_AA64PFR0_EL1, EL3, IMP
143
+
#define FEAT_SEL2 ID_AA64PFR0_EL1, SEL2, IMP
144
144
#define FEAT_AIE ID_AA64MMFR3_EL1, AIE, IMP
145
145
#define FEAT_S2POE ID_AA64MMFR3_EL1, S2POE, IMP
146
146
#define FEAT_S1POE ID_AA64MMFR3_EL1, S1POE, IMP
···
203
201
#define FEAT_ASID2 ID_AA64MMFR4_EL1, ASID2, IMP
204
202
#define FEAT_MEC ID_AA64MMFR3_EL1, MEC, IMP
205
203
#define FEAT_HAFT ID_AA64MMFR1_EL1, HAFDBS, HAFT
204
+
#define FEAT_HDBSS ID_AA64MMFR1_EL1, HAFDBS, HDBSS
205
+
#define FEAT_HPDS2 ID_AA64MMFR1_EL1, HPDS, HPDS2
206
206
#define FEAT_BTI ID_AA64PFR1_EL1, BT, IMP
207
207
#define FEAT_ExS ID_AA64MMFR0_EL1, EXS, IMP
208
208
#define FEAT_IESB ID_AA64MMFR2_EL1, IESB, IMP
···
222
218
#define FEAT_FGT2 ID_AA64MMFR0_EL1, FGT, FGT2
223
219
#define FEAT_MTPMU ID_AA64DFR0_EL1, MTPMU, IMP
224
220
#define FEAT_HCX ID_AA64MMFR1_EL1, HCX, IMP
221
+
#define FEAT_S2PIE ID_AA64MMFR3_EL1, S2PIE, IMP
225
222
226
223
static bool not_feat_aa64el3(struct kvm *kvm)
227
224
{
···
364
359
static bool feat_pmuv3p9(struct kvm *kvm)
365
360
{
366
361
return check_pmu_revision(kvm, V3P9);
362
+
}
363
+
364
+
#define has_feat_s2tgran(k, s) \
365
+
((kvm_has_feat_enum(kvm, ID_AA64MMFR0_EL1, TGRAN##s##_2, TGRAN##s) && \
366
+
kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN##s, IMP)) || \
367
+
kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN##s##_2, IMP))
368
+
369
+
static bool feat_lpa2(struct kvm *kvm)
370
+
{
371
+
return ((kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN4, 52_BIT) ||
372
+
!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN4, IMP)) &&
373
+
(kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN16, 52_BIT) ||
374
+
!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN16, IMP)) &&
375
+
(kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN4_2, 52_BIT) ||
376
+
!has_feat_s2tgran(kvm, 4)) &&
377
+
(kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN16_2, 52_BIT) ||
378
+
!has_feat_s2tgran(kvm, 16)));
379
+
}
380
+
381
+
static bool feat_vmid16(struct kvm *kvm)
382
+
{
383
+
return kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16);
367
384
}
368
385
369
386
static bool compute_hcr_rw(struct kvm *kvm, u64 *bits)
···
1194
1167
static const DECLARE_FEAT_MAP(mdcr_el2_desc, MDCR_EL2,
1195
1168
mdcr_el2_feat_map, FEAT_AA64EL2);
1196
1169
1170
+
static const struct reg_bits_to_feat_map vtcr_el2_feat_map[] = {
1171
+
NEEDS_FEAT(VTCR_EL2_HDBSS, FEAT_HDBSS),
1172
+
NEEDS_FEAT(VTCR_EL2_HAFT, FEAT_HAFT),
1173
+
NEEDS_FEAT(VTCR_EL2_TL0 |
1174
+
VTCR_EL2_TL1 |
1175
+
VTCR_EL2_AssuredOnly |
1176
+
VTCR_EL2_GCSH,
1177
+
FEAT_THE),
1178
+
NEEDS_FEAT(VTCR_EL2_D128, FEAT_D128),
1179
+
NEEDS_FEAT(VTCR_EL2_S2POE, FEAT_S2POE),
1180
+
NEEDS_FEAT(VTCR_EL2_S2PIE, FEAT_S2PIE),
1181
+
NEEDS_FEAT(VTCR_EL2_SL2 |
1182
+
VTCR_EL2_DS,
1183
+
feat_lpa2),
1184
+
NEEDS_FEAT(VTCR_EL2_NSA |
1185
+
VTCR_EL2_NSW,
1186
+
FEAT_SEL2),
1187
+
NEEDS_FEAT(VTCR_EL2_HWU62 |
1188
+
VTCR_EL2_HWU61 |
1189
+
VTCR_EL2_HWU60 |
1190
+
VTCR_EL2_HWU59,
1191
+
FEAT_HPDS2),
1192
+
NEEDS_FEAT(VTCR_EL2_HD, ID_AA64MMFR1_EL1, HAFDBS, DBM),
1193
+
NEEDS_FEAT(VTCR_EL2_HA, ID_AA64MMFR1_EL1, HAFDBS, AF),
1194
+
NEEDS_FEAT(VTCR_EL2_VS, feat_vmid16),
1195
+
NEEDS_FEAT(VTCR_EL2_PS |
1196
+
VTCR_EL2_TG0 |
1197
+
VTCR_EL2_SH0 |
1198
+
VTCR_EL2_ORGN0 |
1199
+
VTCR_EL2_IRGN0 |
1200
+
VTCR_EL2_SL0 |
1201
+
VTCR_EL2_T0SZ,
1202
+
FEAT_AA64EL1),
1203
+
};
1204
+
1205
+
static const DECLARE_FEAT_MAP(vtcr_el2_desc, VTCR_EL2,
1206
+
vtcr_el2_feat_map, FEAT_AA64EL2);
1207
+
1197
1208
static void __init check_feat_map(const struct reg_bits_to_feat_map *map,
1198
-
int map_size, u64 res0, const char *str)
1209
+
int map_size, u64 resx, const char *str)
1199
1210
{
1200
1211
u64 mask = 0;
1201
1212
1202
1213
for (int i = 0; i < map_size; i++)
1203
1214
mask |= map[i].bits;
1204
1215
1205
-
if (mask != ~res0)
1216
+
if (mask != ~resx)
1206
1217
kvm_err("Undefined %s behaviour, bits %016llx\n",
1207
-
str, mask ^ ~res0);
1218
+
str, mask ^ ~resx);
1208
1219
}
1209
1220
1210
1221
static u64 reg_feat_map_bits(const struct reg_bits_to_feat_map *map)
1211
1222
{
1212
-
return map->flags & RES0_POINTER ? ~(*map->res0p) : map->bits;
1223
+
return map->flags & MASKS_POINTER ? (map->masks->mask | map->masks->nmask) : map->bits;
1213
1224
}
1214
1225
1215
1226
static void __init check_reg_desc(const struct reg_feat_map_desc *r)
···
1275
1210
check_reg_desc(&tcr2_el2_desc);
1276
1211
check_reg_desc(&sctlr_el1_desc);
1277
1212
check_reg_desc(&mdcr_el2_desc);
1213
+
check_reg_desc(&vtcr_el2_desc);
1278
1214
}
1279
1215
1280
1216
static bool idreg_feat_match(struct kvm *kvm, const struct reg_bits_to_feat_map *map)
···
1489
1423
case MDCR_EL2:
1490
1424
*res0 = compute_reg_res0_bits(kvm, &mdcr_el2_desc, 0, 0);
1491
1425
*res1 = MDCR_EL2_RES1;
1426
+
break;
1427
+
case VTCR_EL2:
1428
+
*res0 = compute_reg_res0_bits(kvm, &vtcr_el2_desc, 0, 0);
1429
+
*res1 = VTCR_EL2_RES1;
1492
1430
break;
1493
1431
default:
1494
1432
WARN_ON_ONCE(1);
+40
-33
arch/arm64/kvm/emulate-nested.c
+40
-33
arch/arm64/kvm/emulate-nested.c
···
2105
2105
}
2106
2106
2107
2107
#define FGT_MASKS(__n, __m) \
2108
-
struct fgt_masks __n = { .str = #__m, .res0 = __m, }
2108
+
struct fgt_masks __n = { .str = #__m, .res0 = __m ## _RES0, .res1 = __m ## _RES1 }
2109
2109
2110
-
FGT_MASKS(hfgrtr_masks, HFGRTR_EL2_RES0);
2111
-
FGT_MASKS(hfgwtr_masks, HFGWTR_EL2_RES0);
2112
-
FGT_MASKS(hfgitr_masks, HFGITR_EL2_RES0);
2113
-
FGT_MASKS(hdfgrtr_masks, HDFGRTR_EL2_RES0);
2114
-
FGT_MASKS(hdfgwtr_masks, HDFGWTR_EL2_RES0);
2115
-
FGT_MASKS(hafgrtr_masks, HAFGRTR_EL2_RES0);
2116
-
FGT_MASKS(hfgrtr2_masks, HFGRTR2_EL2_RES0);
2117
-
FGT_MASKS(hfgwtr2_masks, HFGWTR2_EL2_RES0);
2118
-
FGT_MASKS(hfgitr2_masks, HFGITR2_EL2_RES0);
2119
-
FGT_MASKS(hdfgrtr2_masks, HDFGRTR2_EL2_RES0);
2120
-
FGT_MASKS(hdfgwtr2_masks, HDFGWTR2_EL2_RES0);
2110
+
FGT_MASKS(hfgrtr_masks, HFGRTR_EL2);
2111
+
FGT_MASKS(hfgwtr_masks, HFGWTR_EL2);
2112
+
FGT_MASKS(hfgitr_masks, HFGITR_EL2);
2113
+
FGT_MASKS(hdfgrtr_masks, HDFGRTR_EL2);
2114
+
FGT_MASKS(hdfgwtr_masks, HDFGWTR_EL2);
2115
+
FGT_MASKS(hafgrtr_masks, HAFGRTR_EL2);
2116
+
FGT_MASKS(hfgrtr2_masks, HFGRTR2_EL2);
2117
+
FGT_MASKS(hfgwtr2_masks, HFGWTR2_EL2);
2118
+
FGT_MASKS(hfgitr2_masks, HFGITR2_EL2);
2119
+
FGT_MASKS(hdfgrtr2_masks, HDFGRTR2_EL2);
2120
+
FGT_MASKS(hdfgwtr2_masks, HDFGWTR2_EL2);
2121
2121
2122
2122
static __init bool aggregate_fgt(union trap_config tc)
2123
2123
{
2124
2124
struct fgt_masks *rmasks, *wmasks;
2125
+
u64 rresx, wresx;
2125
2126
2126
2127
switch (tc.fgt) {
2127
2128
case HFGRTR_GROUP:
···
2155
2154
break;
2156
2155
}
2157
2156
2157
+
rresx = rmasks->res0 | rmasks->res1;
2158
+
if (wmasks)
2159
+
wresx = wmasks->res0 | wmasks->res1;
2160
+
2158
2161
/*
2159
2162
* A bit can be reserved in either the R or W register, but
2160
2163
* not both.
2161
2164
*/
2162
-
if ((BIT(tc.bit) & rmasks->res0) &&
2163
-
(!wmasks || (BIT(tc.bit) & wmasks->res0)))
2165
+
if ((BIT(tc.bit) & rresx) && (!wmasks || (BIT(tc.bit) & wresx)))
2164
2166
return false;
2165
2167
2166
2168
if (tc.pol)
2167
-
rmasks->mask |= BIT(tc.bit) & ~rmasks->res0;
2169
+
rmasks->mask |= BIT(tc.bit) & ~rresx;
2168
2170
else
2169
-
rmasks->nmask |= BIT(tc.bit) & ~rmasks->res0;
2171
+
rmasks->nmask |= BIT(tc.bit) & ~rresx;
2170
2172
2171
2173
if (wmasks) {
2172
2174
if (tc.pol)
2173
-
wmasks->mask |= BIT(tc.bit) & ~wmasks->res0;
2175
+
wmasks->mask |= BIT(tc.bit) & ~wresx;
2174
2176
else
2175
-
wmasks->nmask |= BIT(tc.bit) & ~wmasks->res0;
2177
+
wmasks->nmask |= BIT(tc.bit) & ~wresx;
2176
2178
}
2177
2179
2178
2180
return true;
···
2184
2180
static __init int check_fgt_masks(struct fgt_masks *masks)
2185
2181
{
2186
2182
unsigned long duplicate = masks->mask & masks->nmask;
2187
-
u64 res0 = masks->res0;
2188
2183
int ret = 0;
2189
2184
2190
2185
if (duplicate) {
···
2197
2194
ret = -EINVAL;
2198
2195
}
2199
2196
2200
-
masks->res0 = ~(masks->mask | masks->nmask);
2201
-
if (masks->res0 != res0)
2202
-
kvm_info("Implicit %s = %016llx, expecting %016llx\n",
2203
-
masks->str, masks->res0, res0);
2197
+
if ((masks->res0 | masks->res1 | masks->mask | masks->nmask) != GENMASK(63, 0) ||
2198
+
(masks->res0 & masks->res1) || (masks->res0 & masks->mask) ||
2199
+
(masks->res0 & masks->nmask) || (masks->res1 & masks->mask) ||
2200
+
(masks->res1 & masks->nmask) || (masks->mask & masks->nmask)) {
2201
+
kvm_info("Inconsistent masks for %s (%016llx, %016llx, %016llx, %016llx)\n",
2202
+
masks->str, masks->res0, masks->res1, masks->mask, masks->nmask);
2203
+
masks->res0 = ~(masks->res1 | masks->mask | masks->nmask);
2204
+
}
2204
2205
2205
2206
return ret;
2206
2207
}
···
2276
2269
kvm_info("nv: %ld coarse grained trap handlers\n",
2277
2270
ARRAY_SIZE(encoding_to_cgt));
2278
2271
2279
-
if (!cpus_have_final_cap(ARM64_HAS_FGT))
2280
-
goto check_mcb;
2281
-
2282
2272
for (int i = 0; i < ARRAY_SIZE(encoding_to_fgt); i++) {
2283
2273
const struct encoding_to_trap_config *fgt = &encoding_to_fgt[i];
2284
2274
union trap_config tc;
···
2295
2291
}
2296
2292
2297
2293
tc.val |= fgt->tc.val;
2294
+
2295
+
if (!aggregate_fgt(tc)) {
2296
+
ret = -EINVAL;
2297
+
print_nv_trap_error(fgt, "FGT bit is reserved", ret);
2298
+
}
2299
+
2300
+
if (!cpus_have_final_cap(ARM64_HAS_FGT))
2301
+
continue;
2302
+
2298
2303
prev = xa_store(&sr_forward_xa, enc,
2299
2304
xa_mk_value(tc.val), GFP_KERNEL);
2300
2305
2301
2306
if (xa_is_err(prev)) {
2302
2307
ret = xa_err(prev);
2303
2308
print_nv_trap_error(fgt, "Failed FGT insertion", ret);
2304
-
}
2305
-
2306
-
if (!aggregate_fgt(tc)) {
2307
-
ret = -EINVAL;
2308
-
print_nv_trap_error(fgt, "FGT bit is reserved", ret);
2309
2309
}
2310
2310
}
2311
2311
}
···
2326
2318
kvm_info("nv: %ld fine grained trap handlers\n",
2327
2319
ARRAY_SIZE(encoding_to_fgt));
2328
2320
2329
-
check_mcb:
2330
2321
for (int id = __MULTIPLE_CONTROL_BITS__; id < __COMPLEX_CONDITIONS__; id++) {
2331
2322
const enum cgt_group_id *cgids;
2332
2323
···
2435
2428
2436
2429
masks = kvm->arch.sysreg_masks;
2437
2430
2438
-
return masks->mask[sr - __VNCR_START__].res0;
2431
+
return masks->mask[sr - __SANITISED_REG_START__].res0;
2439
2432
}
2440
2433
2441
2434
static bool check_fgt_bit(struct kvm_vcpu *vcpu, enum vcpu_sysreg sr,
+1
-1
arch/arm64/kvm/hyp/include/hyp/switch.h
+1
-1
arch/arm64/kvm/hyp/include/hyp/switch.h
···
854
854
return false;
855
855
}
856
856
857
-
static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu, u64 *exit_code)
857
+
static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu)
858
858
{
859
859
/*
860
860
* Check for the conditions of Cortex-A510's #2077057. When these occur
+3
arch/arm64/kvm/hyp/nvhe/hyp-main.c
+3
arch/arm64/kvm/hyp/nvhe/hyp-main.c
···
180
180
/* Propagate WFx trapping flags */
181
181
hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI);
182
182
hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_TWE | HCR_TWI);
183
+
} else {
184
+
memcpy(&hyp_vcpu->vcpu.arch.fgt, hyp_vcpu->host_vcpu->arch.fgt,
185
+
sizeof(hyp_vcpu->vcpu.arch.fgt));
183
186
}
184
187
}
185
188
-1
arch/arm64/kvm/hyp/nvhe/pkvm.c
-1
arch/arm64/kvm/hyp/nvhe/pkvm.c
+1
-1
arch/arm64/kvm/hyp/nvhe/switch.c
+1
-1
arch/arm64/kvm/hyp/nvhe/switch.c
+27
-10
arch/arm64/kvm/hyp/pgtable.c
+27
-10
arch/arm64/kvm/hyp/pgtable.c
···
144
144
* page table walk.
145
145
*/
146
146
if (r == -EAGAIN)
147
-
return !(walker->flags & KVM_PGTABLE_WALK_HANDLE_FAULT);
147
+
return walker->flags & KVM_PGTABLE_WALK_IGNORE_EAGAIN;
148
148
149
149
return !r;
150
150
}
···
342
342
if (!(prot & KVM_PGTABLE_PROT_R))
343
343
return -EINVAL;
344
344
345
+
if (!cpus_have_final_cap(ARM64_KVM_HVHE))
346
+
prot &= ~KVM_PGTABLE_PROT_UX;
347
+
345
348
if (prot & KVM_PGTABLE_PROT_X) {
346
349
if (prot & KVM_PGTABLE_PROT_W)
347
350
return -EINVAL;
···
354
351
355
352
if (system_supports_bti_kernel())
356
353
attr |= KVM_PTE_LEAF_ATTR_HI_S1_GP;
354
+
}
355
+
356
+
if (cpus_have_final_cap(ARM64_KVM_HVHE)) {
357
+
if (!(prot & KVM_PGTABLE_PROT_PX))
358
+
attr |= KVM_PTE_LEAF_ATTR_HI_S1_PXN;
359
+
if (!(prot & KVM_PGTABLE_PROT_UX))
360
+
attr |= KVM_PTE_LEAF_ATTR_HI_S1_UXN;
357
361
} else {
358
-
attr |= KVM_PTE_LEAF_ATTR_HI_S1_XN;
362
+
if (!(prot & KVM_PGTABLE_PROT_PX))
363
+
attr |= KVM_PTE_LEAF_ATTR_HI_S1_XN;
359
364
}
360
365
361
366
attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_AP, ap);
···
384
373
if (!kvm_pte_valid(pte))
385
374
return prot;
386
375
387
-
if (!(pte & KVM_PTE_LEAF_ATTR_HI_S1_XN))
388
-
prot |= KVM_PGTABLE_PROT_X;
376
+
if (cpus_have_final_cap(ARM64_KVM_HVHE)) {
377
+
if (!(pte & KVM_PTE_LEAF_ATTR_HI_S1_PXN))
378
+
prot |= KVM_PGTABLE_PROT_PX;
379
+
if (!(pte & KVM_PTE_LEAF_ATTR_HI_S1_UXN))
380
+
prot |= KVM_PGTABLE_PROT_UX;
381
+
} else {
382
+
if (!(pte & KVM_PTE_LEAF_ATTR_HI_S1_XN))
383
+
prot |= KVM_PGTABLE_PROT_PX;
384
+
}
389
385
390
386
ap = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S1_AP, pte);
391
387
if (ap == KVM_PTE_LEAF_ATTR_LO_S1_AP_RO)
···
601
583
u64 vtcr = VTCR_EL2_FLAGS;
602
584
s8 lvls;
603
585
604
-
vtcr |= kvm_get_parange(mmfr0) << VTCR_EL2_PS_SHIFT;
605
-
vtcr |= VTCR_EL2_T0SZ(phys_shift);
586
+
vtcr |= FIELD_PREP(VTCR_EL2_PS, kvm_get_parange(mmfr0));
587
+
vtcr |= FIELD_PREP(VTCR_EL2_T0SZ, (UL(64) - phys_shift));
606
588
/*
607
589
* Use a minimum 2 level page table to prevent splitting
608
590
* host PMD huge pages at stage2.
···
642
624
vtcr |= VTCR_EL2_DS;
643
625
644
626
/* Set the vmid bits */
645
-
vtcr |= (get_vmid_bits(mmfr1) == 16) ?
646
-
VTCR_EL2_VS_16BIT :
647
-
VTCR_EL2_VS_8BIT;
627
+
vtcr |= (get_vmid_bits(mmfr1) == 16) ? VTCR_EL2_VS : 0;
648
628
649
629
return vtcr;
650
630
}
···
1278
1262
{
1279
1263
return stage2_update_leaf_attrs(pgt, addr, size, 0,
1280
1264
KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W,
1281
-
NULL, NULL, 0);
1265
+
NULL, NULL,
1266
+
KVM_PGTABLE_WALK_IGNORE_EAGAIN);
1282
1267
}
1283
1268
1284
1269
void kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,
+1
-1
arch/arm64/kvm/hyp/vhe/switch.c
+1
-1
arch/arm64/kvm/hyp/vhe/switch.c
+5
-7
arch/arm64/kvm/mmu.c
+5
-7
arch/arm64/kvm/mmu.c
···
497
497
this->count = 1;
498
498
rb_link_node(&this->node, parent, node);
499
499
rb_insert_color(&this->node, &hyp_shared_pfns);
500
-
ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, pfn, 1);
500
+
ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, pfn);
501
501
unlock:
502
502
mutex_unlock(&hyp_shared_pfns_lock);
503
503
···
523
523
524
524
rb_erase(&this->node, &hyp_shared_pfns);
525
525
kfree(this);
526
-
ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn, 1);
526
+
ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn);
527
527
unlock:
528
528
mutex_unlock(&hyp_shared_pfns_lock);
529
529
···
1563
1563
*prot &= ~KVM_PGTABLE_PROT_PX;
1564
1564
}
1565
1565
1566
-
#define KVM_PGTABLE_WALK_MEMABORT_FLAGS (KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED)
1567
-
1568
1566
static int gmem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
1569
1567
struct kvm_s2_trans *nested,
1570
1568
struct kvm_memory_slot *memslot, bool is_perm)
1571
1569
{
1572
1570
bool write_fault, exec_fault, writable;
1573
-
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_MEMABORT_FLAGS;
1571
+
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED;
1574
1572
enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
1575
1573
struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt;
1576
1574
unsigned long mmu_seq;
···
1663
1665
struct kvm_pgtable *pgt;
1664
1666
struct page *page;
1665
1667
vm_flags_t vm_flags;
1666
-
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_MEMABORT_FLAGS;
1668
+
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED;
1667
1669
1668
1670
if (fault_is_perm)
1669
1671
fault_granule = kvm_vcpu_trap_get_perm_fault_granule(vcpu);
···
1942
1944
/* Resolve the access fault by making the page young again. */
1943
1945
static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
1944
1946
{
1945
-
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED;
1947
+
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED;
1946
1948
struct kvm_s2_mmu *mmu;
1947
1949
1948
1950
trace_kvm_access_fault(fault_ipa);
+5
-6
arch/arm64/kvm/nested.c
+5
-6
arch/arm64/kvm/nested.c
···
377
377
{
378
378
wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK;
379
379
380
-
switch (vtcr & VTCR_EL2_TG0_MASK) {
380
+
switch (FIELD_GET(VTCR_EL2_TG0_MASK, vtcr)) {
381
381
case VTCR_EL2_TG0_4K:
382
382
wi->pgshift = 12; break;
383
383
case VTCR_EL2_TG0_16K:
···
513
513
514
514
lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock);
515
515
516
-
switch (vtcr & VTCR_EL2_TG0_MASK) {
516
+
switch (FIELD_GET(VTCR_EL2_TG0_MASK, vtcr)) {
517
517
case VTCR_EL2_TG0_4K:
518
518
ttl = (TLBI_TTL_TG_4K << 2);
519
519
break;
···
530
530
531
531
again:
532
532
/* Iteratively compute the block sizes for a particular granule size */
533
-
switch (vtcr & VTCR_EL2_TG0_MASK) {
533
+
switch (FIELD_GET(VTCR_EL2_TG0_MASK, vtcr)) {
534
534
case VTCR_EL2_TG0_4K:
535
535
if (sz < SZ_4K) sz = SZ_4K;
536
536
else if (sz < SZ_2M) sz = SZ_2M;
···
593
593
594
594
if (!max_size) {
595
595
/* Compute the maximum extent of the invalidation */
596
-
switch (mmu->tlb_vtcr & VTCR_EL2_TG0_MASK) {
596
+
switch (FIELD_GET(VTCR_EL2_TG0_MASK, mmu->tlb_vtcr)) {
597
597
case VTCR_EL2_TG0_4K:
598
598
max_size = SZ_1G;
599
599
break;
···
1719
1719
set_sysreg_masks(kvm, VTTBR_EL2, res0, res1);
1720
1720
1721
1721
/* VTCR_EL2 */
1722
-
res0 = GENMASK(63, 32) | GENMASK(30, 20);
1723
-
res1 = BIT(31);
1722
+
get_reg_fixed_bits(kvm, VTCR_EL2, &res0, &res1);
1724
1723
set_sysreg_masks(kvm, VTCR_EL2, res0, res1);
1725
1724
1726
1725
/* VMPIDR_EL2 */
+4
-1
arch/arm64/kvm/sys_regs.c
+4
-1
arch/arm64/kvm/sys_regs.c
···
4668
4668
* that we don't know how to handle. This certainly qualifies
4669
4669
* as a gross bug that should be fixed right away.
4670
4670
*/
4671
-
BUG_ON(!r->access);
4671
+
if (!r->access) {
4672
+
bad_trap(vcpu, params, r, "register access");
4673
+
return;
4674
+
}
4672
4675
4673
4676
/* Skip instruction if instructed so */
4674
4677
if (likely(r->access(vcpu, params, r)))
+60
-3
arch/arm64/tools/sysreg
+60
-3
arch/arm64/tools/sysreg
···
2098
2098
0b0000 NI
2099
2099
0b0001 IMP
2100
2100
EndEnum
2101
-
Enum 43:40 TGRAN4_2
2101
+
UnsignedEnum 43:40 TGRAN4_2
2102
2102
0b0000 TGRAN4
2103
2103
0b0001 NI
2104
2104
0b0010 IMP
2105
2105
0b0011 52_BIT
2106
2106
EndEnum
2107
-
Enum 39:36 TGRAN64_2
2107
+
UnsignedEnum 39:36 TGRAN64_2
2108
2108
0b0000 TGRAN64
2109
2109
0b0001 NI
2110
2110
0b0010 IMP
2111
2111
EndEnum
2112
-
Enum 35:32 TGRAN16_2
2112
+
UnsignedEnum 35:32 TGRAN16_2
2113
2113
0b0000 TGRAN16
2114
2114
0b0001 NI
2115
2115
0b0010 IMP
···
4398
4398
Field 63:57 RESS
4399
4399
Field 56:12 BADDR
4400
4400
Res0 11:0
4401
+
EndSysreg
4402
+
4403
+
Sysreg VTCR_EL2 3 4 2 1 2
4404
+
Res0 63:46
4405
+
Field 45 HDBSS
4406
+
Field 44 HAFT
4407
+
Res0 43:42
4408
+
Field 41 TL0
4409
+
Field 40 GCSH
4410
+
Res0 39
4411
+
Field 38 D128
4412
+
Field 37 S2POE
4413
+
Field 36 S2PIE
4414
+
Field 35 TL1
4415
+
Field 34 AssuredOnly
4416
+
Field 33 SL2
4417
+
Field 32 DS
4418
+
Res1 31
4419
+
Field 30 NSA
4420
+
Field 29 NSW
4421
+
Field 28 HWU62
4422
+
Field 27 HWU61
4423
+
Field 26 HWU60
4424
+
Field 25 HWU59
4425
+
Res0 24:23
4426
+
Field 22 HD
4427
+
Field 21 HA
4428
+
Res0 20
4429
+
Enum 19 VS
4430
+
0b0 8BIT
4431
+
0b1 16BIT
4432
+
EndEnum
4433
+
Field 18:16 PS
4434
+
Enum 15:14 TG0
4435
+
0b00 4K
4436
+
0b01 64K
4437
+
0b10 16K
4438
+
EndEnum
4439
+
Enum 13:12 SH0
4440
+
0b00 NONE
4441
+
0b01 OUTER
4442
+
0b11 INNER
4443
+
EndEnum
4444
+
Enum 11:10 ORGN0
4445
+
0b00 NC
4446
+
0b01 WBWA
4447
+
0b10 WT
4448
+
0b11 WBnWA
4449
+
EndEnum
4450
+
Enum 9:8 IRGN0
4451
+
0b00 NC
4452
+
0b01 WBWA
4453
+
0b10 WT
4454
+
0b11 WBnWA
4455
+
EndEnum
4456
+
Field 7:6 SL0
4457
+
Field 5:0 T0SZ
4401
4458
EndSysreg
4402
4459
4403
4460
Sysreg GCSCR_EL2 3 4 2 5 0