Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'kvm-riscv-6.17-2' of https://github.com/kvm-riscv/linux into HEAD
KVM/riscv changes for 6.17
- Enabled ring-based dirty memory tracking
- Improved perf kvm stat to report interrupt events
- Delegate illegal instruction trap to VS-mode
- MMU related improvements for KVM RISC-V for upcoming
nested virtualization
+1000
-681
+1
-1
Documentation/virt/kvm/api.rst
+1
-1
Documentation/virt/kvm/api.rst
···
8387
8387
7.36 KVM_CAP_DIRTY_LOG_RING/KVM_CAP_DIRTY_LOG_RING_ACQ_REL
8388
8388
----------------------------------------------------------
8389
8389
8390
-
:Architectures: x86, arm64
8390
+
:Architectures: x86, arm64, riscv
8391
8391
:Type: vm
8392
8392
:Parameters: args[0] - size of the dirty log ring
8393
8393
+1
-1
arch/riscv/include/asm/kvm_aia.h
+1
-1
arch/riscv/include/asm/kvm_aia.h
···
150
150
151
151
int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu);
152
152
void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu);
153
-
int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu);
153
+
void kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu);
154
154
void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu);
155
155
156
156
int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index,
+72
arch/riscv/include/asm/kvm_gstage.h
+72
arch/riscv/include/asm/kvm_gstage.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* Copyright (C) 2019 Western Digital Corporation or its affiliates.
4
+
* Copyright (c) 2025 Ventana Micro Systems Inc.
5
+
*/
6
+
7
+
#ifndef __RISCV_KVM_GSTAGE_H_
8
+
#define __RISCV_KVM_GSTAGE_H_
9
+
10
+
#include <linux/kvm_types.h>
11
+
12
+
struct kvm_gstage {
13
+
struct kvm *kvm;
14
+
unsigned long flags;
15
+
#define KVM_GSTAGE_FLAGS_LOCAL BIT(0)
16
+
unsigned long vmid;
17
+
pgd_t *pgd;
18
+
};
19
+
20
+
struct kvm_gstage_mapping {
21
+
gpa_t addr;
22
+
pte_t pte;
23
+
u32 level;
24
+
};
25
+
26
+
#ifdef CONFIG_64BIT
27
+
#define kvm_riscv_gstage_index_bits 9
28
+
#else
29
+
#define kvm_riscv_gstage_index_bits 10
30
+
#endif
31
+
32
+
extern unsigned long kvm_riscv_gstage_mode;
33
+
extern unsigned long kvm_riscv_gstage_pgd_levels;
34
+
35
+
#define kvm_riscv_gstage_pgd_xbits 2
36
+
#define kvm_riscv_gstage_pgd_size (1UL << (HGATP_PAGE_SHIFT + kvm_riscv_gstage_pgd_xbits))
37
+
#define kvm_riscv_gstage_gpa_bits (HGATP_PAGE_SHIFT + \
38
+
(kvm_riscv_gstage_pgd_levels * \
39
+
kvm_riscv_gstage_index_bits) + \
40
+
kvm_riscv_gstage_pgd_xbits)
41
+
#define kvm_riscv_gstage_gpa_size ((gpa_t)(1ULL << kvm_riscv_gstage_gpa_bits))
42
+
43
+
bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr,
44
+
pte_t **ptepp, u32 *ptep_level);
45
+
46
+
int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage,
47
+
struct kvm_mmu_memory_cache *pcache,
48
+
const struct kvm_gstage_mapping *map);
49
+
50
+
int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage,
51
+
struct kvm_mmu_memory_cache *pcache,
52
+
gpa_t gpa, phys_addr_t hpa, unsigned long page_size,
53
+
bool page_rdonly, bool page_exec,
54
+
struct kvm_gstage_mapping *out_map);
55
+
56
+
enum kvm_riscv_gstage_op {
57
+
GSTAGE_OP_NOP = 0, /* Nothing */
58
+
GSTAGE_OP_CLEAR, /* Clear/Unmap */
59
+
GSTAGE_OP_WP, /* Write-protect */
60
+
};
61
+
62
+
void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr,
63
+
pte_t *ptep, u32 ptep_level, enum kvm_riscv_gstage_op op);
64
+
65
+
void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage,
66
+
gpa_t start, gpa_t size, bool may_block);
67
+
68
+
void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end);
69
+
70
+
void kvm_riscv_gstage_mode_detect(void);
71
+
72
+
#endif
+5
-100
arch/riscv/include/asm/kvm_host.h
+5
-100
arch/riscv/include/asm/kvm_host.h
···
16
16
#include <asm/hwcap.h>
17
17
#include <asm/kvm_aia.h>
18
18
#include <asm/ptrace.h>
19
+
#include <asm/kvm_tlb.h>
20
+
#include <asm/kvm_vmid.h>
19
21
#include <asm/kvm_vcpu_fp.h>
20
22
#include <asm/kvm_vcpu_insn.h>
21
23
#include <asm/kvm_vcpu_sbi.h>
···
38
36
#define KVM_REQ_UPDATE_HGATP KVM_ARCH_REQ(2)
39
37
#define KVM_REQ_FENCE_I \
40
38
KVM_ARCH_REQ_FLAGS(3, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
41
-
#define KVM_REQ_HFENCE_GVMA_VMID_ALL KVM_REQ_TLB_FLUSH
42
39
#define KVM_REQ_HFENCE_VVMA_ALL \
43
40
KVM_ARCH_REQ_FLAGS(4, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
44
41
#define KVM_REQ_HFENCE \
45
42
KVM_ARCH_REQ_FLAGS(5, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
46
43
#define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(6)
47
44
45
+
#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE
46
+
48
47
#define KVM_HEDELEG_DEFAULT (BIT(EXC_INST_MISALIGNED) | \
48
+
BIT(EXC_INST_ILLEGAL) | \
49
49
BIT(EXC_BREAKPOINT) | \
50
50
BIT(EXC_SYSCALL) | \
51
51
BIT(EXC_INST_PAGE_FAULT) | \
···
57
53
#define KVM_HIDELEG_DEFAULT (BIT(IRQ_VS_SOFT) | \
58
54
BIT(IRQ_VS_TIMER) | \
59
55
BIT(IRQ_VS_EXT))
60
-
61
-
enum kvm_riscv_hfence_type {
62
-
KVM_RISCV_HFENCE_UNKNOWN = 0,
63
-
KVM_RISCV_HFENCE_GVMA_VMID_GPA,
64
-
KVM_RISCV_HFENCE_VVMA_ASID_GVA,
65
-
KVM_RISCV_HFENCE_VVMA_ASID_ALL,
66
-
KVM_RISCV_HFENCE_VVMA_GVA,
67
-
};
68
-
69
-
struct kvm_riscv_hfence {
70
-
enum kvm_riscv_hfence_type type;
71
-
unsigned long asid;
72
-
unsigned long order;
73
-
gpa_t addr;
74
-
gpa_t size;
75
-
};
76
-
77
-
#define KVM_RISCV_VCPU_MAX_HFENCE 64
78
56
79
57
struct kvm_vm_stat {
80
58
struct kvm_vm_stat_generic generic;
···
81
95
};
82
96
83
97
struct kvm_arch_memory_slot {
84
-
};
85
-
86
-
struct kvm_vmid {
87
-
/*
88
-
* Writes to vmid_version and vmid happen with vmid_lock held
89
-
* whereas reads happen without any lock held.
90
-
*/
91
-
unsigned long vmid_version;
92
-
unsigned long vmid;
93
98
};
94
99
95
100
struct kvm_arch {
···
286
309
static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
287
310
static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
288
311
289
-
#define KVM_RISCV_GSTAGE_TLB_MIN_ORDER 12
290
-
291
-
void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,
292
-
gpa_t gpa, gpa_t gpsz,
293
-
unsigned long order);
294
-
void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid);
295
-
void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz,
296
-
unsigned long order);
297
-
void kvm_riscv_local_hfence_gvma_all(void);
298
-
void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,
299
-
unsigned long asid,
300
-
unsigned long gva,
301
-
unsigned long gvsz,
302
-
unsigned long order);
303
-
void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid,
304
-
unsigned long asid);
305
-
void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,
306
-
unsigned long gva, unsigned long gvsz,
307
-
unsigned long order);
308
-
void kvm_riscv_local_hfence_vvma_all(unsigned long vmid);
309
-
310
-
void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu);
311
-
312
-
void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu);
313
-
void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu);
314
-
void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu);
315
-
void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu);
316
-
317
-
void kvm_riscv_fence_i(struct kvm *kvm,
318
-
unsigned long hbase, unsigned long hmask);
319
-
void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
320
-
unsigned long hbase, unsigned long hmask,
321
-
gpa_t gpa, gpa_t gpsz,
322
-
unsigned long order);
323
-
void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
324
-
unsigned long hbase, unsigned long hmask);
325
-
void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
326
-
unsigned long hbase, unsigned long hmask,
327
-
unsigned long gva, unsigned long gvsz,
328
-
unsigned long order, unsigned long asid);
329
-
void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
330
-
unsigned long hbase, unsigned long hmask,
331
-
unsigned long asid);
332
-
void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
333
-
unsigned long hbase, unsigned long hmask,
334
-
unsigned long gva, unsigned long gvsz,
335
-
unsigned long order);
336
-
void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
337
-
unsigned long hbase, unsigned long hmask);
338
-
339
-
int kvm_riscv_gstage_ioremap(struct kvm *kvm, gpa_t gpa,
340
-
phys_addr_t hpa, unsigned long size,
341
-
bool writable, bool in_atomic);
342
-
void kvm_riscv_gstage_iounmap(struct kvm *kvm, gpa_t gpa,
343
-
unsigned long size);
344
-
int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu,
345
-
struct kvm_memory_slot *memslot,
346
-
gpa_t gpa, unsigned long hva, bool is_write);
347
-
int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm);
348
-
void kvm_riscv_gstage_free_pgd(struct kvm *kvm);
349
-
void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu);
350
-
void __init kvm_riscv_gstage_mode_detect(void);
351
-
unsigned long __init kvm_riscv_gstage_mode(void);
352
-
int kvm_riscv_gstage_gpa_bits(void);
353
-
354
-
void __init kvm_riscv_gstage_vmid_detect(void);
355
-
unsigned long kvm_riscv_gstage_vmid_bits(void);
356
-
int kvm_riscv_gstage_vmid_init(struct kvm *kvm);
357
-
bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid);
358
-
void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu);
359
-
360
312
int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines);
361
313
362
314
void __kvm_riscv_unpriv_trap(void);
···
321
415
void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu);
322
416
bool kvm_riscv_vcpu_stopped(struct kvm_vcpu *vcpu);
323
417
324
-
void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu);
325
418
void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu);
326
419
327
420
#endif /* __RISCV_KVM_HOST_H__ */
+21
arch/riscv/include/asm/kvm_mmu.h
+21
arch/riscv/include/asm/kvm_mmu.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* Copyright (c) 2025 Ventana Micro Systems Inc.
4
+
*/
5
+
6
+
#ifndef __RISCV_KVM_MMU_H_
7
+
#define __RISCV_KVM_MMU_H_
8
+
9
+
#include <asm/kvm_gstage.h>
10
+
11
+
int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,
12
+
unsigned long size, bool writable, bool in_atomic);
13
+
void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size);
14
+
int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
15
+
gpa_t gpa, unsigned long hva, bool is_write,
16
+
struct kvm_gstage_mapping *out_map);
17
+
int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm);
18
+
void kvm_riscv_mmu_free_pgd(struct kvm *kvm);
19
+
void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu);
20
+
21
+
#endif
+84
arch/riscv/include/asm/kvm_tlb.h
+84
arch/riscv/include/asm/kvm_tlb.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* Copyright (c) 2025 Ventana Micro Systems Inc.
4
+
*/
5
+
6
+
#ifndef __RISCV_KVM_TLB_H_
7
+
#define __RISCV_KVM_TLB_H_
8
+
9
+
#include <linux/kvm_types.h>
10
+
11
+
enum kvm_riscv_hfence_type {
12
+
KVM_RISCV_HFENCE_UNKNOWN = 0,
13
+
KVM_RISCV_HFENCE_GVMA_VMID_GPA,
14
+
KVM_RISCV_HFENCE_GVMA_VMID_ALL,
15
+
KVM_RISCV_HFENCE_VVMA_ASID_GVA,
16
+
KVM_RISCV_HFENCE_VVMA_ASID_ALL,
17
+
KVM_RISCV_HFENCE_VVMA_GVA,
18
+
KVM_RISCV_HFENCE_VVMA_ALL
19
+
};
20
+
21
+
struct kvm_riscv_hfence {
22
+
enum kvm_riscv_hfence_type type;
23
+
unsigned long asid;
24
+
unsigned long vmid;
25
+
unsigned long order;
26
+
gpa_t addr;
27
+
gpa_t size;
28
+
};
29
+
30
+
#define KVM_RISCV_VCPU_MAX_HFENCE 64
31
+
32
+
#define KVM_RISCV_GSTAGE_TLB_MIN_ORDER 12
33
+
34
+
void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,
35
+
gpa_t gpa, gpa_t gpsz,
36
+
unsigned long order);
37
+
void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid);
38
+
void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz,
39
+
unsigned long order);
40
+
void kvm_riscv_local_hfence_gvma_all(void);
41
+
void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,
42
+
unsigned long asid,
43
+
unsigned long gva,
44
+
unsigned long gvsz,
45
+
unsigned long order);
46
+
void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid,
47
+
unsigned long asid);
48
+
void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,
49
+
unsigned long gva, unsigned long gvsz,
50
+
unsigned long order);
51
+
void kvm_riscv_local_hfence_vvma_all(unsigned long vmid);
52
+
53
+
void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu);
54
+
55
+
void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu);
56
+
void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu);
57
+
void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu);
58
+
59
+
void kvm_riscv_fence_i(struct kvm *kvm,
60
+
unsigned long hbase, unsigned long hmask);
61
+
void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
62
+
unsigned long hbase, unsigned long hmask,
63
+
gpa_t gpa, gpa_t gpsz,
64
+
unsigned long order, unsigned long vmid);
65
+
void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
66
+
unsigned long hbase, unsigned long hmask,
67
+
unsigned long vmid);
68
+
void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
69
+
unsigned long hbase, unsigned long hmask,
70
+
unsigned long gva, unsigned long gvsz,
71
+
unsigned long order, unsigned long asid,
72
+
unsigned long vmid);
73
+
void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
74
+
unsigned long hbase, unsigned long hmask,
75
+
unsigned long asid, unsigned long vmid);
76
+
void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
77
+
unsigned long hbase, unsigned long hmask,
78
+
unsigned long gva, unsigned long gvsz,
79
+
unsigned long order, unsigned long vmid);
80
+
void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
81
+
unsigned long hbase, unsigned long hmask,
82
+
unsigned long vmid);
83
+
84
+
#endif
+12
arch/riscv/include/asm/kvm_vcpu_sbi.h
+12
arch/riscv/include/asm/kvm_vcpu_sbi.h
···
49
49
50
50
/* Extension specific probe function */
51
51
unsigned long (*probe)(struct kvm_vcpu *vcpu);
52
+
53
+
/*
54
+
* Init/deinit function called once during VCPU init/destroy. These
55
+
* might be use if the SBI extensions need to allocate or do specific
56
+
* init time only configuration.
57
+
*/
58
+
int (*init)(struct kvm_vcpu *vcpu);
59
+
void (*deinit)(struct kvm_vcpu *vcpu);
60
+
61
+
void (*reset)(struct kvm_vcpu *vcpu);
52
62
};
53
63
54
64
void kvm_riscv_vcpu_sbi_forward(struct kvm_vcpu *vcpu, struct kvm_run *run);
···
82
72
bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx);
83
73
int kvm_riscv_vcpu_sbi_ecall(struct kvm_vcpu *vcpu, struct kvm_run *run);
84
74
void kvm_riscv_vcpu_sbi_init(struct kvm_vcpu *vcpu);
75
+
void kvm_riscv_vcpu_sbi_deinit(struct kvm_vcpu *vcpu);
76
+
void kvm_riscv_vcpu_sbi_reset(struct kvm_vcpu *vcpu);
85
77
86
78
int kvm_riscv_vcpu_get_reg_sbi_sta(struct kvm_vcpu *vcpu, unsigned long reg_num,
87
79
unsigned long *reg_val);
+27
arch/riscv/include/asm/kvm_vmid.h
+27
arch/riscv/include/asm/kvm_vmid.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* Copyright (c) 2025 Ventana Micro Systems Inc.
4
+
*/
5
+
6
+
#ifndef __RISCV_KVM_VMID_H_
7
+
#define __RISCV_KVM_VMID_H_
8
+
9
+
#include <linux/kvm_types.h>
10
+
11
+
struct kvm_vmid {
12
+
/*
13
+
* Writes to vmid_version and vmid happen with vmid_lock held
14
+
* whereas reads happen without any lock held.
15
+
*/
16
+
unsigned long vmid_version;
17
+
unsigned long vmid;
18
+
};
19
+
20
+
void __init kvm_riscv_gstage_vmid_detect(void);
21
+
unsigned long kvm_riscv_gstage_vmid_bits(void);
22
+
int kvm_riscv_gstage_vmid_init(struct kvm *kvm);
23
+
bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid);
24
+
void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu);
25
+
void kvm_riscv_gstage_vmid_sanitize(struct kvm_vcpu *vcpu);
26
+
27
+
#endif
+1
arch/riscv/include/uapi/asm/kvm.h
+1
arch/riscv/include/uapi/asm/kvm.h
+1
arch/riscv/kvm/Kconfig
+1
arch/riscv/kvm/Kconfig
+1
arch/riscv/kvm/Makefile
+1
arch/riscv/kvm/Makefile
+2
-4
arch/riscv/kvm/aia_device.c
+2
-4
arch/riscv/kvm/aia_device.c
···
509
509
kvm_riscv_vcpu_aia_imsic_reset(vcpu);
510
510
}
511
511
512
-
int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu)
512
+
void kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu)
513
513
{
514
514
struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context;
515
515
516
516
if (!kvm_riscv_aia_available())
517
-
return 0;
517
+
return;
518
518
519
519
/*
520
520
* We don't do any memory allocations over here because these
···
526
526
/* Initialize default values in AIA vcpu context */
527
527
vaia->imsic_addr = KVM_RISCV_AIA_UNDEF_ADDR;
528
528
vaia->hart_index = vcpu->vcpu_idx;
529
-
530
-
return 0;
531
529
}
532
530
533
531
void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu)
+6
-6
arch/riscv/kvm/aia_imsic.c
+6
-6
arch/riscv/kvm/aia_imsic.c
···
16
16
#include <linux/swab.h>
17
17
#include <kvm/iodev.h>
18
18
#include <asm/csr.h>
19
+
#include <asm/kvm_mmu.h>
19
20
20
21
#define IMSIC_MAX_EIX (IMSIC_MAX_ID / BITS_PER_TYPE(u64))
21
22
···
746
745
*/
747
746
748
747
/* Purge the G-stage mapping */
749
-
kvm_riscv_gstage_iounmap(vcpu->kvm,
750
-
vcpu->arch.aia_context.imsic_addr,
751
-
IMSIC_MMIO_PAGE_SZ);
748
+
kvm_riscv_mmu_iounmap(vcpu->kvm, vcpu->arch.aia_context.imsic_addr,
749
+
IMSIC_MMIO_PAGE_SZ);
752
750
753
751
/* TODO: Purge the IOMMU mapping ??? */
754
752
···
830
830
imsic_vsfile_local_clear(new_vsfile_hgei, imsic->nr_hw_eix);
831
831
832
832
/* Update G-stage mapping for the new IMSIC VS-file */
833
-
ret = kvm_riscv_gstage_ioremap(kvm, vcpu->arch.aia_context.imsic_addr,
834
-
new_vsfile_pa, IMSIC_MMIO_PAGE_SZ,
835
-
true, true);
833
+
ret = kvm_riscv_mmu_ioremap(kvm, vcpu->arch.aia_context.imsic_addr,
834
+
new_vsfile_pa, IMSIC_MMIO_PAGE_SZ,
835
+
true, true);
836
836
if (ret)
837
837
goto fail_free_vsfile_hgei;
838
838
+338
arch/riscv/kvm/gstage.c
+338
arch/riscv/kvm/gstage.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* Copyright (C) 2019 Western Digital Corporation or its affiliates.
4
+
* Copyright (c) 2025 Ventana Micro Systems Inc.
5
+
*/
6
+
7
+
#include <linux/bitops.h>
8
+
#include <linux/errno.h>
9
+
#include <linux/kvm_host.h>
10
+
#include <linux/module.h>
11
+
#include <linux/pgtable.h>
12
+
#include <asm/kvm_gstage.h>
13
+
14
+
#ifdef CONFIG_64BIT
15
+
unsigned long kvm_riscv_gstage_mode __ro_after_init = HGATP_MODE_SV39X4;
16
+
unsigned long kvm_riscv_gstage_pgd_levels __ro_after_init = 3;
17
+
#else
18
+
unsigned long kvm_riscv_gstage_mode __ro_after_init = HGATP_MODE_SV32X4;
19
+
unsigned long kvm_riscv_gstage_pgd_levels __ro_after_init = 2;
20
+
#endif
21
+
22
+
#define gstage_pte_leaf(__ptep) \
23
+
(pte_val(*(__ptep)) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC))
24
+
25
+
static inline unsigned long gstage_pte_index(gpa_t addr, u32 level)
26
+
{
27
+
unsigned long mask;
28
+
unsigned long shift = HGATP_PAGE_SHIFT + (kvm_riscv_gstage_index_bits * level);
29
+
30
+
if (level == (kvm_riscv_gstage_pgd_levels - 1))
31
+
mask = (PTRS_PER_PTE * (1UL << kvm_riscv_gstage_pgd_xbits)) - 1;
32
+
else
33
+
mask = PTRS_PER_PTE - 1;
34
+
35
+
return (addr >> shift) & mask;
36
+
}
37
+
38
+
static inline unsigned long gstage_pte_page_vaddr(pte_t pte)
39
+
{
40
+
return (unsigned long)pfn_to_virt(__page_val_to_pfn(pte_val(pte)));
41
+
}
42
+
43
+
static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level)
44
+
{
45
+
u32 i;
46
+
unsigned long psz = 1UL << 12;
47
+
48
+
for (i = 0; i < kvm_riscv_gstage_pgd_levels; i++) {
49
+
if (page_size == (psz << (i * kvm_riscv_gstage_index_bits))) {
50
+
*out_level = i;
51
+
return 0;
52
+
}
53
+
}
54
+
55
+
return -EINVAL;
56
+
}
57
+
58
+
static int gstage_level_to_page_order(u32 level, unsigned long *out_pgorder)
59
+
{
60
+
if (kvm_riscv_gstage_pgd_levels < level)
61
+
return -EINVAL;
62
+
63
+
*out_pgorder = 12 + (level * kvm_riscv_gstage_index_bits);
64
+
return 0;
65
+
}
66
+
67
+
static int gstage_level_to_page_size(u32 level, unsigned long *out_pgsize)
68
+
{
69
+
int rc;
70
+
unsigned long page_order = PAGE_SHIFT;
71
+
72
+
rc = gstage_level_to_page_order(level, &page_order);
73
+
if (rc)
74
+
return rc;
75
+
76
+
*out_pgsize = BIT(page_order);
77
+
return 0;
78
+
}
79
+
80
+
bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr,
81
+
pte_t **ptepp, u32 *ptep_level)
82
+
{
83
+
pte_t *ptep;
84
+
u32 current_level = kvm_riscv_gstage_pgd_levels - 1;
85
+
86
+
*ptep_level = current_level;
87
+
ptep = (pte_t *)gstage->pgd;
88
+
ptep = &ptep[gstage_pte_index(addr, current_level)];
89
+
while (ptep && pte_val(ptep_get(ptep))) {
90
+
if (gstage_pte_leaf(ptep)) {
91
+
*ptep_level = current_level;
92
+
*ptepp = ptep;
93
+
return true;
94
+
}
95
+
96
+
if (current_level) {
97
+
current_level--;
98
+
*ptep_level = current_level;
99
+
ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
100
+
ptep = &ptep[gstage_pte_index(addr, current_level)];
101
+
} else {
102
+
ptep = NULL;
103
+
}
104
+
}
105
+
106
+
return false;
107
+
}
108
+
109
+
static void gstage_tlb_flush(struct kvm_gstage *gstage, u32 level, gpa_t addr)
110
+
{
111
+
unsigned long order = PAGE_SHIFT;
112
+
113
+
if (gstage_level_to_page_order(level, &order))
114
+
return;
115
+
addr &= ~(BIT(order) - 1);
116
+
117
+
if (gstage->flags & KVM_GSTAGE_FLAGS_LOCAL)
118
+
kvm_riscv_local_hfence_gvma_vmid_gpa(gstage->vmid, addr, BIT(order), order);
119
+
else
120
+
kvm_riscv_hfence_gvma_vmid_gpa(gstage->kvm, -1UL, 0, addr, BIT(order), order,
121
+
gstage->vmid);
122
+
}
123
+
124
+
int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage,
125
+
struct kvm_mmu_memory_cache *pcache,
126
+
const struct kvm_gstage_mapping *map)
127
+
{
128
+
u32 current_level = kvm_riscv_gstage_pgd_levels - 1;
129
+
pte_t *next_ptep = (pte_t *)gstage->pgd;
130
+
pte_t *ptep = &next_ptep[gstage_pte_index(map->addr, current_level)];
131
+
132
+
if (current_level < map->level)
133
+
return -EINVAL;
134
+
135
+
while (current_level != map->level) {
136
+
if (gstage_pte_leaf(ptep))
137
+
return -EEXIST;
138
+
139
+
if (!pte_val(ptep_get(ptep))) {
140
+
if (!pcache)
141
+
return -ENOMEM;
142
+
next_ptep = kvm_mmu_memory_cache_alloc(pcache);
143
+
if (!next_ptep)
144
+
return -ENOMEM;
145
+
set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)),
146
+
__pgprot(_PAGE_TABLE)));
147
+
} else {
148
+
if (gstage_pte_leaf(ptep))
149
+
return -EEXIST;
150
+
next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
151
+
}
152
+
153
+
current_level--;
154
+
ptep = &next_ptep[gstage_pte_index(map->addr, current_level)];
155
+
}
156
+
157
+
if (pte_val(*ptep) != pte_val(map->pte)) {
158
+
set_pte(ptep, map->pte);
159
+
if (gstage_pte_leaf(ptep))
160
+
gstage_tlb_flush(gstage, current_level, map->addr);
161
+
}
162
+
163
+
return 0;
164
+
}
165
+
166
+
int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage,
167
+
struct kvm_mmu_memory_cache *pcache,
168
+
gpa_t gpa, phys_addr_t hpa, unsigned long page_size,
169
+
bool page_rdonly, bool page_exec,
170
+
struct kvm_gstage_mapping *out_map)
171
+
{
172
+
pgprot_t prot;
173
+
int ret;
174
+
175
+
out_map->addr = gpa;
176
+
out_map->level = 0;
177
+
178
+
ret = gstage_page_size_to_level(page_size, &out_map->level);
179
+
if (ret)
180
+
return ret;
181
+
182
+
/*
183
+
* A RISC-V implementation can choose to either:
184
+
* 1) Update 'A' and 'D' PTE bits in hardware
185
+
* 2) Generate page fault when 'A' and/or 'D' bits are not set
186
+
* PTE so that software can update these bits.
187
+
*
188
+
* We support both options mentioned above. To achieve this, we
189
+
* always set 'A' and 'D' PTE bits at time of creating G-stage
190
+
* mapping. To support KVM dirty page logging with both options
191
+
* mentioned above, we will write-protect G-stage PTEs to track
192
+
* dirty pages.
193
+
*/
194
+
195
+
if (page_exec) {
196
+
if (page_rdonly)
197
+
prot = PAGE_READ_EXEC;
198
+
else
199
+
prot = PAGE_WRITE_EXEC;
200
+
} else {
201
+
if (page_rdonly)
202
+
prot = PAGE_READ;
203
+
else
204
+
prot = PAGE_WRITE;
205
+
}
206
+
out_map->pte = pfn_pte(PFN_DOWN(hpa), prot);
207
+
out_map->pte = pte_mkdirty(out_map->pte);
208
+
209
+
return kvm_riscv_gstage_set_pte(gstage, pcache, out_map);
210
+
}
211
+
212
+
void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr,
213
+
pte_t *ptep, u32 ptep_level, enum kvm_riscv_gstage_op op)
214
+
{
215
+
int i, ret;
216
+
pte_t old_pte, *next_ptep;
217
+
u32 next_ptep_level;
218
+
unsigned long next_page_size, page_size;
219
+
220
+
ret = gstage_level_to_page_size(ptep_level, &page_size);
221
+
if (ret)
222
+
return;
223
+
224
+
WARN_ON(addr & (page_size - 1));
225
+
226
+
if (!pte_val(ptep_get(ptep)))
227
+
return;
228
+
229
+
if (ptep_level && !gstage_pte_leaf(ptep)) {
230
+
next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
231
+
next_ptep_level = ptep_level - 1;
232
+
ret = gstage_level_to_page_size(next_ptep_level, &next_page_size);
233
+
if (ret)
234
+
return;
235
+
236
+
if (op == GSTAGE_OP_CLEAR)
237
+
set_pte(ptep, __pte(0));
238
+
for (i = 0; i < PTRS_PER_PTE; i++)
239
+
kvm_riscv_gstage_op_pte(gstage, addr + i * next_page_size,
240
+
&next_ptep[i], next_ptep_level, op);
241
+
if (op == GSTAGE_OP_CLEAR)
242
+
put_page(virt_to_page(next_ptep));
243
+
} else {
244
+
old_pte = *ptep;
245
+
if (op == GSTAGE_OP_CLEAR)
246
+
set_pte(ptep, __pte(0));
247
+
else if (op == GSTAGE_OP_WP)
248
+
set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE));
249
+
if (pte_val(*ptep) != pte_val(old_pte))
250
+
gstage_tlb_flush(gstage, ptep_level, addr);
251
+
}
252
+
}
253
+
254
+
void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage,
255
+
gpa_t start, gpa_t size, bool may_block)
256
+
{
257
+
int ret;
258
+
pte_t *ptep;
259
+
u32 ptep_level;
260
+
bool found_leaf;
261
+
unsigned long page_size;
262
+
gpa_t addr = start, end = start + size;
263
+
264
+
while (addr < end) {
265
+
found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level);
266
+
ret = gstage_level_to_page_size(ptep_level, &page_size);
267
+
if (ret)
268
+
break;
269
+
270
+
if (!found_leaf)
271
+
goto next;
272
+
273
+
if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))
274
+
kvm_riscv_gstage_op_pte(gstage, addr, ptep,
275
+
ptep_level, GSTAGE_OP_CLEAR);
276
+
277
+
next:
278
+
addr += page_size;
279
+
280
+
/*
281
+
* If the range is too large, release the kvm->mmu_lock
282
+
* to prevent starvation and lockup detector warnings.
283
+
*/
284
+
if (!(gstage->flags & KVM_GSTAGE_FLAGS_LOCAL) && may_block && addr < end)
285
+
cond_resched_lock(&gstage->kvm->mmu_lock);
286
+
}
287
+
}
288
+
289
+
void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end)
290
+
{
291
+
int ret;
292
+
pte_t *ptep;
293
+
u32 ptep_level;
294
+
bool found_leaf;
295
+
gpa_t addr = start;
296
+
unsigned long page_size;
297
+
298
+
while (addr < end) {
299
+
found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level);
300
+
ret = gstage_level_to_page_size(ptep_level, &page_size);
301
+
if (ret)
302
+
break;
303
+
304
+
if (!found_leaf)
305
+
goto next;
306
+
307
+
if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))
308
+
kvm_riscv_gstage_op_pte(gstage, addr, ptep,
309
+
ptep_level, GSTAGE_OP_WP);
310
+
311
+
next:
312
+
addr += page_size;
313
+
}
314
+
}
315
+
316
+
void __init kvm_riscv_gstage_mode_detect(void)
317
+
{
318
+
#ifdef CONFIG_64BIT
319
+
/* Try Sv57x4 G-stage mode */
320
+
csr_write(CSR_HGATP, HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT);
321
+
if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV57X4) {
322
+
kvm_riscv_gstage_mode = HGATP_MODE_SV57X4;
323
+
kvm_riscv_gstage_pgd_levels = 5;
324
+
goto skip_sv48x4_test;
325
+
}
326
+
327
+
/* Try Sv48x4 G-stage mode */
328
+
csr_write(CSR_HGATP, HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT);
329
+
if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) {
330
+
kvm_riscv_gstage_mode = HGATP_MODE_SV48X4;
331
+
kvm_riscv_gstage_pgd_levels = 4;
332
+
}
333
+
skip_sv48x4_test:
334
+
335
+
csr_write(CSR_HGATP, 0);
336
+
kvm_riscv_local_hfence_gvma_all();
337
+
#endif
338
+
}
+2
-1
arch/riscv/kvm/main.c
+2
-1
arch/riscv/kvm/main.c
···
11
11
#include <linux/module.h>
12
12
#include <linux/kvm_host.h>
13
13
#include <asm/cpufeature.h>
14
+
#include <asm/kvm_mmu.h>
14
15
#include <asm/kvm_nacl.h>
15
16
#include <asm/sbi.h>
16
17
···
135
134
(rc) ? slist : "no features");
136
135
}
137
136
138
-
switch (kvm_riscv_gstage_mode()) {
137
+
switch (kvm_riscv_gstage_mode) {
139
138
case HGATP_MODE_SV32X4:
140
139
str = "Sv32x4";
141
140
break;
+112
-397
arch/riscv/kvm/mmu.c
+112
-397
arch/riscv/kvm/mmu.c
···
6
6
* Anup Patel <anup.patel@wdc.com>
7
7
*/
8
8
9
-
#include <linux/bitops.h>
10
9
#include <linux/errno.h>
11
-
#include <linux/err.h>
12
10
#include <linux/hugetlb.h>
13
11
#include <linux/module.h>
14
12
#include <linux/uaccess.h>
15
13
#include <linux/vmalloc.h>
16
14
#include <linux/kvm_host.h>
17
15
#include <linux/sched/signal.h>
16
+
#include <asm/kvm_mmu.h>
18
17
#include <asm/kvm_nacl.h>
19
-
#include <asm/page.h>
20
-
#include <asm/pgtable.h>
21
18
22
-
#ifdef CONFIG_64BIT
23
-
static unsigned long gstage_mode __ro_after_init = (HGATP_MODE_SV39X4 << HGATP_MODE_SHIFT);
24
-
static unsigned long gstage_pgd_levels __ro_after_init = 3;
25
-
#define gstage_index_bits 9
26
-
#else
27
-
static unsigned long gstage_mode __ro_after_init = (HGATP_MODE_SV32X4 << HGATP_MODE_SHIFT);
28
-
static unsigned long gstage_pgd_levels __ro_after_init = 2;
29
-
#define gstage_index_bits 10
30
-
#endif
31
-
32
-
#define gstage_pgd_xbits 2
33
-
#define gstage_pgd_size (1UL << (HGATP_PAGE_SHIFT + gstage_pgd_xbits))
34
-
#define gstage_gpa_bits (HGATP_PAGE_SHIFT + \
35
-
(gstage_pgd_levels * gstage_index_bits) + \
36
-
gstage_pgd_xbits)
37
-
#define gstage_gpa_size ((gpa_t)(1ULL << gstage_gpa_bits))
38
-
39
-
#define gstage_pte_leaf(__ptep) \
40
-
(pte_val(*(__ptep)) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC))
41
-
42
-
static inline unsigned long gstage_pte_index(gpa_t addr, u32 level)
43
-
{
44
-
unsigned long mask;
45
-
unsigned long shift = HGATP_PAGE_SHIFT + (gstage_index_bits * level);
46
-
47
-
if (level == (gstage_pgd_levels - 1))
48
-
mask = (PTRS_PER_PTE * (1UL << gstage_pgd_xbits)) - 1;
49
-
else
50
-
mask = PTRS_PER_PTE - 1;
51
-
52
-
return (addr >> shift) & mask;
53
-
}
54
-
55
-
static inline unsigned long gstage_pte_page_vaddr(pte_t pte)
56
-
{
57
-
return (unsigned long)pfn_to_virt(__page_val_to_pfn(pte_val(pte)));
58
-
}
59
-
60
-
static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level)
61
-
{
62
-
u32 i;
63
-
unsigned long psz = 1UL << 12;
64
-
65
-
for (i = 0; i < gstage_pgd_levels; i++) {
66
-
if (page_size == (psz << (i * gstage_index_bits))) {
67
-
*out_level = i;
68
-
return 0;
69
-
}
70
-
}
71
-
72
-
return -EINVAL;
73
-
}
74
-
75
-
static int gstage_level_to_page_order(u32 level, unsigned long *out_pgorder)
76
-
{
77
-
if (gstage_pgd_levels < level)
78
-
return -EINVAL;
79
-
80
-
*out_pgorder = 12 + (level * gstage_index_bits);
81
-
return 0;
82
-
}
83
-
84
-
static int gstage_level_to_page_size(u32 level, unsigned long *out_pgsize)
85
-
{
86
-
int rc;
87
-
unsigned long page_order = PAGE_SHIFT;
88
-
89
-
rc = gstage_level_to_page_order(level, &page_order);
90
-
if (rc)
91
-
return rc;
92
-
93
-
*out_pgsize = BIT(page_order);
94
-
return 0;
95
-
}
96
-
97
-
static bool gstage_get_leaf_entry(struct kvm *kvm, gpa_t addr,
98
-
pte_t **ptepp, u32 *ptep_level)
99
-
{
100
-
pte_t *ptep;
101
-
u32 current_level = gstage_pgd_levels - 1;
102
-
103
-
*ptep_level = current_level;
104
-
ptep = (pte_t *)kvm->arch.pgd;
105
-
ptep = &ptep[gstage_pte_index(addr, current_level)];
106
-
while (ptep && pte_val(ptep_get(ptep))) {
107
-
if (gstage_pte_leaf(ptep)) {
108
-
*ptep_level = current_level;
109
-
*ptepp = ptep;
110
-
return true;
111
-
}
112
-
113
-
if (current_level) {
114
-
current_level--;
115
-
*ptep_level = current_level;
116
-
ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
117
-
ptep = &ptep[gstage_pte_index(addr, current_level)];
118
-
} else {
119
-
ptep = NULL;
120
-
}
121
-
}
122
-
123
-
return false;
124
-
}
125
-
126
-
static void gstage_remote_tlb_flush(struct kvm *kvm, u32 level, gpa_t addr)
127
-
{
128
-
unsigned long order = PAGE_SHIFT;
129
-
130
-
if (gstage_level_to_page_order(level, &order))
131
-
return;
132
-
addr &= ~(BIT(order) - 1);
133
-
134
-
kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0, addr, BIT(order), order);
135
-
}
136
-
137
-
static int gstage_set_pte(struct kvm *kvm, u32 level,
138
-
struct kvm_mmu_memory_cache *pcache,
139
-
gpa_t addr, const pte_t *new_pte)
140
-
{
141
-
u32 current_level = gstage_pgd_levels - 1;
142
-
pte_t *next_ptep = (pte_t *)kvm->arch.pgd;
143
-
pte_t *ptep = &next_ptep[gstage_pte_index(addr, current_level)];
144
-
145
-
if (current_level < level)
146
-
return -EINVAL;
147
-
148
-
while (current_level != level) {
149
-
if (gstage_pte_leaf(ptep))
150
-
return -EEXIST;
151
-
152
-
if (!pte_val(ptep_get(ptep))) {
153
-
if (!pcache)
154
-
return -ENOMEM;
155
-
next_ptep = kvm_mmu_memory_cache_alloc(pcache);
156
-
if (!next_ptep)
157
-
return -ENOMEM;
158
-
set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)),
159
-
__pgprot(_PAGE_TABLE)));
160
-
} else {
161
-
if (gstage_pte_leaf(ptep))
162
-
return -EEXIST;
163
-
next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
164
-
}
165
-
166
-
current_level--;
167
-
ptep = &next_ptep[gstage_pte_index(addr, current_level)];
168
-
}
169
-
170
-
set_pte(ptep, *new_pte);
171
-
if (gstage_pte_leaf(ptep))
172
-
gstage_remote_tlb_flush(kvm, current_level, addr);
173
-
174
-
return 0;
175
-
}
176
-
177
-
static int gstage_map_page(struct kvm *kvm,
178
-
struct kvm_mmu_memory_cache *pcache,
179
-
gpa_t gpa, phys_addr_t hpa,
180
-
unsigned long page_size,
181
-
bool page_rdonly, bool page_exec)
182
-
{
183
-
int ret;
184
-
u32 level = 0;
185
-
pte_t new_pte;
186
-
pgprot_t prot;
187
-
188
-
ret = gstage_page_size_to_level(page_size, &level);
189
-
if (ret)
190
-
return ret;
191
-
192
-
/*
193
-
* A RISC-V implementation can choose to either:
194
-
* 1) Update 'A' and 'D' PTE bits in hardware
195
-
* 2) Generate page fault when 'A' and/or 'D' bits are not set
196
-
* PTE so that software can update these bits.
197
-
*
198
-
* We support both options mentioned above. To achieve this, we
199
-
* always set 'A' and 'D' PTE bits at time of creating G-stage
200
-
* mapping. To support KVM dirty page logging with both options
201
-
* mentioned above, we will write-protect G-stage PTEs to track
202
-
* dirty pages.
203
-
*/
204
-
205
-
if (page_exec) {
206
-
if (page_rdonly)
207
-
prot = PAGE_READ_EXEC;
208
-
else
209
-
prot = PAGE_WRITE_EXEC;
210
-
} else {
211
-
if (page_rdonly)
212
-
prot = PAGE_READ;
213
-
else
214
-
prot = PAGE_WRITE;
215
-
}
216
-
new_pte = pfn_pte(PFN_DOWN(hpa), prot);
217
-
new_pte = pte_mkdirty(new_pte);
218
-
219
-
return gstage_set_pte(kvm, level, pcache, gpa, &new_pte);
220
-
}
221
-
222
-
enum gstage_op {
223
-
GSTAGE_OP_NOP = 0, /* Nothing */
224
-
GSTAGE_OP_CLEAR, /* Clear/Unmap */
225
-
GSTAGE_OP_WP, /* Write-protect */
226
-
};
227
-
228
-
static void gstage_op_pte(struct kvm *kvm, gpa_t addr,
229
-
pte_t *ptep, u32 ptep_level, enum gstage_op op)
230
-
{
231
-
int i, ret;
232
-
pte_t *next_ptep;
233
-
u32 next_ptep_level;
234
-
unsigned long next_page_size, page_size;
235
-
236
-
ret = gstage_level_to_page_size(ptep_level, &page_size);
237
-
if (ret)
238
-
return;
239
-
240
-
BUG_ON(addr & (page_size - 1));
241
-
242
-
if (!pte_val(ptep_get(ptep)))
243
-
return;
244
-
245
-
if (ptep_level && !gstage_pte_leaf(ptep)) {
246
-
next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
247
-
next_ptep_level = ptep_level - 1;
248
-
ret = gstage_level_to_page_size(next_ptep_level,
249
-
&next_page_size);
250
-
if (ret)
251
-
return;
252
-
253
-
if (op == GSTAGE_OP_CLEAR)
254
-
set_pte(ptep, __pte(0));
255
-
for (i = 0; i < PTRS_PER_PTE; i++)
256
-
gstage_op_pte(kvm, addr + i * next_page_size,
257
-
&next_ptep[i], next_ptep_level, op);
258
-
if (op == GSTAGE_OP_CLEAR)
259
-
put_page(virt_to_page(next_ptep));
260
-
} else {
261
-
if (op == GSTAGE_OP_CLEAR)
262
-
set_pte(ptep, __pte(0));
263
-
else if (op == GSTAGE_OP_WP)
264
-
set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE));
265
-
gstage_remote_tlb_flush(kvm, ptep_level, addr);
266
-
}
267
-
}
268
-
269
-
static void gstage_unmap_range(struct kvm *kvm, gpa_t start,
270
-
gpa_t size, bool may_block)
271
-
{
272
-
int ret;
273
-
pte_t *ptep;
274
-
u32 ptep_level;
275
-
bool found_leaf;
276
-
unsigned long page_size;
277
-
gpa_t addr = start, end = start + size;
278
-
279
-
while (addr < end) {
280
-
found_leaf = gstage_get_leaf_entry(kvm, addr,
281
-
&ptep, &ptep_level);
282
-
ret = gstage_level_to_page_size(ptep_level, &page_size);
283
-
if (ret)
284
-
break;
285
-
286
-
if (!found_leaf)
287
-
goto next;
288
-
289
-
if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))
290
-
gstage_op_pte(kvm, addr, ptep,
291
-
ptep_level, GSTAGE_OP_CLEAR);
292
-
293
-
next:
294
-
addr += page_size;
295
-
296
-
/*
297
-
* If the range is too large, release the kvm->mmu_lock
298
-
* to prevent starvation and lockup detector warnings.
299
-
*/
300
-
if (may_block && addr < end)
301
-
cond_resched_lock(&kvm->mmu_lock);
302
-
}
303
-
}
304
-
305
-
static void gstage_wp_range(struct kvm *kvm, gpa_t start, gpa_t end)
306
-
{
307
-
int ret;
308
-
pte_t *ptep;
309
-
u32 ptep_level;
310
-
bool found_leaf;
311
-
gpa_t addr = start;
312
-
unsigned long page_size;
313
-
314
-
while (addr < end) {
315
-
found_leaf = gstage_get_leaf_entry(kvm, addr,
316
-
&ptep, &ptep_level);
317
-
ret = gstage_level_to_page_size(ptep_level, &page_size);
318
-
if (ret)
319
-
break;
320
-
321
-
if (!found_leaf)
322
-
goto next;
323
-
324
-
if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))
325
-
gstage_op_pte(kvm, addr, ptep,
326
-
ptep_level, GSTAGE_OP_WP);
327
-
328
-
next:
329
-
addr += page_size;
330
-
}
331
-
}
332
-
333
-
static void gstage_wp_memory_region(struct kvm *kvm, int slot)
19
+
static void mmu_wp_memory_region(struct kvm *kvm, int slot)
334
20
{
335
21
struct kvm_memslots *slots = kvm_memslots(kvm);
336
22
struct kvm_memory_slot *memslot = id_to_memslot(slots, slot);
337
23
phys_addr_t start = memslot->base_gfn << PAGE_SHIFT;
338
24
phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
25
+
struct kvm_gstage gstage;
26
+
27
+
gstage.kvm = kvm;
28
+
gstage.flags = 0;
29
+
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
30
+
gstage.pgd = kvm->arch.pgd;
339
31
340
32
spin_lock(&kvm->mmu_lock);
341
-
gstage_wp_range(kvm, start, end);
33
+
kvm_riscv_gstage_wp_range(&gstage, start, end);
342
34
spin_unlock(&kvm->mmu_lock);
343
-
kvm_flush_remote_tlbs(kvm);
35
+
kvm_flush_remote_tlbs_memslot(kvm, memslot);
344
36
}
345
37
346
-
int kvm_riscv_gstage_ioremap(struct kvm *kvm, gpa_t gpa,
347
-
phys_addr_t hpa, unsigned long size,
348
-
bool writable, bool in_atomic)
38
+
int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,
39
+
unsigned long size, bool writable, bool in_atomic)
349
40
{
350
-
pte_t pte;
351
41
int ret = 0;
352
42
unsigned long pfn;
353
43
phys_addr_t addr, end;
···
45
355
.gfp_custom = (in_atomic) ? GFP_ATOMIC | __GFP_ACCOUNT : 0,
46
356
.gfp_zero = __GFP_ZERO,
47
357
};
358
+
struct kvm_gstage_mapping map;
359
+
struct kvm_gstage gstage;
360
+
361
+
gstage.kvm = kvm;
362
+
gstage.flags = 0;
363
+
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
364
+
gstage.pgd = kvm->arch.pgd;
48
365
49
366
end = (gpa + size + PAGE_SIZE - 1) & PAGE_MASK;
50
367
pfn = __phys_to_pfn(hpa);
51
368
52
369
for (addr = gpa; addr < end; addr += PAGE_SIZE) {
53
-
pte = pfn_pte(pfn, PAGE_KERNEL_IO);
370
+
map.addr = addr;
371
+
map.pte = pfn_pte(pfn, PAGE_KERNEL_IO);
372
+
map.level = 0;
54
373
55
374
if (!writable)
56
-
pte = pte_wrprotect(pte);
375
+
map.pte = pte_wrprotect(map.pte);
57
376
58
-
ret = kvm_mmu_topup_memory_cache(&pcache, gstage_pgd_levels);
377
+
ret = kvm_mmu_topup_memory_cache(&pcache, kvm_riscv_gstage_pgd_levels);
59
378
if (ret)
60
379
goto out;
61
380
62
381
spin_lock(&kvm->mmu_lock);
63
-
ret = gstage_set_pte(kvm, 0, &pcache, addr, &pte);
382
+
ret = kvm_riscv_gstage_set_pte(&gstage, &pcache, &map);
64
383
spin_unlock(&kvm->mmu_lock);
65
384
if (ret)
66
385
goto out;
···
82
383
return ret;
83
384
}
84
385
85
-
void kvm_riscv_gstage_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size)
386
+
void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size)
86
387
{
388
+
struct kvm_gstage gstage;
389
+
390
+
gstage.kvm = kvm;
391
+
gstage.flags = 0;
392
+
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
393
+
gstage.pgd = kvm->arch.pgd;
394
+
87
395
spin_lock(&kvm->mmu_lock);
88
-
gstage_unmap_range(kvm, gpa, size, false);
396
+
kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false);
89
397
spin_unlock(&kvm->mmu_lock);
90
398
}
91
399
···
104
398
phys_addr_t base_gfn = slot->base_gfn + gfn_offset;
105
399
phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT;
106
400
phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
401
+
struct kvm_gstage gstage;
107
402
108
-
gstage_wp_range(kvm, start, end);
403
+
gstage.kvm = kvm;
404
+
gstage.flags = 0;
405
+
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
406
+
gstage.pgd = kvm->arch.pgd;
407
+
408
+
kvm_riscv_gstage_wp_range(&gstage, start, end);
109
409
}
110
410
111
411
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
···
128
416
129
417
void kvm_arch_flush_shadow_all(struct kvm *kvm)
130
418
{
131
-
kvm_riscv_gstage_free_pgd(kvm);
419
+
kvm_riscv_mmu_free_pgd(kvm);
132
420
}
133
421
134
422
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
···
136
424
{
137
425
gpa_t gpa = slot->base_gfn << PAGE_SHIFT;
138
426
phys_addr_t size = slot->npages << PAGE_SHIFT;
427
+
struct kvm_gstage gstage;
428
+
429
+
gstage.kvm = kvm;
430
+
gstage.flags = 0;
431
+
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
432
+
gstage.pgd = kvm->arch.pgd;
139
433
140
434
spin_lock(&kvm->mmu_lock);
141
-
gstage_unmap_range(kvm, gpa, size, false);
435
+
kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false);
142
436
spin_unlock(&kvm->mmu_lock);
143
437
}
144
438
···
159
441
* the memory slot is write protected.
160
442
*/
161
443
if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES)
162
-
gstage_wp_memory_region(kvm, new->id);
444
+
mmu_wp_memory_region(kvm, new->id);
163
445
}
164
446
165
447
int kvm_arch_prepare_memory_region(struct kvm *kvm,
···
181
463
* space addressable by the KVM guest GPA space.
182
464
*/
183
465
if ((new->base_gfn + new->npages) >=
184
-
(gstage_gpa_size >> PAGE_SHIFT))
466
+
(kvm_riscv_gstage_gpa_size >> PAGE_SHIFT))
185
467
return -EFAULT;
186
468
187
469
hva = new->userspace_addr;
···
205
487
* +--------------------------------------------+
206
488
*/
207
489
do {
208
-
struct vm_area_struct *vma = find_vma(current->mm, hva);
490
+
struct vm_area_struct *vma;
209
491
hva_t vm_start, vm_end;
210
492
211
-
if (!vma || vma->vm_start >= reg_end)
493
+
vma = find_vma_intersection(current->mm, hva, reg_end);
494
+
if (!vma)
212
495
break;
213
496
214
497
/*
···
238
519
goto out;
239
520
}
240
521
241
-
ret = kvm_riscv_gstage_ioremap(kvm, gpa, pa,
242
-
vm_end - vm_start,
243
-
writable, false);
522
+
ret = kvm_riscv_mmu_ioremap(kvm, gpa, pa, vm_end - vm_start,
523
+
writable, false);
244
524
if (ret)
245
525
break;
246
526
}
···
250
532
goto out;
251
533
252
534
if (ret)
253
-
kvm_riscv_gstage_iounmap(kvm, base_gpa, size);
535
+
kvm_riscv_mmu_iounmap(kvm, base_gpa, size);
254
536
255
537
out:
256
538
mmap_read_unlock(current->mm);
···
259
541
260
542
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
261
543
{
544
+
struct kvm_gstage gstage;
545
+
262
546
if (!kvm->arch.pgd)
263
547
return false;
264
548
265
-
gstage_unmap_range(kvm, range->start << PAGE_SHIFT,
266
-
(range->end - range->start) << PAGE_SHIFT,
267
-
range->may_block);
549
+
gstage.kvm = kvm;
550
+
gstage.flags = 0;
551
+
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
552
+
gstage.pgd = kvm->arch.pgd;
553
+
kvm_riscv_gstage_unmap_range(&gstage, range->start << PAGE_SHIFT,
554
+
(range->end - range->start) << PAGE_SHIFT,
555
+
range->may_block);
268
556
return false;
269
557
}
270
558
···
279
555
pte_t *ptep;
280
556
u32 ptep_level = 0;
281
557
u64 size = (range->end - range->start) << PAGE_SHIFT;
558
+
struct kvm_gstage gstage;
282
559
283
560
if (!kvm->arch.pgd)
284
561
return false;
285
562
286
563
WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE);
287
564
288
-
if (!gstage_get_leaf_entry(kvm, range->start << PAGE_SHIFT,
289
-
&ptep, &ptep_level))
565
+
gstage.kvm = kvm;
566
+
gstage.flags = 0;
567
+
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
568
+
gstage.pgd = kvm->arch.pgd;
569
+
if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT,
570
+
&ptep, &ptep_level))
290
571
return false;
291
572
292
573
return ptep_test_and_clear_young(NULL, 0, ptep);
···
302
573
pte_t *ptep;
303
574
u32 ptep_level = 0;
304
575
u64 size = (range->end - range->start) << PAGE_SHIFT;
576
+
struct kvm_gstage gstage;
305
577
306
578
if (!kvm->arch.pgd)
307
579
return false;
308
580
309
581
WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE);
310
582
311
-
if (!gstage_get_leaf_entry(kvm, range->start << PAGE_SHIFT,
312
-
&ptep, &ptep_level))
583
+
gstage.kvm = kvm;
584
+
gstage.flags = 0;
585
+
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
586
+
gstage.pgd = kvm->arch.pgd;
587
+
if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT,
588
+
&ptep, &ptep_level))
313
589
return false;
314
590
315
591
return pte_young(ptep_get(ptep));
316
592
}
317
593
318
-
int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu,
319
-
struct kvm_memory_slot *memslot,
320
-
gpa_t gpa, unsigned long hva, bool is_write)
594
+
int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
595
+
gpa_t gpa, unsigned long hva, bool is_write,
596
+
struct kvm_gstage_mapping *out_map)
321
597
{
322
598
int ret;
323
599
kvm_pfn_t hfn;
···
335
601
bool logging = (memslot->dirty_bitmap &&
336
602
!(memslot->flags & KVM_MEM_READONLY)) ? true : false;
337
603
unsigned long vma_pagesize, mmu_seq;
604
+
struct kvm_gstage gstage;
338
605
struct page *page;
339
606
607
+
gstage.kvm = kvm;
608
+
gstage.flags = 0;
609
+
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
610
+
gstage.pgd = kvm->arch.pgd;
611
+
612
+
/* Setup initial state of output mapping */
613
+
memset(out_map, 0, sizeof(*out_map));
614
+
340
615
/* We need minimum second+third level pages */
341
-
ret = kvm_mmu_topup_memory_cache(pcache, gstage_pgd_levels);
616
+
ret = kvm_mmu_topup_memory_cache(pcache, kvm_riscv_gstage_pgd_levels);
342
617
if (ret) {
343
618
kvm_err("Failed to topup G-stage cache\n");
344
619
return ret;
···
391
648
return -EFAULT;
392
649
}
393
650
394
-
hfn = kvm_faultin_pfn(vcpu, gfn, is_write, &writable, &page);
651
+
hfn = __kvm_faultin_pfn(memslot, gfn, is_write ? FOLL_WRITE : 0,
652
+
&writable, &page);
395
653
if (hfn == KVM_PFN_ERR_HWPOISON) {
396
654
send_sig_mceerr(BUS_MCEERR_AR, (void __user *)hva,
397
655
vma_pageshift, current);
···
414
670
goto out_unlock;
415
671
416
672
if (writable) {
417
-
mark_page_dirty(kvm, gfn);
418
-
ret = gstage_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT,
419
-
vma_pagesize, false, true);
673
+
mark_page_dirty_in_slot(kvm, memslot, gfn);
674
+
ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT,
675
+
vma_pagesize, false, true, out_map);
420
676
} else {
421
-
ret = gstage_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT,
422
-
vma_pagesize, true, true);
677
+
ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT,
678
+
vma_pagesize, true, true, out_map);
423
679
}
424
680
425
681
if (ret)
···
431
687
return ret;
432
688
}
433
689
434
-
int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm)
690
+
int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm)
435
691
{
436
692
struct page *pgd_page;
437
693
···
441
697
}
442
698
443
699
pgd_page = alloc_pages(GFP_KERNEL | __GFP_ZERO,
444
-
get_order(gstage_pgd_size));
700
+
get_order(kvm_riscv_gstage_pgd_size));
445
701
if (!pgd_page)
446
702
return -ENOMEM;
447
703
kvm->arch.pgd = page_to_virt(pgd_page);
···
450
706
return 0;
451
707
}
452
708
453
-
void kvm_riscv_gstage_free_pgd(struct kvm *kvm)
709
+
void kvm_riscv_mmu_free_pgd(struct kvm *kvm)
454
710
{
711
+
struct kvm_gstage gstage;
455
712
void *pgd = NULL;
456
713
457
714
spin_lock(&kvm->mmu_lock);
458
715
if (kvm->arch.pgd) {
459
-
gstage_unmap_range(kvm, 0UL, gstage_gpa_size, false);
716
+
gstage.kvm = kvm;
717
+
gstage.flags = 0;
718
+
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
719
+
gstage.pgd = kvm->arch.pgd;
720
+
kvm_riscv_gstage_unmap_range(&gstage, 0UL, kvm_riscv_gstage_gpa_size, false);
460
721
pgd = READ_ONCE(kvm->arch.pgd);
461
722
kvm->arch.pgd = NULL;
462
723
kvm->arch.pgd_phys = 0;
···
469
720
spin_unlock(&kvm->mmu_lock);
470
721
471
722
if (pgd)
472
-
free_pages((unsigned long)pgd, get_order(gstage_pgd_size));
723
+
free_pages((unsigned long)pgd, get_order(kvm_riscv_gstage_pgd_size));
473
724
}
474
725
475
-
void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu)
726
+
void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu)
476
727
{
477
-
unsigned long hgatp = gstage_mode;
728
+
unsigned long hgatp = kvm_riscv_gstage_mode << HGATP_MODE_SHIFT;
478
729
struct kvm_arch *k = &vcpu->kvm->arch;
479
730
480
731
hgatp |= (READ_ONCE(k->vmid.vmid) << HGATP_VMID_SHIFT) & HGATP_VMID;
···
484
735
485
736
if (!kvm_riscv_gstage_vmid_bits())
486
737
kvm_riscv_local_hfence_gvma_all();
487
-
}
488
-
489
-
void __init kvm_riscv_gstage_mode_detect(void)
490
-
{
491
-
#ifdef CONFIG_64BIT
492
-
/* Try Sv57x4 G-stage mode */
493
-
csr_write(CSR_HGATP, HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT);
494
-
if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV57X4) {
495
-
gstage_mode = (HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT);
496
-
gstage_pgd_levels = 5;
497
-
goto skip_sv48x4_test;
498
-
}
499
-
500
-
/* Try Sv48x4 G-stage mode */
501
-
csr_write(CSR_HGATP, HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT);
502
-
if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) {
503
-
gstage_mode = (HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT);
504
-
gstage_pgd_levels = 4;
505
-
}
506
-
skip_sv48x4_test:
507
-
508
-
csr_write(CSR_HGATP, 0);
509
-
kvm_riscv_local_hfence_gvma_all();
510
-
#endif
511
-
}
512
-
513
-
unsigned long __init kvm_riscv_gstage_mode(void)
514
-
{
515
-
return gstage_mode >> HGATP_MODE_SHIFT;
516
-
}
517
-
518
-
int kvm_riscv_gstage_gpa_bits(void)
519
-
{
520
-
return gstage_gpa_bits;
521
738
}
+59
-51
arch/riscv/kvm/tlb.c
+59
-51
arch/riscv/kvm/tlb.c
···
15
15
#include <asm/cpufeature.h>
16
16
#include <asm/insn-def.h>
17
17
#include <asm/kvm_nacl.h>
18
+
#include <asm/kvm_tlb.h>
19
+
#include <asm/kvm_vmid.h>
18
20
19
21
#define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
20
22
···
158
156
csr_write(CSR_HGATP, hgatp);
159
157
}
160
158
161
-
void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu)
162
-
{
163
-
unsigned long vmid;
164
-
165
-
if (!kvm_riscv_gstage_vmid_bits() ||
166
-
vcpu->arch.last_exit_cpu == vcpu->cpu)
167
-
return;
168
-
169
-
/*
170
-
* On RISC-V platforms with hardware VMID support, we share same
171
-
* VMID for all VCPUs of a particular Guest/VM. This means we might
172
-
* have stale G-stage TLB entries on the current Host CPU due to
173
-
* some other VCPU of the same Guest which ran previously on the
174
-
* current Host CPU.
175
-
*
176
-
* To cleanup stale TLB entries, we simply flush all G-stage TLB
177
-
* entries by VMID whenever underlying Host CPU changes for a VCPU.
178
-
*/
179
-
180
-
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
181
-
kvm_riscv_local_hfence_gvma_vmid_all(vmid);
182
-
}
183
-
184
159
void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu)
185
160
{
186
161
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_RCVD);
187
162
local_flush_icache_all();
188
163
}
189
164
190
-
void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu)
165
+
void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu)
191
166
{
192
167
struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
193
168
unsigned long vmid = READ_ONCE(v->vmid);
···
237
258
238
259
void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu)
239
260
{
240
-
unsigned long vmid;
241
261
struct kvm_riscv_hfence d = { 0 };
242
-
struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
243
262
244
263
while (vcpu_hfence_dequeue(vcpu, &d)) {
245
264
switch (d.type) {
246
265
case KVM_RISCV_HFENCE_UNKNOWN:
247
266
break;
248
267
case KVM_RISCV_HFENCE_GVMA_VMID_GPA:
249
-
vmid = READ_ONCE(v->vmid);
250
268
if (kvm_riscv_nacl_available())
251
-
nacl_hfence_gvma_vmid(nacl_shmem(), vmid,
269
+
nacl_hfence_gvma_vmid(nacl_shmem(), d.vmid,
252
270
d.addr, d.size, d.order);
253
271
else
254
-
kvm_riscv_local_hfence_gvma_vmid_gpa(vmid, d.addr,
272
+
kvm_riscv_local_hfence_gvma_vmid_gpa(d.vmid, d.addr,
255
273
d.size, d.order);
274
+
break;
275
+
case KVM_RISCV_HFENCE_GVMA_VMID_ALL:
276
+
if (kvm_riscv_nacl_available())
277
+
nacl_hfence_gvma_vmid_all(nacl_shmem(), d.vmid);
278
+
else
279
+
kvm_riscv_local_hfence_gvma_vmid_all(d.vmid);
256
280
break;
257
281
case KVM_RISCV_HFENCE_VVMA_ASID_GVA:
258
282
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD);
259
-
vmid = READ_ONCE(v->vmid);
260
283
if (kvm_riscv_nacl_available())
261
-
nacl_hfence_vvma_asid(nacl_shmem(), vmid, d.asid,
284
+
nacl_hfence_vvma_asid(nacl_shmem(), d.vmid, d.asid,
262
285
d.addr, d.size, d.order);
263
286
else
264
-
kvm_riscv_local_hfence_vvma_asid_gva(vmid, d.asid, d.addr,
287
+
kvm_riscv_local_hfence_vvma_asid_gva(d.vmid, d.asid, d.addr,
265
288
d.size, d.order);
266
289
break;
267
290
case KVM_RISCV_HFENCE_VVMA_ASID_ALL:
268
291
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD);
269
-
vmid = READ_ONCE(v->vmid);
270
292
if (kvm_riscv_nacl_available())
271
-
nacl_hfence_vvma_asid_all(nacl_shmem(), vmid, d.asid);
293
+
nacl_hfence_vvma_asid_all(nacl_shmem(), d.vmid, d.asid);
272
294
else
273
-
kvm_riscv_local_hfence_vvma_asid_all(vmid, d.asid);
295
+
kvm_riscv_local_hfence_vvma_asid_all(d.vmid, d.asid);
274
296
break;
275
297
case KVM_RISCV_HFENCE_VVMA_GVA:
276
298
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD);
277
-
vmid = READ_ONCE(v->vmid);
278
299
if (kvm_riscv_nacl_available())
279
-
nacl_hfence_vvma(nacl_shmem(), vmid,
300
+
nacl_hfence_vvma(nacl_shmem(), d.vmid,
280
301
d.addr, d.size, d.order);
281
302
else
282
-
kvm_riscv_local_hfence_vvma_gva(vmid, d.addr,
303
+
kvm_riscv_local_hfence_vvma_gva(d.vmid, d.addr,
283
304
d.size, d.order);
305
+
break;
306
+
case KVM_RISCV_HFENCE_VVMA_ALL:
307
+
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD);
308
+
if (kvm_riscv_nacl_available())
309
+
nacl_hfence_vvma_all(nacl_shmem(), d.vmid);
310
+
else
311
+
kvm_riscv_local_hfence_vvma_all(d.vmid);
284
312
break;
285
313
default:
286
314
break;
···
341
355
void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
342
356
unsigned long hbase, unsigned long hmask,
343
357
gpa_t gpa, gpa_t gpsz,
344
-
unsigned long order)
358
+
unsigned long order, unsigned long vmid)
345
359
{
346
360
struct kvm_riscv_hfence data;
347
361
348
362
data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA;
349
363
data.asid = 0;
364
+
data.vmid = vmid;
350
365
data.addr = gpa;
351
366
data.size = gpsz;
352
367
data.order = order;
353
368
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
354
-
KVM_REQ_HFENCE_GVMA_VMID_ALL, &data);
369
+
KVM_REQ_TLB_FLUSH, &data);
355
370
}
356
371
357
372
void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
358
-
unsigned long hbase, unsigned long hmask)
373
+
unsigned long hbase, unsigned long hmask,
374
+
unsigned long vmid)
359
375
{
360
-
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_GVMA_VMID_ALL,
361
-
KVM_REQ_HFENCE_GVMA_VMID_ALL, NULL);
376
+
struct kvm_riscv_hfence data = {0};
377
+
378
+
data.type = KVM_RISCV_HFENCE_GVMA_VMID_ALL;
379
+
data.vmid = vmid;
380
+
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
381
+
KVM_REQ_TLB_FLUSH, &data);
362
382
}
363
383
364
384
void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
365
385
unsigned long hbase, unsigned long hmask,
366
386
unsigned long gva, unsigned long gvsz,
367
-
unsigned long order, unsigned long asid)
387
+
unsigned long order, unsigned long asid,
388
+
unsigned long vmid)
368
389
{
369
390
struct kvm_riscv_hfence data;
370
391
371
392
data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA;
372
393
data.asid = asid;
394
+
data.vmid = vmid;
373
395
data.addr = gva;
374
396
data.size = gvsz;
375
397
data.order = order;
···
387
393
388
394
void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
389
395
unsigned long hbase, unsigned long hmask,
390
-
unsigned long asid)
396
+
unsigned long asid, unsigned long vmid)
391
397
{
392
-
struct kvm_riscv_hfence data;
398
+
struct kvm_riscv_hfence data = {0};
393
399
394
400
data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL;
395
401
data.asid = asid;
396
-
data.addr = data.size = data.order = 0;
402
+
data.vmid = vmid;
397
403
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
398
404
KVM_REQ_HFENCE_VVMA_ALL, &data);
399
405
}
···
401
407
void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
402
408
unsigned long hbase, unsigned long hmask,
403
409
unsigned long gva, unsigned long gvsz,
404
-
unsigned long order)
410
+
unsigned long order, unsigned long vmid)
405
411
{
406
412
struct kvm_riscv_hfence data;
407
413
408
414
data.type = KVM_RISCV_HFENCE_VVMA_GVA;
409
415
data.asid = 0;
416
+
data.vmid = vmid;
410
417
data.addr = gva;
411
418
data.size = gvsz;
412
419
data.order = order;
···
416
421
}
417
422
418
423
void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
419
-
unsigned long hbase, unsigned long hmask)
424
+
unsigned long hbase, unsigned long hmask,
425
+
unsigned long vmid)
420
426
{
421
-
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_VVMA_ALL,
422
-
KVM_REQ_HFENCE_VVMA_ALL, NULL);
427
+
struct kvm_riscv_hfence data = {0};
428
+
429
+
data.type = KVM_RISCV_HFENCE_VVMA_ALL;
430
+
data.vmid = vmid;
431
+
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
432
+
KVM_REQ_HFENCE_VVMA_ALL, &data);
433
+
}
434
+
435
+
int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages)
436
+
{
437
+
kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0,
438
+
gfn << PAGE_SHIFT, nr_pages << PAGE_SHIFT,
439
+
PAGE_SHIFT, READ_ONCE(kvm->arch.vmid.vmid));
440
+
return 0;
423
441
}
+30
-18
arch/riscv/kvm/vcpu.c
+30
-18
arch/riscv/kvm/vcpu.c
···
18
18
#include <linux/fs.h>
19
19
#include <linux/kvm_host.h>
20
20
#include <asm/cacheflush.h>
21
+
#include <asm/kvm_mmu.h>
21
22
#include <asm/kvm_nacl.h>
22
23
#include <asm/kvm_vcpu_vector.h>
23
24
···
112
111
vcpu->arch.hfence_tail = 0;
113
112
memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue));
114
113
115
-
kvm_riscv_vcpu_sbi_sta_reset(vcpu);
114
+
kvm_riscv_vcpu_sbi_reset(vcpu);
116
115
117
116
/* Reset the guest CSRs for hotplug usecase */
118
117
if (loaded)
···
149
148
150
149
spin_lock_init(&vcpu->arch.reset_state.lock);
151
150
152
-
if (kvm_riscv_vcpu_alloc_vector_context(vcpu))
153
-
return -ENOMEM;
151
+
rc = kvm_riscv_vcpu_alloc_vector_context(vcpu);
152
+
if (rc)
153
+
return rc;
154
154
155
155
/* Setup VCPU timer */
156
156
kvm_riscv_vcpu_timer_init(vcpu);
···
160
158
kvm_riscv_vcpu_pmu_init(vcpu);
161
159
162
160
/* Setup VCPU AIA */
163
-
rc = kvm_riscv_vcpu_aia_init(vcpu);
164
-
if (rc)
165
-
return rc;
161
+
kvm_riscv_vcpu_aia_init(vcpu);
166
162
167
163
/*
168
164
* Setup SBI extensions
···
187
187
188
188
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
189
189
{
190
+
kvm_riscv_vcpu_sbi_deinit(vcpu);
191
+
190
192
/* Cleanup VCPU AIA context */
191
193
kvm_riscv_vcpu_aia_deinit(vcpu);
192
194
···
622
620
}
623
621
}
624
622
625
-
kvm_riscv_gstage_update_hgatp(vcpu);
623
+
kvm_riscv_mmu_update_hgatp(vcpu);
626
624
627
625
kvm_riscv_vcpu_timer_restore(vcpu);
628
626
···
682
680
}
683
681
}
684
682
685
-
static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
683
+
/**
684
+
* check_vcpu_requests - check and handle pending vCPU requests
685
+
* @vcpu: the VCPU pointer
686
+
*
687
+
* Return: 1 if we should enter the guest
688
+
* 0 if we should exit to userspace
689
+
*/
690
+
static int kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
686
691
{
687
692
struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
688
693
···
714
705
kvm_riscv_reset_vcpu(vcpu, true);
715
706
716
707
if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu))
717
-
kvm_riscv_gstage_update_hgatp(vcpu);
708
+
kvm_riscv_mmu_update_hgatp(vcpu);
718
709
719
710
if (kvm_check_request(KVM_REQ_FENCE_I, vcpu))
720
711
kvm_riscv_fence_i_process(vcpu);
721
712
722
-
/*
723
-
* The generic KVM_REQ_TLB_FLUSH is same as
724
-
* KVM_REQ_HFENCE_GVMA_VMID_ALL
725
-
*/
726
-
if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu))
727
-
kvm_riscv_hfence_gvma_vmid_all_process(vcpu);
713
+
if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
714
+
kvm_riscv_tlb_flush_process(vcpu);
728
715
729
716
if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu))
730
717
kvm_riscv_hfence_vvma_all_process(vcpu);
···
730
725
731
726
if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
732
727
kvm_riscv_vcpu_record_steal_time(vcpu);
728
+
729
+
if (kvm_dirty_ring_check_request(vcpu))
730
+
return 0;
733
731
}
732
+
733
+
return 1;
734
734
}
735
735
736
736
static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu)
···
917
907
918
908
kvm_riscv_gstage_vmid_update(vcpu);
919
909
920
-
kvm_riscv_check_vcpu_requests(vcpu);
910
+
ret = kvm_riscv_check_vcpu_requests(vcpu);
911
+
if (ret <= 0)
912
+
continue;
921
913
922
914
preempt_disable();
923
915
···
963
951
}
964
952
965
953
/*
966
-
* Cleanup stale TLB enteries
954
+
* Sanitize VMID mappings cached (TLB) on current CPU
967
955
*
968
956
* Note: This should be done after G-stage VMID has been
969
957
* updated using kvm_riscv_gstage_vmid_ver_changed()
970
958
*/
971
-
kvm_riscv_local_tlb_sanitize(vcpu);
959
+
kvm_riscv_gstage_vmid_sanitize(vcpu);
972
960
973
961
trace_kvm_entry(vcpu);
974
962
+12
-8
arch/riscv/kvm/vcpu_exit.c
+12
-8
arch/riscv/kvm/vcpu_exit.c
···
9
9
#include <linux/kvm_host.h>
10
10
#include <asm/csr.h>
11
11
#include <asm/insn-def.h>
12
+
#include <asm/kvm_mmu.h>
13
+
#include <asm/kvm_nacl.h>
12
14
13
15
static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run,
14
16
struct kvm_cpu_trap *trap)
15
17
{
18
+
struct kvm_gstage_mapping host_map;
16
19
struct kvm_memory_slot *memslot;
17
20
unsigned long hva, fault_addr;
18
21
bool writable;
···
43
40
};
44
41
}
45
42
46
-
ret = kvm_riscv_gstage_map(vcpu, memslot, fault_addr, hva,
47
-
(trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false);
43
+
ret = kvm_riscv_mmu_map(vcpu, memslot, fault_addr, hva,
44
+
(trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false,
45
+
&host_map);
48
46
if (ret < 0)
49
47
return ret;
50
48
···
139
135
void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu,
140
136
struct kvm_cpu_trap *trap)
141
137
{
142
-
unsigned long vsstatus = csr_read(CSR_VSSTATUS);
138
+
unsigned long vsstatus = ncsr_read(CSR_VSSTATUS);
143
139
144
140
/* Change Guest SSTATUS.SPP bit */
145
141
vsstatus &= ~SR_SPP;
···
155
151
vsstatus &= ~SR_SIE;
156
152
157
153
/* Update Guest SSTATUS */
158
-
csr_write(CSR_VSSTATUS, vsstatus);
154
+
ncsr_write(CSR_VSSTATUS, vsstatus);
159
155
160
156
/* Update Guest SCAUSE, STVAL, and SEPC */
161
-
csr_write(CSR_VSCAUSE, trap->scause);
162
-
csr_write(CSR_VSTVAL, trap->stval);
163
-
csr_write(CSR_VSEPC, trap->sepc);
157
+
ncsr_write(CSR_VSCAUSE, trap->scause);
158
+
ncsr_write(CSR_VSTVAL, trap->stval);
159
+
ncsr_write(CSR_VSEPC, trap->sepc);
164
160
165
161
/* Set Guest PC to Guest exception vector */
166
-
vcpu->arch.guest_context.sepc = csr_read(CSR_VSTVEC);
162
+
vcpu->arch.guest_context.sepc = ncsr_read(CSR_VSTVEC);
167
163
168
164
/* Set Guest privilege mode to supervisor */
169
165
vcpu->arch.guest_context.sstatus |= SR_SPP;
+53
-30
arch/riscv/kvm/vcpu_onereg.c
+53
-30
arch/riscv/kvm/vcpu_onereg.c
···
23
23
#define KVM_ISA_EXT_ARR(ext) \
24
24
[KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext
25
25
26
-
/* Mapping between KVM ISA Extension ID & Host ISA extension ID */
26
+
/* Mapping between KVM ISA Extension ID & guest ISA extension ID */
27
27
static const unsigned long kvm_isa_ext_arr[] = {
28
28
/* Single letter extensions (alphabetically sorted) */
29
29
[KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a,
···
35
35
[KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m,
36
36
[KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v,
37
37
/* Multi letter extensions (alphabetically sorted) */
38
-
[KVM_RISCV_ISA_EXT_SMNPM] = RISCV_ISA_EXT_SSNPM,
38
+
KVM_ISA_EXT_ARR(SMNPM),
39
39
KVM_ISA_EXT_ARR(SMSTATEEN),
40
40
KVM_ISA_EXT_ARR(SSAIA),
41
41
KVM_ISA_EXT_ARR(SSCOFPMF),
···
110
110
}
111
111
112
112
return KVM_RISCV_ISA_EXT_MAX;
113
+
}
114
+
115
+
static int kvm_riscv_vcpu_isa_check_host(unsigned long kvm_ext, unsigned long *guest_ext)
116
+
{
117
+
unsigned long host_ext;
118
+
119
+
if (kvm_ext >= KVM_RISCV_ISA_EXT_MAX ||
120
+
kvm_ext >= ARRAY_SIZE(kvm_isa_ext_arr))
121
+
return -ENOENT;
122
+
123
+
*guest_ext = kvm_isa_ext_arr[kvm_ext];
124
+
switch (*guest_ext) {
125
+
case RISCV_ISA_EXT_SMNPM:
126
+
/*
127
+
* Pointer masking effective in (H)S-mode is provided by the
128
+
* Smnpm extension, so that extension is reported to the guest,
129
+
* even though the CSR bits for configuring VS-mode pointer
130
+
* masking on the host side are part of the Ssnpm extension.
131
+
*/
132
+
host_ext = RISCV_ISA_EXT_SSNPM;
133
+
break;
134
+
default:
135
+
host_ext = *guest_ext;
136
+
break;
137
+
}
138
+
139
+
if (!__riscv_isa_extension_available(NULL, host_ext))
140
+
return -ENOENT;
141
+
142
+
return 0;
113
143
}
114
144
115
145
static bool kvm_riscv_vcpu_isa_enable_allowed(unsigned long ext)
···
249
219
250
220
void kvm_riscv_vcpu_setup_isa(struct kvm_vcpu *vcpu)
251
221
{
252
-
unsigned long host_isa, i;
222
+
unsigned long guest_ext, i;
253
223
254
224
for (i = 0; i < ARRAY_SIZE(kvm_isa_ext_arr); i++) {
255
-
host_isa = kvm_isa_ext_arr[i];
256
-
if (__riscv_isa_extension_available(NULL, host_isa) &&
257
-
kvm_riscv_vcpu_isa_enable_allowed(i))
258
-
set_bit(host_isa, vcpu->arch.isa);
225
+
if (kvm_riscv_vcpu_isa_check_host(i, &guest_ext))
226
+
continue;
227
+
if (kvm_riscv_vcpu_isa_enable_allowed(i))
228
+
set_bit(guest_ext, vcpu->arch.isa);
259
229
}
260
230
}
261
231
···
637
607
unsigned long reg_num,
638
608
unsigned long *reg_val)
639
609
{
640
-
unsigned long host_isa_ext;
610
+
unsigned long guest_ext;
611
+
int ret;
641
612
642
-
if (reg_num >= KVM_RISCV_ISA_EXT_MAX ||
643
-
reg_num >= ARRAY_SIZE(kvm_isa_ext_arr))
644
-
return -ENOENT;
645
-
646
-
host_isa_ext = kvm_isa_ext_arr[reg_num];
647
-
if (!__riscv_isa_extension_available(NULL, host_isa_ext))
648
-
return -ENOENT;
613
+
ret = kvm_riscv_vcpu_isa_check_host(reg_num, &guest_ext);
614
+
if (ret)
615
+
return ret;
649
616
650
617
*reg_val = 0;
651
-
if (__riscv_isa_extension_available(vcpu->arch.isa, host_isa_ext))
618
+
if (__riscv_isa_extension_available(vcpu->arch.isa, guest_ext))
652
619
*reg_val = 1; /* Mark the given extension as available */
653
620
654
621
return 0;
···
655
628
unsigned long reg_num,
656
629
unsigned long reg_val)
657
630
{
658
-
unsigned long host_isa_ext;
631
+
unsigned long guest_ext;
632
+
int ret;
659
633
660
-
if (reg_num >= KVM_RISCV_ISA_EXT_MAX ||
661
-
reg_num >= ARRAY_SIZE(kvm_isa_ext_arr))
662
-
return -ENOENT;
634
+
ret = kvm_riscv_vcpu_isa_check_host(reg_num, &guest_ext);
635
+
if (ret)
636
+
return ret;
663
637
664
-
host_isa_ext = kvm_isa_ext_arr[reg_num];
665
-
if (!__riscv_isa_extension_available(NULL, host_isa_ext))
666
-
return -ENOENT;
667
-
668
-
if (reg_val == test_bit(host_isa_ext, vcpu->arch.isa))
638
+
if (reg_val == test_bit(guest_ext, vcpu->arch.isa))
669
639
return 0;
670
640
671
641
if (!vcpu->arch.ran_atleast_once) {
···
672
648
*/
673
649
if (reg_val == 1 &&
674
650
kvm_riscv_vcpu_isa_enable_allowed(reg_num))
675
-
set_bit(host_isa_ext, vcpu->arch.isa);
651
+
set_bit(guest_ext, vcpu->arch.isa);
676
652
else if (!reg_val &&
677
653
kvm_riscv_vcpu_isa_disable_allowed(reg_num))
678
-
clear_bit(host_isa_ext, vcpu->arch.isa);
654
+
clear_bit(guest_ext, vcpu->arch.isa);
679
655
else
680
656
return -EINVAL;
681
657
kvm_riscv_vcpu_fp_reset(vcpu);
···
1033
1009
static int copy_isa_ext_reg_indices(const struct kvm_vcpu *vcpu,
1034
1010
u64 __user *uindices)
1035
1011
{
1012
+
unsigned long guest_ext;
1036
1013
unsigned int n = 0;
1037
-
unsigned long isa_ext;
1038
1014
1039
1015
for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) {
1040
1016
u64 size = IS_ENABLED(CONFIG_32BIT) ?
1041
1017
KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64;
1042
1018
u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_ISA_EXT | i;
1043
1019
1044
-
isa_ext = kvm_isa_ext_arr[i];
1045
-
if (!__riscv_isa_extension_available(NULL, isa_ext))
1020
+
if (kvm_riscv_vcpu_isa_check_host(i, &guest_ext))
1046
1021
continue;
1047
1022
1048
1023
if (uindices) {
+49
arch/riscv/kvm/vcpu_sbi.c
+49
arch/riscv/kvm/vcpu_sbi.c
···
536
536
scontext->ext_status[idx] = ext->default_disabled ?
537
537
KVM_RISCV_SBI_EXT_STATUS_DISABLED :
538
538
KVM_RISCV_SBI_EXT_STATUS_ENABLED;
539
+
540
+
if (ext->init && ext->init(vcpu) != 0)
541
+
scontext->ext_status[idx] = KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE;
542
+
}
543
+
}
544
+
545
+
void kvm_riscv_vcpu_sbi_deinit(struct kvm_vcpu *vcpu)
546
+
{
547
+
struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context;
548
+
const struct kvm_riscv_sbi_extension_entry *entry;
549
+
const struct kvm_vcpu_sbi_extension *ext;
550
+
int idx, i;
551
+
552
+
for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) {
553
+
entry = &sbi_ext[i];
554
+
ext = entry->ext_ptr;
555
+
idx = entry->ext_idx;
556
+
557
+
if (idx < 0 || idx >= ARRAY_SIZE(scontext->ext_status))
558
+
continue;
559
+
560
+
if (scontext->ext_status[idx] == KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE ||
561
+
!ext->deinit)
562
+
continue;
563
+
564
+
ext->deinit(vcpu);
565
+
}
566
+
}
567
+
568
+
void kvm_riscv_vcpu_sbi_reset(struct kvm_vcpu *vcpu)
569
+
{
570
+
struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context;
571
+
const struct kvm_riscv_sbi_extension_entry *entry;
572
+
const struct kvm_vcpu_sbi_extension *ext;
573
+
int idx, i;
574
+
575
+
for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) {
576
+
entry = &sbi_ext[i];
577
+
ext = entry->ext_ptr;
578
+
idx = entry->ext_idx;
579
+
580
+
if (idx < 0 || idx >= ARRAY_SIZE(scontext->ext_status))
581
+
continue;
582
+
583
+
if (scontext->ext_status[idx] != KVM_RISCV_SBI_EXT_STATUS_ENABLED ||
584
+
!ext->reset)
585
+
continue;
586
+
587
+
ext->reset(vcpu);
539
588
}
540
589
}
+9
-8
arch/riscv/kvm/vcpu_sbi_replace.c
+9
-8
arch/riscv/kvm/vcpu_sbi_replace.c
···
96
96
unsigned long hmask = cp->a0;
97
97
unsigned long hbase = cp->a1;
98
98
unsigned long funcid = cp->a6;
99
+
unsigned long vmid;
99
100
100
101
switch (funcid) {
101
102
case SBI_EXT_RFENCE_REMOTE_FENCE_I:
···
104
103
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_SENT);
105
104
break;
106
105
case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
106
+
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
107
107
if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL)
108
-
kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask);
108
+
kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask, vmid);
109
109
else
110
110
kvm_riscv_hfence_vvma_gva(vcpu->kvm, hbase, hmask,
111
-
cp->a2, cp->a3, PAGE_SHIFT);
111
+
cp->a2, cp->a3, PAGE_SHIFT, vmid);
112
112
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_SENT);
113
113
break;
114
114
case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
115
+
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
115
116
if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL)
116
-
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm,
117
-
hbase, hmask, cp->a4);
117
+
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, hbase, hmask,
118
+
cp->a4, vmid);
118
119
else
119
-
kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm,
120
-
hbase, hmask,
121
-
cp->a2, cp->a3,
122
-
PAGE_SHIFT, cp->a4);
120
+
kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, hbase, hmask, cp->a2,
121
+
cp->a3, PAGE_SHIFT, cp->a4, vmid);
123
122
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_SENT);
124
123
break;
125
124
case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA:
+2
-1
arch/riscv/kvm/vcpu_sbi_sta.c
+2
-1
arch/riscv/kvm/vcpu_sbi_sta.c
···
16
16
#include <asm/sbi.h>
17
17
#include <asm/uaccess.h>
18
18
19
-
void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu)
19
+
static void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu)
20
20
{
21
21
vcpu->arch.sta.shmem = INVALID_GPA;
22
22
vcpu->arch.sta.last_steal = 0;
···
156
156
.extid_end = SBI_EXT_STA,
157
157
.handler = kvm_sbi_ext_sta_handler,
158
158
.probe = kvm_sbi_ext_sta_probe,
159
+
.reset = kvm_riscv_vcpu_sbi_sta_reset,
159
160
};
160
161
161
162
int kvm_riscv_vcpu_get_reg_sbi_sta(struct kvm_vcpu *vcpu,
+11
-14
arch/riscv/kvm/vcpu_sbi_v01.c
+11
-14
arch/riscv/kvm/vcpu_sbi_v01.c
···
23
23
struct kvm *kvm = vcpu->kvm;
24
24
struct kvm_cpu_context *cp = &vcpu->arch.guest_context;
25
25
struct kvm_cpu_trap *utrap = retdata->utrap;
26
+
unsigned long vmid;
26
27
27
28
switch (cp->a7) {
28
29
case SBI_EXT_0_1_CONSOLE_GETCHAR:
···
79
78
if (cp->a7 == SBI_EXT_0_1_REMOTE_FENCE_I)
80
79
kvm_riscv_fence_i(vcpu->kvm, 0, hmask);
81
80
else if (cp->a7 == SBI_EXT_0_1_REMOTE_SFENCE_VMA) {
81
+
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
82
82
if (cp->a1 == 0 && cp->a2 == 0)
83
-
kvm_riscv_hfence_vvma_all(vcpu->kvm,
84
-
0, hmask);
83
+
kvm_riscv_hfence_vvma_all(vcpu->kvm, 0, hmask, vmid);
85
84
else
86
-
kvm_riscv_hfence_vvma_gva(vcpu->kvm,
87
-
0, hmask,
88
-
cp->a1, cp->a2,
89
-
PAGE_SHIFT);
85
+
kvm_riscv_hfence_vvma_gva(vcpu->kvm, 0, hmask, cp->a1,
86
+
cp->a2, PAGE_SHIFT, vmid);
90
87
} else {
88
+
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
91
89
if (cp->a1 == 0 && cp->a2 == 0)
92
-
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm,
93
-
0, hmask,
94
-
cp->a3);
90
+
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, 0, hmask,
91
+
cp->a3, vmid);
95
92
else
96
-
kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm,
97
-
0, hmask,
98
-
cp->a1, cp->a2,
99
-
PAGE_SHIFT,
100
-
cp->a3);
93
+
kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, 0, hmask,
94
+
cp->a1, cp->a2, PAGE_SHIFT,
95
+
cp->a3, vmid);
101
96
}
102
97
break;
103
98
default:
+4
-3
arch/riscv/kvm/vm.c
+4
-3
arch/riscv/kvm/vm.c
···
11
11
#include <linux/module.h>
12
12
#include <linux/uaccess.h>
13
13
#include <linux/kvm_host.h>
14
+
#include <asm/kvm_mmu.h>
14
15
15
16
const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
16
17
KVM_GENERIC_VM_STATS()
···
32
31
{
33
32
int r;
34
33
35
-
r = kvm_riscv_gstage_alloc_pgd(kvm);
34
+
r = kvm_riscv_mmu_alloc_pgd(kvm);
36
35
if (r)
37
36
return r;
38
37
39
38
r = kvm_riscv_gstage_vmid_init(kvm);
40
39
if (r) {
41
-
kvm_riscv_gstage_free_pgd(kvm);
40
+
kvm_riscv_mmu_free_pgd(kvm);
42
41
return r;
43
42
}
44
43
···
200
199
r = KVM_USER_MEM_SLOTS;
201
200
break;
202
201
case KVM_CAP_VM_GPA_BITS:
203
-
r = kvm_riscv_gstage_gpa_bits();
202
+
r = kvm_riscv_gstage_gpa_bits;
204
203
break;
205
204
default:
206
205
r = 0;
+25
arch/riscv/kvm/vmid.c
+25
arch/riscv/kvm/vmid.c
···
14
14
#include <linux/smp.h>
15
15
#include <linux/kvm_host.h>
16
16
#include <asm/csr.h>
17
+
#include <asm/kvm_tlb.h>
18
+
#include <asm/kvm_vmid.h>
17
19
18
20
static unsigned long vmid_version = 1;
19
21
static unsigned long vmid_next;
···
123
121
/* Request G-stage page table update for all VCPUs */
124
122
kvm_for_each_vcpu(i, v, vcpu->kvm)
125
123
kvm_make_request(KVM_REQ_UPDATE_HGATP, v);
124
+
}
125
+
126
+
void kvm_riscv_gstage_vmid_sanitize(struct kvm_vcpu *vcpu)
127
+
{
128
+
unsigned long vmid;
129
+
130
+
if (!kvm_riscv_gstage_vmid_bits() ||
131
+
vcpu->arch.last_exit_cpu == vcpu->cpu)
132
+
return;
133
+
134
+
/*
135
+
* On RISC-V platforms with hardware VMID support, we share same
136
+
* VMID for all VCPUs of a particular Guest/VM. This means we might
137
+
* have stale G-stage TLB entries on the current Host CPU due to
138
+
* some other VCPU of the same Guest which ran previously on the
139
+
* current Host CPU.
140
+
*
141
+
* To cleanup stale TLB entries, we simply flush all G-stage TLB
142
+
* entries by VMID whenever underlying Host CPU changes for a VCPU.
143
+
*/
144
+
145
+
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
146
+
kvm_riscv_local_hfence_gvma_vmid_all(vmid);
126
147
}
+3
-3
tools/perf/arch/riscv/util/kvm-stat.c
+3
-3
tools/perf/arch/riscv/util/kvm-stat.c
···
9
9
#include <memory.h>
10
10
#include "../../../util/evsel.h"
11
11
#include "../../../util/kvm-stat.h"
12
-
#include "riscv_exception_types.h"
12
+
#include "riscv_trap_types.h"
13
13
#include "debug.h"
14
14
15
-
define_exit_reasons_table(riscv_exit_reasons, kvm_riscv_exception_class);
15
+
define_exit_reasons_table(riscv_exit_reasons, kvm_riscv_trap_class);
16
16
17
17
const char *vcpu_id_str = "id";
18
18
const char *kvm_exit_reason = "scause";
···
30
30
struct event_key *key)
31
31
{
32
32
key->info = 0;
33
-
key->key = evsel__intval(evsel, sample, kvm_exit_reason);
33
+
key->key = evsel__intval(evsel, sample, kvm_exit_reason) & ~CAUSE_IRQ_FLAG;
34
34
key->exit_reasons = riscv_exit_reasons;
35
35
}
36
36
-35
tools/perf/arch/riscv/util/riscv_exception_types.h
-35
tools/perf/arch/riscv/util/riscv_exception_types.h
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
#ifndef ARCH_PERF_RISCV_EXCEPTION_TYPES_H
3
-
#define ARCH_PERF_RISCV_EXCEPTION_TYPES_H
4
-
5
-
#define EXC_INST_MISALIGNED 0
6
-
#define EXC_INST_ACCESS 1
7
-
#define EXC_INST_ILLEGAL 2
8
-
#define EXC_BREAKPOINT 3
9
-
#define EXC_LOAD_MISALIGNED 4
10
-
#define EXC_LOAD_ACCESS 5
11
-
#define EXC_STORE_MISALIGNED 6
12
-
#define EXC_STORE_ACCESS 7
13
-
#define EXC_SYSCALL 8
14
-
#define EXC_HYPERVISOR_SYSCALL 9
15
-
#define EXC_SUPERVISOR_SYSCALL 10
16
-
#define EXC_INST_PAGE_FAULT 12
17
-
#define EXC_LOAD_PAGE_FAULT 13
18
-
#define EXC_STORE_PAGE_FAULT 15
19
-
#define EXC_INST_GUEST_PAGE_FAULT 20
20
-
#define EXC_LOAD_GUEST_PAGE_FAULT 21
21
-
#define EXC_VIRTUAL_INST_FAULT 22
22
-
#define EXC_STORE_GUEST_PAGE_FAULT 23
23
-
24
-
#define EXC(x) {EXC_##x, #x }
25
-
26
-
#define kvm_riscv_exception_class \
27
-
EXC(INST_MISALIGNED), EXC(INST_ACCESS), EXC(INST_ILLEGAL), \
28
-
EXC(BREAKPOINT), EXC(LOAD_MISALIGNED), EXC(LOAD_ACCESS), \
29
-
EXC(STORE_MISALIGNED), EXC(STORE_ACCESS), EXC(SYSCALL), \
30
-
EXC(HYPERVISOR_SYSCALL), EXC(SUPERVISOR_SYSCALL), \
31
-
EXC(INST_PAGE_FAULT), EXC(LOAD_PAGE_FAULT), EXC(STORE_PAGE_FAULT), \
32
-
EXC(INST_GUEST_PAGE_FAULT), EXC(LOAD_GUEST_PAGE_FAULT), \
33
-
EXC(VIRTUAL_INST_FAULT), EXC(STORE_GUEST_PAGE_FAULT)
34
-
35
-
#endif /* ARCH_PERF_RISCV_EXCEPTION_TYPES_H */
+57
tools/perf/arch/riscv/util/riscv_trap_types.h
+57
tools/perf/arch/riscv/util/riscv_trap_types.h
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#ifndef ARCH_PERF_RISCV_TRAP_TYPES_H
3
+
#define ARCH_PERF_RISCV_TRAP_TYPES_H
4
+
5
+
/* Exception cause high bit - is an interrupt if set */
6
+
#define CAUSE_IRQ_FLAG (_AC(1, UL) << (__riscv_xlen - 1))
7
+
8
+
/* Interrupt causes (minus the high bit) */
9
+
#define IRQ_S_SOFT 1
10
+
#define IRQ_VS_SOFT 2
11
+
#define IRQ_M_SOFT 3
12
+
#define IRQ_S_TIMER 5
13
+
#define IRQ_VS_TIMER 6
14
+
#define IRQ_M_TIMER 7
15
+
#define IRQ_S_EXT 9
16
+
#define IRQ_VS_EXT 10
17
+
#define IRQ_M_EXT 11
18
+
#define IRQ_S_GEXT 12
19
+
#define IRQ_PMU_OVF 13
20
+
21
+
/* Exception causes */
22
+
#define EXC_INST_MISALIGNED 0
23
+
#define EXC_INST_ACCESS 1
24
+
#define EXC_INST_ILLEGAL 2
25
+
#define EXC_BREAKPOINT 3
26
+
#define EXC_LOAD_MISALIGNED 4
27
+
#define EXC_LOAD_ACCESS 5
28
+
#define EXC_STORE_MISALIGNED 6
29
+
#define EXC_STORE_ACCESS 7
30
+
#define EXC_SYSCALL 8
31
+
#define EXC_HYPERVISOR_SYSCALL 9
32
+
#define EXC_SUPERVISOR_SYSCALL 10
33
+
#define EXC_INST_PAGE_FAULT 12
34
+
#define EXC_LOAD_PAGE_FAULT 13
35
+
#define EXC_STORE_PAGE_FAULT 15
36
+
#define EXC_INST_GUEST_PAGE_FAULT 20
37
+
#define EXC_LOAD_GUEST_PAGE_FAULT 21
38
+
#define EXC_VIRTUAL_INST_FAULT 22
39
+
#define EXC_STORE_GUEST_PAGE_FAULT 23
40
+
41
+
#define TRAP(x) { x, #x }
42
+
43
+
#define kvm_riscv_trap_class \
44
+
TRAP(IRQ_S_SOFT), TRAP(IRQ_VS_SOFT), TRAP(IRQ_M_SOFT), \
45
+
TRAP(IRQ_S_TIMER), TRAP(IRQ_VS_TIMER), TRAP(IRQ_M_TIMER), \
46
+
TRAP(IRQ_S_EXT), TRAP(IRQ_VS_EXT), TRAP(IRQ_M_EXT), \
47
+
TRAP(IRQ_S_GEXT), TRAP(IRQ_PMU_OVF), \
48
+
TRAP(EXC_INST_MISALIGNED), TRAP(EXC_INST_ACCESS), TRAP(EXC_INST_ILLEGAL), \
49
+
TRAP(EXC_BREAKPOINT), TRAP(EXC_LOAD_MISALIGNED), TRAP(EXC_LOAD_ACCESS), \
50
+
TRAP(EXC_STORE_MISALIGNED), TRAP(EXC_STORE_ACCESS), TRAP(EXC_SYSCALL), \
51
+
TRAP(EXC_HYPERVISOR_SYSCALL), TRAP(EXC_SUPERVISOR_SYSCALL), \
52
+
TRAP(EXC_INST_PAGE_FAULT), TRAP(EXC_LOAD_PAGE_FAULT), \
53
+
TRAP(EXC_STORE_PAGE_FAULT), TRAP(EXC_INST_GUEST_PAGE_FAULT), \
54
+
TRAP(EXC_LOAD_GUEST_PAGE_FAULT), TRAP(EXC_VIRTUAL_INST_FAULT), \
55
+
TRAP(EXC_STORE_GUEST_PAGE_FAULT)
56
+
57
+
#endif /* ARCH_PERF_RISCV_TRAP_TYPES_H */