Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'kvm-riscv-6.5-1' of https://github.com/kvm-riscv/linux into HEAD
KVM/riscv changes for 6.5
- Redirect AMO load/store misaligned traps to KVM guest
- Trap-n-emulate AIA in-kernel irqchip for KVM guest
- Svnapot support for KVM Guest
+3100
-61
+2
arch/riscv/include/asm/csr.h
+2
arch/riscv/include/asm/csr.h
···
82
82
#define EXC_INST_ACCESS 1
83
83
#define EXC_INST_ILLEGAL 2
84
84
#define EXC_BREAKPOINT 3
85
+
#define EXC_LOAD_MISALIGNED 4
85
86
#define EXC_LOAD_ACCESS 5
87
+
#define EXC_STORE_MISALIGNED 6
86
88
#define EXC_STORE_ACCESS 7
87
89
#define EXC_SYSCALL 8
88
90
#define EXC_HYPERVISOR_SYSCALL 9
+77
-30
arch/riscv/include/asm/kvm_aia.h
+77
-30
arch/riscv/include/asm/kvm_aia.h
···
20
20
21
21
/* In-kernel irqchip initialized */
22
22
bool initialized;
23
+
24
+
/* Virtualization mode (Emulation, HW Accelerated, or Auto) */
25
+
u32 mode;
26
+
27
+
/* Number of MSIs */
28
+
u32 nr_ids;
29
+
30
+
/* Number of wired IRQs */
31
+
u32 nr_sources;
32
+
33
+
/* Number of group bits in IMSIC address */
34
+
u32 nr_group_bits;
35
+
36
+
/* Position of group bits in IMSIC address */
37
+
u32 nr_group_shift;
38
+
39
+
/* Number of hart bits in IMSIC address */
40
+
u32 nr_hart_bits;
41
+
42
+
/* Number of guest bits in IMSIC address */
43
+
u32 nr_guest_bits;
44
+
45
+
/* Guest physical address of APLIC */
46
+
gpa_t aplic_addr;
47
+
48
+
/* Internal state of APLIC */
49
+
void *aplic_state;
23
50
};
24
51
25
52
struct kvm_vcpu_aia_csr {
···
65
38
66
39
/* CPU AIA CSR context upon Guest VCPU reset */
67
40
struct kvm_vcpu_aia_csr guest_reset_csr;
41
+
42
+
/* Guest physical address of IMSIC for this VCPU */
43
+
gpa_t imsic_addr;
44
+
45
+
/* HART index of IMSIC extacted from guest physical address */
46
+
u32 hart_index;
47
+
48
+
/* Internal state of IMSIC for this VCPU */
49
+
void *imsic_state;
68
50
};
51
+
52
+
#define KVM_RISCV_AIA_UNDEF_ADDR (-1)
69
53
70
54
#define kvm_riscv_aia_initialized(k) ((k)->arch.aia.initialized)
71
55
72
56
#define irqchip_in_kernel(k) ((k)->arch.aia.in_kernel)
73
57
58
+
extern unsigned int kvm_riscv_aia_nr_hgei;
59
+
extern unsigned int kvm_riscv_aia_max_ids;
74
60
DECLARE_STATIC_KEY_FALSE(kvm_riscv_aia_available);
75
61
#define kvm_riscv_aia_available() \
76
62
static_branch_unlikely(&kvm_riscv_aia_available)
77
63
64
+
extern struct kvm_device_ops kvm_riscv_aia_device_ops;
65
+
66
+
void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu);
67
+
int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu);
68
+
78
69
#define KVM_RISCV_AIA_IMSIC_TOPEI (ISELECT_MASK + 1)
79
-
static inline int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu,
80
-
unsigned long isel,
81
-
unsigned long *val,
82
-
unsigned long new_val,
83
-
unsigned long wr_mask)
84
-
{
85
-
return 0;
86
-
}
70
+
int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, unsigned long isel,
71
+
unsigned long *val, unsigned long new_val,
72
+
unsigned long wr_mask);
73
+
int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type,
74
+
bool write, unsigned long *val);
75
+
int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type);
76
+
void kvm_riscv_vcpu_aia_imsic_reset(struct kvm_vcpu *vcpu);
77
+
int kvm_riscv_vcpu_aia_imsic_inject(struct kvm_vcpu *vcpu,
78
+
u32 guest_index, u32 offset, u32 iid);
79
+
int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu);
80
+
void kvm_riscv_vcpu_aia_imsic_cleanup(struct kvm_vcpu *vcpu);
81
+
82
+
int kvm_riscv_aia_aplic_set_attr(struct kvm *kvm, unsigned long type, u32 v);
83
+
int kvm_riscv_aia_aplic_get_attr(struct kvm *kvm, unsigned long type, u32 *v);
84
+
int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type);
85
+
int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level);
86
+
int kvm_riscv_aia_aplic_init(struct kvm *kvm);
87
+
void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm);
87
88
88
89
#ifdef CONFIG_32BIT
89
90
void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu);
···
148
93
{ .base = CSR_SIREG, .count = 1, .func = kvm_riscv_vcpu_aia_rmw_ireg }, \
149
94
{ .base = CSR_STOPEI, .count = 1, .func = kvm_riscv_vcpu_aia_rmw_topei },
150
95
151
-
static inline int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu)
152
-
{
153
-
return 1;
154
-
}
96
+
int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu);
97
+
void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu);
98
+
int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu);
99
+
void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu);
155
100
156
-
static inline void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu)
157
-
{
158
-
}
101
+
int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index,
102
+
u32 guest_index, u32 iid);
103
+
int kvm_riscv_aia_inject_msi(struct kvm *kvm, struct kvm_msi *msi);
104
+
int kvm_riscv_aia_inject_irq(struct kvm *kvm, unsigned int irq, bool level);
159
105
160
-
static inline int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu)
161
-
{
162
-
return 0;
163
-
}
106
+
void kvm_riscv_aia_init_vm(struct kvm *kvm);
107
+
void kvm_riscv_aia_destroy_vm(struct kvm *kvm);
164
108
165
-
static inline void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu)
166
-
{
167
-
}
168
-
169
-
static inline void kvm_riscv_aia_init_vm(struct kvm *kvm)
170
-
{
171
-
}
172
-
173
-
static inline void kvm_riscv_aia_destroy_vm(struct kvm *kvm)
174
-
{
175
-
}
109
+
int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner,
110
+
void __iomem **hgei_va, phys_addr_t *hgei_pa);
111
+
void kvm_riscv_aia_free_hgei(int cpu, int hgei);
112
+
void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable);
176
113
177
114
void kvm_riscv_aia_enable(void);
178
115
void kvm_riscv_aia_disable(void);
+58
arch/riscv/include/asm/kvm_aia_aplic.h
+58
arch/riscv/include/asm/kvm_aia_aplic.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* Copyright (C) 2021 Western Digital Corporation or its affiliates.
4
+
* Copyright (C) 2022 Ventana Micro Systems Inc.
5
+
*/
6
+
#ifndef __KVM_RISCV_AIA_IMSIC_H
7
+
#define __KVM_RISCV_AIA_IMSIC_H
8
+
9
+
#include <linux/bitops.h>
10
+
11
+
#define APLIC_MAX_IDC BIT(14)
12
+
#define APLIC_MAX_SOURCE 1024
13
+
14
+
#define APLIC_DOMAINCFG 0x0000
15
+
#define APLIC_DOMAINCFG_RDONLY 0x80000000
16
+
#define APLIC_DOMAINCFG_IE BIT(8)
17
+
#define APLIC_DOMAINCFG_DM BIT(2)
18
+
#define APLIC_DOMAINCFG_BE BIT(0)
19
+
20
+
#define APLIC_SOURCECFG_BASE 0x0004
21
+
#define APLIC_SOURCECFG_D BIT(10)
22
+
#define APLIC_SOURCECFG_CHILDIDX_MASK 0x000003ff
23
+
#define APLIC_SOURCECFG_SM_MASK 0x00000007
24
+
#define APLIC_SOURCECFG_SM_INACTIVE 0x0
25
+
#define APLIC_SOURCECFG_SM_DETACH 0x1
26
+
#define APLIC_SOURCECFG_SM_EDGE_RISE 0x4
27
+
#define APLIC_SOURCECFG_SM_EDGE_FALL 0x5
28
+
#define APLIC_SOURCECFG_SM_LEVEL_HIGH 0x6
29
+
#define APLIC_SOURCECFG_SM_LEVEL_LOW 0x7
30
+
31
+
#define APLIC_IRQBITS_PER_REG 32
32
+
33
+
#define APLIC_SETIP_BASE 0x1c00
34
+
#define APLIC_SETIPNUM 0x1cdc
35
+
36
+
#define APLIC_CLRIP_BASE 0x1d00
37
+
#define APLIC_CLRIPNUM 0x1ddc
38
+
39
+
#define APLIC_SETIE_BASE 0x1e00
40
+
#define APLIC_SETIENUM 0x1edc
41
+
42
+
#define APLIC_CLRIE_BASE 0x1f00
43
+
#define APLIC_CLRIENUM 0x1fdc
44
+
45
+
#define APLIC_SETIPNUM_LE 0x2000
46
+
#define APLIC_SETIPNUM_BE 0x2004
47
+
48
+
#define APLIC_GENMSI 0x3000
49
+
50
+
#define APLIC_TARGET_BASE 0x3004
51
+
#define APLIC_TARGET_HART_IDX_SHIFT 18
52
+
#define APLIC_TARGET_HART_IDX_MASK 0x3fff
53
+
#define APLIC_TARGET_GUEST_IDX_SHIFT 12
54
+
#define APLIC_TARGET_GUEST_IDX_MASK 0x3f
55
+
#define APLIC_TARGET_IPRIO_MASK 0xff
56
+
#define APLIC_TARGET_EIID_MASK 0x7ff
57
+
58
+
#endif
+38
arch/riscv/include/asm/kvm_aia_imsic.h
+38
arch/riscv/include/asm/kvm_aia_imsic.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* Copyright (C) 2021 Western Digital Corporation or its affiliates.
4
+
* Copyright (C) 2022 Ventana Micro Systems Inc.
5
+
*/
6
+
#ifndef __KVM_RISCV_AIA_IMSIC_H
7
+
#define __KVM_RISCV_AIA_IMSIC_H
8
+
9
+
#include <linux/types.h>
10
+
#include <asm/csr.h>
11
+
12
+
#define IMSIC_MMIO_PAGE_SHIFT 12
13
+
#define IMSIC_MMIO_PAGE_SZ (1UL << IMSIC_MMIO_PAGE_SHIFT)
14
+
#define IMSIC_MMIO_PAGE_LE 0x00
15
+
#define IMSIC_MMIO_PAGE_BE 0x04
16
+
17
+
#define IMSIC_MIN_ID 63
18
+
#define IMSIC_MAX_ID 2048
19
+
20
+
#define IMSIC_EIDELIVERY 0x70
21
+
22
+
#define IMSIC_EITHRESHOLD 0x72
23
+
24
+
#define IMSIC_EIP0 0x80
25
+
#define IMSIC_EIP63 0xbf
26
+
#define IMSIC_EIPx_BITS 32
27
+
28
+
#define IMSIC_EIE0 0xc0
29
+
#define IMSIC_EIE63 0xff
30
+
#define IMSIC_EIEx_BITS 32
31
+
32
+
#define IMSIC_FIRST IMSIC_EIDELIVERY
33
+
#define IMSIC_LAST IMSIC_EIE63
34
+
35
+
#define IMSIC_MMIO_SETIPNUM_LE 0x00
36
+
#define IMSIC_MMIO_SETIPNUM_BE 0x04
37
+
38
+
#endif
+4
arch/riscv/include/asm/kvm_host.h
+4
arch/riscv/include/asm/kvm_host.h
···
27
27
28
28
#define KVM_VCPU_MAX_FEATURES 0
29
29
30
+
#define KVM_IRQCHIP_NUM_PINS 1024
31
+
30
32
#define KVM_REQ_SLEEP \
31
33
KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
32
34
#define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(1)
···
319
317
int kvm_riscv_gstage_vmid_init(struct kvm *kvm);
320
318
bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid);
321
319
void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu);
320
+
321
+
int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines);
322
322
323
323
void __kvm_riscv_unpriv_trap(void);
324
324
+10
-1
arch/riscv/include/asm/kvm_vcpu_sbi.h
+10
-1
arch/riscv/include/asm/kvm_vcpu_sbi.h
···
14
14
#define KVM_SBI_VERSION_MAJOR 1
15
15
#define KVM_SBI_VERSION_MINOR 0
16
16
17
+
enum kvm_riscv_sbi_ext_status {
18
+
KVM_RISCV_SBI_EXT_UNINITIALIZED,
19
+
KVM_RISCV_SBI_EXT_AVAILABLE,
20
+
KVM_RISCV_SBI_EXT_UNAVAILABLE,
21
+
};
22
+
17
23
struct kvm_vcpu_sbi_context {
18
24
int return_handled;
19
-
bool extension_disabled[KVM_RISCV_SBI_EXT_MAX];
25
+
enum kvm_riscv_sbi_ext_status ext_status[KVM_RISCV_SBI_EXT_MAX];
20
26
};
21
27
22
28
struct kvm_vcpu_sbi_return {
···
72
66
extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental;
73
67
extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor;
74
68
69
+
#ifdef CONFIG_RISCV_PMU_SBI
70
+
extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_pmu;
71
+
#endif
75
72
#endif /* __RISCV_KVM_VCPU_SBI_H__ */
+73
arch/riscv/include/uapi/asm/kvm.h
+73
arch/riscv/include/uapi/asm/kvm.h
···
15
15
#include <asm/bitsperlong.h>
16
16
#include <asm/ptrace.h>
17
17
18
+
#define __KVM_HAVE_IRQ_LINE
18
19
#define __KVM_HAVE_READONLY_MEM
19
20
20
21
#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
···
122
121
KVM_RISCV_ISA_EXT_ZICBOZ,
123
122
KVM_RISCV_ISA_EXT_ZBB,
124
123
KVM_RISCV_ISA_EXT_SSAIA,
124
+
KVM_RISCV_ISA_EXT_SVNAPOT,
125
125
KVM_RISCV_ISA_EXT_MAX,
126
126
};
127
127
···
204
202
(1UL << ((__ext_id) % __BITS_PER_LONG))
205
203
#define KVM_REG_RISCV_SBI_MULTI_REG_LAST \
206
204
KVM_REG_RISCV_SBI_MULTI_REG(KVM_RISCV_SBI_EXT_MAX - 1)
205
+
206
+
/* Device Control API: RISC-V AIA */
207
+
#define KVM_DEV_RISCV_APLIC_ALIGN 0x1000
208
+
#define KVM_DEV_RISCV_APLIC_SIZE 0x4000
209
+
#define KVM_DEV_RISCV_APLIC_MAX_HARTS 0x4000
210
+
#define KVM_DEV_RISCV_IMSIC_ALIGN 0x1000
211
+
#define KVM_DEV_RISCV_IMSIC_SIZE 0x1000
212
+
213
+
#define KVM_DEV_RISCV_AIA_GRP_CONFIG 0
214
+
#define KVM_DEV_RISCV_AIA_CONFIG_MODE 0
215
+
#define KVM_DEV_RISCV_AIA_CONFIG_IDS 1
216
+
#define KVM_DEV_RISCV_AIA_CONFIG_SRCS 2
217
+
#define KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS 3
218
+
#define KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT 4
219
+
#define KVM_DEV_RISCV_AIA_CONFIG_HART_BITS 5
220
+
#define KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS 6
221
+
222
+
/*
223
+
* Modes of RISC-V AIA device:
224
+
* 1) EMUL (aka Emulation): Trap-n-emulate IMSIC
225
+
* 2) HWACCEL (aka HW Acceleration): Virtualize IMSIC using IMSIC guest files
226
+
* 3) AUTO (aka Automatic): Virtualize IMSIC using IMSIC guest files whenever
227
+
* available otherwise fallback to trap-n-emulation
228
+
*/
229
+
#define KVM_DEV_RISCV_AIA_MODE_EMUL 0
230
+
#define KVM_DEV_RISCV_AIA_MODE_HWACCEL 1
231
+
#define KVM_DEV_RISCV_AIA_MODE_AUTO 2
232
+
233
+
#define KVM_DEV_RISCV_AIA_IDS_MIN 63
234
+
#define KVM_DEV_RISCV_AIA_IDS_MAX 2048
235
+
#define KVM_DEV_RISCV_AIA_SRCS_MAX 1024
236
+
#define KVM_DEV_RISCV_AIA_GROUP_BITS_MAX 8
237
+
#define KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN 24
238
+
#define KVM_DEV_RISCV_AIA_GROUP_SHIFT_MAX 56
239
+
#define KVM_DEV_RISCV_AIA_HART_BITS_MAX 16
240
+
#define KVM_DEV_RISCV_AIA_GUEST_BITS_MAX 8
241
+
242
+
#define KVM_DEV_RISCV_AIA_GRP_ADDR 1
243
+
#define KVM_DEV_RISCV_AIA_ADDR_APLIC 0
244
+
#define KVM_DEV_RISCV_AIA_ADDR_IMSIC(__vcpu) (1 + (__vcpu))
245
+
#define KVM_DEV_RISCV_AIA_ADDR_MAX \
246
+
(1 + KVM_DEV_RISCV_APLIC_MAX_HARTS)
247
+
248
+
#define KVM_DEV_RISCV_AIA_GRP_CTRL 2
249
+
#define KVM_DEV_RISCV_AIA_CTRL_INIT 0
250
+
251
+
/*
252
+
* The device attribute type contains the memory mapped offset of the
253
+
* APLIC register (range 0x0000-0x3FFF) and it must be 4-byte aligned.
254
+
*/
255
+
#define KVM_DEV_RISCV_AIA_GRP_APLIC 3
256
+
257
+
/*
258
+
* The lower 12-bits of the device attribute type contains the iselect
259
+
* value of the IMSIC register (range 0x70-0xFF) whereas the higher order
260
+
* bits contains the VCPU id.
261
+
*/
262
+
#define KVM_DEV_RISCV_AIA_GRP_IMSIC 4
263
+
#define KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS 12
264
+
#define KVM_DEV_RISCV_AIA_IMSIC_ISEL_MASK \
265
+
((1U << KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS) - 1)
266
+
#define KVM_DEV_RISCV_AIA_IMSIC_MKATTR(__vcpu, __isel) \
267
+
(((__vcpu) << KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS) | \
268
+
((__isel) & KVM_DEV_RISCV_AIA_IMSIC_ISEL_MASK))
269
+
#define KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(__attr) \
270
+
((__attr) & KVM_DEV_RISCV_AIA_IMSIC_ISEL_MASK)
271
+
#define KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(__attr) \
272
+
((__attr) >> KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS)
273
+
274
+
/* One single KVM irqchip, ie. the AIA */
275
+
#define KVM_NR_IRQCHIPS 1
207
276
208
277
#endif
209
278
+4
arch/riscv/kvm/Kconfig
+4
arch/riscv/kvm/Kconfig
···
21
21
tristate "Kernel-based Virtual Machine (KVM) support (EXPERIMENTAL)"
22
22
depends on RISCV_SBI && MMU
23
23
select HAVE_KVM_EVENTFD
24
+
select HAVE_KVM_IRQCHIP
25
+
select HAVE_KVM_IRQFD
26
+
select HAVE_KVM_IRQ_ROUTING
27
+
select HAVE_KVM_MSI
24
28
select HAVE_KVM_VCPU_ASYNC_IOCTL
25
29
select KVM_GENERIC_DIRTYLOG_READ_PROTECT
26
30
select KVM_GENERIC_HARDWARE_ENABLING
+3
arch/riscv/kvm/Makefile
+3
arch/riscv/kvm/Makefile
+272
-2
arch/riscv/kvm/aia.c
+272
-2
arch/riscv/kvm/aia.c
···
8
8
*/
9
9
10
10
#include <linux/kernel.h>
11
+
#include <linux/bitops.h>
12
+
#include <linux/irq.h>
13
+
#include <linux/irqdomain.h>
11
14
#include <linux/kvm_host.h>
15
+
#include <linux/percpu.h>
16
+
#include <linux/spinlock.h>
12
17
#include <asm/hwcap.h>
18
+
#include <asm/kvm_aia_imsic.h>
13
19
20
+
struct aia_hgei_control {
21
+
raw_spinlock_t lock;
22
+
unsigned long free_bitmap;
23
+
struct kvm_vcpu *owners[BITS_PER_LONG];
24
+
};
25
+
static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei);
26
+
static int hgei_parent_irq;
27
+
28
+
unsigned int kvm_riscv_aia_nr_hgei;
29
+
unsigned int kvm_riscv_aia_max_ids;
14
30
DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available);
31
+
32
+
static int aia_find_hgei(struct kvm_vcpu *owner)
33
+
{
34
+
int i, hgei;
35
+
unsigned long flags;
36
+
struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei);
37
+
38
+
raw_spin_lock_irqsave(&hgctrl->lock, flags);
39
+
40
+
hgei = -1;
41
+
for (i = 1; i <= kvm_riscv_aia_nr_hgei; i++) {
42
+
if (hgctrl->owners[i] == owner) {
43
+
hgei = i;
44
+
break;
45
+
}
46
+
}
47
+
48
+
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
49
+
50
+
put_cpu_ptr(&aia_hgei);
51
+
return hgei;
52
+
}
15
53
16
54
static void aia_set_hvictl(bool ext_irq_pending)
17
55
{
···
94
56
95
57
bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
96
58
{
59
+
int hgei;
97
60
unsigned long seip;
98
61
99
62
if (!kvm_riscv_aia_available())
···
112
73
113
74
if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip)
114
75
return false;
76
+
77
+
hgei = aia_find_hgei(vcpu);
78
+
if (hgei > 0)
79
+
return !!(csr_read(CSR_HGEIP) & BIT(hgei));
115
80
116
81
return false;
117
82
}
···
366
323
return KVM_INSN_CONTINUE_NEXT_SEPC;
367
324
}
368
325
369
-
#define IMSIC_FIRST 0x70
370
-
#define IMSIC_LAST 0xff
371
326
int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num,
372
327
unsigned long *val, unsigned long new_val,
373
328
unsigned long wr_mask)
···
389
348
return KVM_INSN_EXIT_TO_USER_SPACE;
390
349
}
391
350
351
+
int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner,
352
+
void __iomem **hgei_va, phys_addr_t *hgei_pa)
353
+
{
354
+
int ret = -ENOENT;
355
+
unsigned long flags;
356
+
struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
357
+
358
+
if (!kvm_riscv_aia_available() || !hgctrl)
359
+
return -ENODEV;
360
+
361
+
raw_spin_lock_irqsave(&hgctrl->lock, flags);
362
+
363
+
if (hgctrl->free_bitmap) {
364
+
ret = __ffs(hgctrl->free_bitmap);
365
+
hgctrl->free_bitmap &= ~BIT(ret);
366
+
hgctrl->owners[ret] = owner;
367
+
}
368
+
369
+
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
370
+
371
+
/* TODO: To be updated later by AIA IMSIC HW guest file support */
372
+
if (hgei_va)
373
+
*hgei_va = NULL;
374
+
if (hgei_pa)
375
+
*hgei_pa = 0;
376
+
377
+
return ret;
378
+
}
379
+
380
+
void kvm_riscv_aia_free_hgei(int cpu, int hgei)
381
+
{
382
+
unsigned long flags;
383
+
struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
384
+
385
+
if (!kvm_riscv_aia_available() || !hgctrl)
386
+
return;
387
+
388
+
raw_spin_lock_irqsave(&hgctrl->lock, flags);
389
+
390
+
if (hgei > 0 && hgei <= kvm_riscv_aia_nr_hgei) {
391
+
if (!(hgctrl->free_bitmap & BIT(hgei))) {
392
+
hgctrl->free_bitmap |= BIT(hgei);
393
+
hgctrl->owners[hgei] = NULL;
394
+
}
395
+
}
396
+
397
+
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
398
+
}
399
+
400
+
void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable)
401
+
{
402
+
int hgei;
403
+
404
+
if (!kvm_riscv_aia_available())
405
+
return;
406
+
407
+
hgei = aia_find_hgei(owner);
408
+
if (hgei > 0) {
409
+
if (enable)
410
+
csr_set(CSR_HGEIE, BIT(hgei));
411
+
else
412
+
csr_clear(CSR_HGEIE, BIT(hgei));
413
+
}
414
+
}
415
+
416
+
static irqreturn_t hgei_interrupt(int irq, void *dev_id)
417
+
{
418
+
int i;
419
+
unsigned long hgei_mask, flags;
420
+
struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei);
421
+
422
+
hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE);
423
+
csr_clear(CSR_HGEIE, hgei_mask);
424
+
425
+
raw_spin_lock_irqsave(&hgctrl->lock, flags);
426
+
427
+
for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) {
428
+
if (hgctrl->owners[i])
429
+
kvm_vcpu_kick(hgctrl->owners[i]);
430
+
}
431
+
432
+
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
433
+
434
+
put_cpu_ptr(&aia_hgei);
435
+
return IRQ_HANDLED;
436
+
}
437
+
438
+
static int aia_hgei_init(void)
439
+
{
440
+
int cpu, rc;
441
+
struct irq_domain *domain;
442
+
struct aia_hgei_control *hgctrl;
443
+
444
+
/* Initialize per-CPU guest external interrupt line management */
445
+
for_each_possible_cpu(cpu) {
446
+
hgctrl = per_cpu_ptr(&aia_hgei, cpu);
447
+
raw_spin_lock_init(&hgctrl->lock);
448
+
if (kvm_riscv_aia_nr_hgei) {
449
+
hgctrl->free_bitmap =
450
+
BIT(kvm_riscv_aia_nr_hgei + 1) - 1;
451
+
hgctrl->free_bitmap &= ~BIT(0);
452
+
} else
453
+
hgctrl->free_bitmap = 0;
454
+
}
455
+
456
+
/* Find INTC irq domain */
457
+
domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(),
458
+
DOMAIN_BUS_ANY);
459
+
if (!domain) {
460
+
kvm_err("unable to find INTC domain\n");
461
+
return -ENOENT;
462
+
}
463
+
464
+
/* Map per-CPU SGEI interrupt from INTC domain */
465
+
hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT);
466
+
if (!hgei_parent_irq) {
467
+
kvm_err("unable to map SGEI IRQ\n");
468
+
return -ENOMEM;
469
+
}
470
+
471
+
/* Request per-CPU SGEI interrupt */
472
+
rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt,
473
+
"riscv-kvm", &aia_hgei);
474
+
if (rc) {
475
+
kvm_err("failed to request SGEI IRQ\n");
476
+
return rc;
477
+
}
478
+
479
+
return 0;
480
+
}
481
+
482
+
static void aia_hgei_exit(void)
483
+
{
484
+
/* Free per-CPU SGEI interrupt */
485
+
free_percpu_irq(hgei_parent_irq, &aia_hgei);
486
+
}
487
+
392
488
void kvm_riscv_aia_enable(void)
393
489
{
394
490
if (!kvm_riscv_aia_available())
···
540
362
csr_write(CSR_HVIPRIO1H, 0x0);
541
363
csr_write(CSR_HVIPRIO2H, 0x0);
542
364
#endif
365
+
366
+
/* Enable per-CPU SGEI interrupt */
367
+
enable_percpu_irq(hgei_parent_irq,
368
+
irq_get_trigger_type(hgei_parent_irq));
369
+
csr_set(CSR_HIE, BIT(IRQ_S_GEXT));
543
370
}
544
371
545
372
void kvm_riscv_aia_disable(void)
546
373
{
374
+
int i;
375
+
unsigned long flags;
376
+
struct kvm_vcpu *vcpu;
377
+
struct aia_hgei_control *hgctrl;
378
+
547
379
if (!kvm_riscv_aia_available())
548
380
return;
381
+
hgctrl = get_cpu_ptr(&aia_hgei);
382
+
383
+
/* Disable per-CPU SGEI interrupt */
384
+
csr_clear(CSR_HIE, BIT(IRQ_S_GEXT));
385
+
disable_percpu_irq(hgei_parent_irq);
549
386
550
387
aia_set_hvictl(false);
388
+
389
+
raw_spin_lock_irqsave(&hgctrl->lock, flags);
390
+
391
+
for (i = 0; i <= kvm_riscv_aia_nr_hgei; i++) {
392
+
vcpu = hgctrl->owners[i];
393
+
if (!vcpu)
394
+
continue;
395
+
396
+
/*
397
+
* We release hgctrl->lock before notifying IMSIC
398
+
* so that we don't have lock ordering issues.
399
+
*/
400
+
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
401
+
402
+
/* Notify IMSIC */
403
+
kvm_riscv_vcpu_aia_imsic_release(vcpu);
404
+
405
+
/*
406
+
* Wakeup VCPU if it was blocked so that it can
407
+
* run on other HARTs
408
+
*/
409
+
if (csr_read(CSR_HGEIE) & BIT(i)) {
410
+
csr_clear(CSR_HGEIE, BIT(i));
411
+
kvm_vcpu_kick(vcpu);
412
+
}
413
+
414
+
raw_spin_lock_irqsave(&hgctrl->lock, flags);
415
+
}
416
+
417
+
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
418
+
419
+
put_cpu_ptr(&aia_hgei);
551
420
}
552
421
553
422
int kvm_riscv_aia_init(void)
554
423
{
424
+
int rc;
425
+
555
426
if (!riscv_isa_extension_available(NULL, SxAIA))
556
427
return -ENODEV;
428
+
429
+
/* Figure-out number of bits in HGEIE */
430
+
csr_write(CSR_HGEIE, -1UL);
431
+
kvm_riscv_aia_nr_hgei = fls_long(csr_read(CSR_HGEIE));
432
+
csr_write(CSR_HGEIE, 0);
433
+
if (kvm_riscv_aia_nr_hgei)
434
+
kvm_riscv_aia_nr_hgei--;
435
+
436
+
/*
437
+
* Number of usable HGEI lines should be minimum of per-HART
438
+
* IMSIC guest files and number of bits in HGEIE
439
+
*
440
+
* TODO: To be updated later by AIA IMSIC HW guest file support
441
+
*/
442
+
kvm_riscv_aia_nr_hgei = 0;
443
+
444
+
/*
445
+
* Find number of guest MSI IDs
446
+
*
447
+
* TODO: To be updated later by AIA IMSIC HW guest file support
448
+
*/
449
+
kvm_riscv_aia_max_ids = IMSIC_MAX_ID;
450
+
451
+
/* Initialize guest external interrupt line management */
452
+
rc = aia_hgei_init();
453
+
if (rc)
454
+
return rc;
455
+
456
+
/* Register device operations */
457
+
rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops,
458
+
KVM_DEV_TYPE_RISCV_AIA);
459
+
if (rc) {
460
+
aia_hgei_exit();
461
+
return rc;
462
+
}
557
463
558
464
/* Enable KVM AIA support */
559
465
static_branch_enable(&kvm_riscv_aia_available);
···
647
385
648
386
void kvm_riscv_aia_exit(void)
649
387
{
388
+
if (!kvm_riscv_aia_available())
389
+
return;
390
+
391
+
/* Unregister device operations */
392
+
kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA);
393
+
394
+
/* Cleanup the HGEI state */
395
+
aia_hgei_exit();
650
396
}
+619
arch/riscv/kvm/aia_aplic.c
+619
arch/riscv/kvm/aia_aplic.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* Copyright (C) 2021 Western Digital Corporation or its affiliates.
4
+
* Copyright (C) 2022 Ventana Micro Systems Inc.
5
+
*
6
+
* Authors:
7
+
* Anup Patel <apatel@ventanamicro.com>
8
+
*/
9
+
10
+
#include <linux/kvm_host.h>
11
+
#include <linux/math.h>
12
+
#include <linux/spinlock.h>
13
+
#include <linux/swab.h>
14
+
#include <kvm/iodev.h>
15
+
#include <asm/kvm_aia_aplic.h>
16
+
17
+
struct aplic_irq {
18
+
raw_spinlock_t lock;
19
+
u32 sourcecfg;
20
+
u32 state;
21
+
#define APLIC_IRQ_STATE_PENDING BIT(0)
22
+
#define APLIC_IRQ_STATE_ENABLED BIT(1)
23
+
#define APLIC_IRQ_STATE_ENPEND (APLIC_IRQ_STATE_PENDING | \
24
+
APLIC_IRQ_STATE_ENABLED)
25
+
#define APLIC_IRQ_STATE_INPUT BIT(8)
26
+
u32 target;
27
+
};
28
+
29
+
struct aplic {
30
+
struct kvm_io_device iodev;
31
+
32
+
u32 domaincfg;
33
+
u32 genmsi;
34
+
35
+
u32 nr_irqs;
36
+
u32 nr_words;
37
+
struct aplic_irq *irqs;
38
+
};
39
+
40
+
static u32 aplic_read_sourcecfg(struct aplic *aplic, u32 irq)
41
+
{
42
+
u32 ret;
43
+
unsigned long flags;
44
+
struct aplic_irq *irqd;
45
+
46
+
if (!irq || aplic->nr_irqs <= irq)
47
+
return 0;
48
+
irqd = &aplic->irqs[irq];
49
+
50
+
raw_spin_lock_irqsave(&irqd->lock, flags);
51
+
ret = irqd->sourcecfg;
52
+
raw_spin_unlock_irqrestore(&irqd->lock, flags);
53
+
54
+
return ret;
55
+
}
56
+
57
+
static void aplic_write_sourcecfg(struct aplic *aplic, u32 irq, u32 val)
58
+
{
59
+
unsigned long flags;
60
+
struct aplic_irq *irqd;
61
+
62
+
if (!irq || aplic->nr_irqs <= irq)
63
+
return;
64
+
irqd = &aplic->irqs[irq];
65
+
66
+
if (val & APLIC_SOURCECFG_D)
67
+
val = 0;
68
+
else
69
+
val &= APLIC_SOURCECFG_SM_MASK;
70
+
71
+
raw_spin_lock_irqsave(&irqd->lock, flags);
72
+
irqd->sourcecfg = val;
73
+
raw_spin_unlock_irqrestore(&irqd->lock, flags);
74
+
}
75
+
76
+
static u32 aplic_read_target(struct aplic *aplic, u32 irq)
77
+
{
78
+
u32 ret;
79
+
unsigned long flags;
80
+
struct aplic_irq *irqd;
81
+
82
+
if (!irq || aplic->nr_irqs <= irq)
83
+
return 0;
84
+
irqd = &aplic->irqs[irq];
85
+
86
+
raw_spin_lock_irqsave(&irqd->lock, flags);
87
+
ret = irqd->target;
88
+
raw_spin_unlock_irqrestore(&irqd->lock, flags);
89
+
90
+
return ret;
91
+
}
92
+
93
+
static void aplic_write_target(struct aplic *aplic, u32 irq, u32 val)
94
+
{
95
+
unsigned long flags;
96
+
struct aplic_irq *irqd;
97
+
98
+
if (!irq || aplic->nr_irqs <= irq)
99
+
return;
100
+
irqd = &aplic->irqs[irq];
101
+
102
+
val &= APLIC_TARGET_EIID_MASK |
103
+
(APLIC_TARGET_HART_IDX_MASK << APLIC_TARGET_HART_IDX_SHIFT) |
104
+
(APLIC_TARGET_GUEST_IDX_MASK << APLIC_TARGET_GUEST_IDX_SHIFT);
105
+
106
+
raw_spin_lock_irqsave(&irqd->lock, flags);
107
+
irqd->target = val;
108
+
raw_spin_unlock_irqrestore(&irqd->lock, flags);
109
+
}
110
+
111
+
static bool aplic_read_pending(struct aplic *aplic, u32 irq)
112
+
{
113
+
bool ret;
114
+
unsigned long flags;
115
+
struct aplic_irq *irqd;
116
+
117
+
if (!irq || aplic->nr_irqs <= irq)
118
+
return false;
119
+
irqd = &aplic->irqs[irq];
120
+
121
+
raw_spin_lock_irqsave(&irqd->lock, flags);
122
+
ret = (irqd->state & APLIC_IRQ_STATE_PENDING) ? true : false;
123
+
raw_spin_unlock_irqrestore(&irqd->lock, flags);
124
+
125
+
return ret;
126
+
}
127
+
128
+
static void aplic_write_pending(struct aplic *aplic, u32 irq, bool pending)
129
+
{
130
+
unsigned long flags, sm;
131
+
struct aplic_irq *irqd;
132
+
133
+
if (!irq || aplic->nr_irqs <= irq)
134
+
return;
135
+
irqd = &aplic->irqs[irq];
136
+
137
+
raw_spin_lock_irqsave(&irqd->lock, flags);
138
+
139
+
sm = irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK;
140
+
if (!pending &&
141
+
((sm == APLIC_SOURCECFG_SM_LEVEL_HIGH) ||
142
+
(sm == APLIC_SOURCECFG_SM_LEVEL_LOW)))
143
+
goto skip_write_pending;
144
+
145
+
if (pending)
146
+
irqd->state |= APLIC_IRQ_STATE_PENDING;
147
+
else
148
+
irqd->state &= ~APLIC_IRQ_STATE_PENDING;
149
+
150
+
skip_write_pending:
151
+
raw_spin_unlock_irqrestore(&irqd->lock, flags);
152
+
}
153
+
154
+
static bool aplic_read_enabled(struct aplic *aplic, u32 irq)
155
+
{
156
+
bool ret;
157
+
unsigned long flags;
158
+
struct aplic_irq *irqd;
159
+
160
+
if (!irq || aplic->nr_irqs <= irq)
161
+
return false;
162
+
irqd = &aplic->irqs[irq];
163
+
164
+
raw_spin_lock_irqsave(&irqd->lock, flags);
165
+
ret = (irqd->state & APLIC_IRQ_STATE_ENABLED) ? true : false;
166
+
raw_spin_unlock_irqrestore(&irqd->lock, flags);
167
+
168
+
return ret;
169
+
}
170
+
171
+
static void aplic_write_enabled(struct aplic *aplic, u32 irq, bool enabled)
172
+
{
173
+
unsigned long flags;
174
+
struct aplic_irq *irqd;
175
+
176
+
if (!irq || aplic->nr_irqs <= irq)
177
+
return;
178
+
irqd = &aplic->irqs[irq];
179
+
180
+
raw_spin_lock_irqsave(&irqd->lock, flags);
181
+
if (enabled)
182
+
irqd->state |= APLIC_IRQ_STATE_ENABLED;
183
+
else
184
+
irqd->state &= ~APLIC_IRQ_STATE_ENABLED;
185
+
raw_spin_unlock_irqrestore(&irqd->lock, flags);
186
+
}
187
+
188
+
static bool aplic_read_input(struct aplic *aplic, u32 irq)
189
+
{
190
+
bool ret;
191
+
unsigned long flags;
192
+
struct aplic_irq *irqd;
193
+
194
+
if (!irq || aplic->nr_irqs <= irq)
195
+
return false;
196
+
irqd = &aplic->irqs[irq];
197
+
198
+
raw_spin_lock_irqsave(&irqd->lock, flags);
199
+
ret = (irqd->state & APLIC_IRQ_STATE_INPUT) ? true : false;
200
+
raw_spin_unlock_irqrestore(&irqd->lock, flags);
201
+
202
+
return ret;
203
+
}
204
+
205
+
static void aplic_inject_msi(struct kvm *kvm, u32 irq, u32 target)
206
+
{
207
+
u32 hart_idx, guest_idx, eiid;
208
+
209
+
hart_idx = target >> APLIC_TARGET_HART_IDX_SHIFT;
210
+
hart_idx &= APLIC_TARGET_HART_IDX_MASK;
211
+
guest_idx = target >> APLIC_TARGET_GUEST_IDX_SHIFT;
212
+
guest_idx &= APLIC_TARGET_GUEST_IDX_MASK;
213
+
eiid = target & APLIC_TARGET_EIID_MASK;
214
+
kvm_riscv_aia_inject_msi_by_id(kvm, hart_idx, guest_idx, eiid);
215
+
}
216
+
217
+
static void aplic_update_irq_range(struct kvm *kvm, u32 first, u32 last)
218
+
{
219
+
bool inject;
220
+
u32 irq, target;
221
+
unsigned long flags;
222
+
struct aplic_irq *irqd;
223
+
struct aplic *aplic = kvm->arch.aia.aplic_state;
224
+
225
+
if (!(aplic->domaincfg & APLIC_DOMAINCFG_IE))
226
+
return;
227
+
228
+
for (irq = first; irq <= last; irq++) {
229
+
if (!irq || aplic->nr_irqs <= irq)
230
+
continue;
231
+
irqd = &aplic->irqs[irq];
232
+
233
+
raw_spin_lock_irqsave(&irqd->lock, flags);
234
+
235
+
inject = false;
236
+
target = irqd->target;
237
+
if ((irqd->state & APLIC_IRQ_STATE_ENPEND) ==
238
+
APLIC_IRQ_STATE_ENPEND) {
239
+
irqd->state &= ~APLIC_IRQ_STATE_PENDING;
240
+
inject = true;
241
+
}
242
+
243
+
raw_spin_unlock_irqrestore(&irqd->lock, flags);
244
+
245
+
if (inject)
246
+
aplic_inject_msi(kvm, irq, target);
247
+
}
248
+
}
249
+
250
+
int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level)
251
+
{
252
+
u32 target;
253
+
bool inject = false, ie;
254
+
unsigned long flags;
255
+
struct aplic_irq *irqd;
256
+
struct aplic *aplic = kvm->arch.aia.aplic_state;
257
+
258
+
if (!aplic || !source || (aplic->nr_irqs <= source))
259
+
return -ENODEV;
260
+
irqd = &aplic->irqs[source];
261
+
ie = (aplic->domaincfg & APLIC_DOMAINCFG_IE) ? true : false;
262
+
263
+
raw_spin_lock_irqsave(&irqd->lock, flags);
264
+
265
+
if (irqd->sourcecfg & APLIC_SOURCECFG_D)
266
+
goto skip_unlock;
267
+
268
+
switch (irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK) {
269
+
case APLIC_SOURCECFG_SM_EDGE_RISE:
270
+
if (level && !(irqd->state & APLIC_IRQ_STATE_INPUT) &&
271
+
!(irqd->state & APLIC_IRQ_STATE_PENDING))
272
+
irqd->state |= APLIC_IRQ_STATE_PENDING;
273
+
break;
274
+
case APLIC_SOURCECFG_SM_EDGE_FALL:
275
+
if (!level && (irqd->state & APLIC_IRQ_STATE_INPUT) &&
276
+
!(irqd->state & APLIC_IRQ_STATE_PENDING))
277
+
irqd->state |= APLIC_IRQ_STATE_PENDING;
278
+
break;
279
+
case APLIC_SOURCECFG_SM_LEVEL_HIGH:
280
+
if (level && !(irqd->state & APLIC_IRQ_STATE_PENDING))
281
+
irqd->state |= APLIC_IRQ_STATE_PENDING;
282
+
break;
283
+
case APLIC_SOURCECFG_SM_LEVEL_LOW:
284
+
if (!level && !(irqd->state & APLIC_IRQ_STATE_PENDING))
285
+
irqd->state |= APLIC_IRQ_STATE_PENDING;
286
+
break;
287
+
}
288
+
289
+
if (level)
290
+
irqd->state |= APLIC_IRQ_STATE_INPUT;
291
+
else
292
+
irqd->state &= ~APLIC_IRQ_STATE_INPUT;
293
+
294
+
target = irqd->target;
295
+
if (ie && ((irqd->state & APLIC_IRQ_STATE_ENPEND) ==
296
+
APLIC_IRQ_STATE_ENPEND)) {
297
+
irqd->state &= ~APLIC_IRQ_STATE_PENDING;
298
+
inject = true;
299
+
}
300
+
301
+
skip_unlock:
302
+
raw_spin_unlock_irqrestore(&irqd->lock, flags);
303
+
304
+
if (inject)
305
+
aplic_inject_msi(kvm, source, target);
306
+
307
+
return 0;
308
+
}
309
+
310
+
static u32 aplic_read_input_word(struct aplic *aplic, u32 word)
311
+
{
312
+
u32 i, ret = 0;
313
+
314
+
for (i = 0; i < 32; i++)
315
+
ret |= aplic_read_input(aplic, word * 32 + i) ? BIT(i) : 0;
316
+
317
+
return ret;
318
+
}
319
+
320
+
static u32 aplic_read_pending_word(struct aplic *aplic, u32 word)
321
+
{
322
+
u32 i, ret = 0;
323
+
324
+
for (i = 0; i < 32; i++)
325
+
ret |= aplic_read_pending(aplic, word * 32 + i) ? BIT(i) : 0;
326
+
327
+
return ret;
328
+
}
329
+
330
+
static void aplic_write_pending_word(struct aplic *aplic, u32 word,
331
+
u32 val, bool pending)
332
+
{
333
+
u32 i;
334
+
335
+
for (i = 0; i < 32; i++) {
336
+
if (val & BIT(i))
337
+
aplic_write_pending(aplic, word * 32 + i, pending);
338
+
}
339
+
}
340
+
341
+
static u32 aplic_read_enabled_word(struct aplic *aplic, u32 word)
342
+
{
343
+
u32 i, ret = 0;
344
+
345
+
for (i = 0; i < 32; i++)
346
+
ret |= aplic_read_enabled(aplic, word * 32 + i) ? BIT(i) : 0;
347
+
348
+
return ret;
349
+
}
350
+
351
+
static void aplic_write_enabled_word(struct aplic *aplic, u32 word,
352
+
u32 val, bool enabled)
353
+
{
354
+
u32 i;
355
+
356
+
for (i = 0; i < 32; i++) {
357
+
if (val & BIT(i))
358
+
aplic_write_enabled(aplic, word * 32 + i, enabled);
359
+
}
360
+
}
361
+
362
+
static int aplic_mmio_read_offset(struct kvm *kvm, gpa_t off, u32 *val32)
363
+
{
364
+
u32 i;
365
+
struct aplic *aplic = kvm->arch.aia.aplic_state;
366
+
367
+
if ((off & 0x3) != 0)
368
+
return -EOPNOTSUPP;
369
+
370
+
if (off == APLIC_DOMAINCFG) {
371
+
*val32 = APLIC_DOMAINCFG_RDONLY |
372
+
aplic->domaincfg | APLIC_DOMAINCFG_DM;
373
+
} else if ((off >= APLIC_SOURCECFG_BASE) &&
374
+
(off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) {
375
+
i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1;
376
+
*val32 = aplic_read_sourcecfg(aplic, i);
377
+
} else if ((off >= APLIC_SETIP_BASE) &&
378
+
(off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) {
379
+
i = (off - APLIC_SETIP_BASE) >> 2;
380
+
*val32 = aplic_read_pending_word(aplic, i);
381
+
} else if (off == APLIC_SETIPNUM) {
382
+
*val32 = 0;
383
+
} else if ((off >= APLIC_CLRIP_BASE) &&
384
+
(off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) {
385
+
i = (off - APLIC_CLRIP_BASE) >> 2;
386
+
*val32 = aplic_read_input_word(aplic, i);
387
+
} else if (off == APLIC_CLRIPNUM) {
388
+
*val32 = 0;
389
+
} else if ((off >= APLIC_SETIE_BASE) &&
390
+
(off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) {
391
+
i = (off - APLIC_SETIE_BASE) >> 2;
392
+
*val32 = aplic_read_enabled_word(aplic, i);
393
+
} else if (off == APLIC_SETIENUM) {
394
+
*val32 = 0;
395
+
} else if ((off >= APLIC_CLRIE_BASE) &&
396
+
(off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) {
397
+
*val32 = 0;
398
+
} else if (off == APLIC_CLRIENUM) {
399
+
*val32 = 0;
400
+
} else if (off == APLIC_SETIPNUM_LE) {
401
+
*val32 = 0;
402
+
} else if (off == APLIC_SETIPNUM_BE) {
403
+
*val32 = 0;
404
+
} else if (off == APLIC_GENMSI) {
405
+
*val32 = aplic->genmsi;
406
+
} else if ((off >= APLIC_TARGET_BASE) &&
407
+
(off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) {
408
+
i = ((off - APLIC_TARGET_BASE) >> 2) + 1;
409
+
*val32 = aplic_read_target(aplic, i);
410
+
} else
411
+
return -ENODEV;
412
+
413
+
return 0;
414
+
}
415
+
416
+
static int aplic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
417
+
gpa_t addr, int len, void *val)
418
+
{
419
+
if (len != 4)
420
+
return -EOPNOTSUPP;
421
+
422
+
return aplic_mmio_read_offset(vcpu->kvm,
423
+
addr - vcpu->kvm->arch.aia.aplic_addr,
424
+
val);
425
+
}
426
+
427
+
static int aplic_mmio_write_offset(struct kvm *kvm, gpa_t off, u32 val32)
428
+
{
429
+
u32 i;
430
+
struct aplic *aplic = kvm->arch.aia.aplic_state;
431
+
432
+
if ((off & 0x3) != 0)
433
+
return -EOPNOTSUPP;
434
+
435
+
if (off == APLIC_DOMAINCFG) {
436
+
/* Only IE bit writeable */
437
+
aplic->domaincfg = val32 & APLIC_DOMAINCFG_IE;
438
+
} else if ((off >= APLIC_SOURCECFG_BASE) &&
439
+
(off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) {
440
+
i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1;
441
+
aplic_write_sourcecfg(aplic, i, val32);
442
+
} else if ((off >= APLIC_SETIP_BASE) &&
443
+
(off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) {
444
+
i = (off - APLIC_SETIP_BASE) >> 2;
445
+
aplic_write_pending_word(aplic, i, val32, true);
446
+
} else if (off == APLIC_SETIPNUM) {
447
+
aplic_write_pending(aplic, val32, true);
448
+
} else if ((off >= APLIC_CLRIP_BASE) &&
449
+
(off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) {
450
+
i = (off - APLIC_CLRIP_BASE) >> 2;
451
+
aplic_write_pending_word(aplic, i, val32, false);
452
+
} else if (off == APLIC_CLRIPNUM) {
453
+
aplic_write_pending(aplic, val32, false);
454
+
} else if ((off >= APLIC_SETIE_BASE) &&
455
+
(off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) {
456
+
i = (off - APLIC_SETIE_BASE) >> 2;
457
+
aplic_write_enabled_word(aplic, i, val32, true);
458
+
} else if (off == APLIC_SETIENUM) {
459
+
aplic_write_enabled(aplic, val32, true);
460
+
} else if ((off >= APLIC_CLRIE_BASE) &&
461
+
(off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) {
462
+
i = (off - APLIC_CLRIE_BASE) >> 2;
463
+
aplic_write_enabled_word(aplic, i, val32, false);
464
+
} else if (off == APLIC_CLRIENUM) {
465
+
aplic_write_enabled(aplic, val32, false);
466
+
} else if (off == APLIC_SETIPNUM_LE) {
467
+
aplic_write_pending(aplic, val32, true);
468
+
} else if (off == APLIC_SETIPNUM_BE) {
469
+
aplic_write_pending(aplic, __swab32(val32), true);
470
+
} else if (off == APLIC_GENMSI) {
471
+
aplic->genmsi = val32 & ~(APLIC_TARGET_GUEST_IDX_MASK <<
472
+
APLIC_TARGET_GUEST_IDX_SHIFT);
473
+
kvm_riscv_aia_inject_msi_by_id(kvm,
474
+
val32 >> APLIC_TARGET_HART_IDX_SHIFT, 0,
475
+
val32 & APLIC_TARGET_EIID_MASK);
476
+
} else if ((off >= APLIC_TARGET_BASE) &&
477
+
(off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) {
478
+
i = ((off - APLIC_TARGET_BASE) >> 2) + 1;
479
+
aplic_write_target(aplic, i, val32);
480
+
} else
481
+
return -ENODEV;
482
+
483
+
aplic_update_irq_range(kvm, 1, aplic->nr_irqs - 1);
484
+
485
+
return 0;
486
+
}
487
+
488
+
static int aplic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
489
+
gpa_t addr, int len, const void *val)
490
+
{
491
+
if (len != 4)
492
+
return -EOPNOTSUPP;
493
+
494
+
return aplic_mmio_write_offset(vcpu->kvm,
495
+
addr - vcpu->kvm->arch.aia.aplic_addr,
496
+
*((const u32 *)val));
497
+
}
498
+
499
+
static struct kvm_io_device_ops aplic_iodoev_ops = {
500
+
.read = aplic_mmio_read,
501
+
.write = aplic_mmio_write,
502
+
};
503
+
504
+
int kvm_riscv_aia_aplic_set_attr(struct kvm *kvm, unsigned long type, u32 v)
505
+
{
506
+
int rc;
507
+
508
+
if (!kvm->arch.aia.aplic_state)
509
+
return -ENODEV;
510
+
511
+
rc = aplic_mmio_write_offset(kvm, type, v);
512
+
if (rc)
513
+
return rc;
514
+
515
+
return 0;
516
+
}
517
+
518
+
int kvm_riscv_aia_aplic_get_attr(struct kvm *kvm, unsigned long type, u32 *v)
519
+
{
520
+
int rc;
521
+
522
+
if (!kvm->arch.aia.aplic_state)
523
+
return -ENODEV;
524
+
525
+
rc = aplic_mmio_read_offset(kvm, type, v);
526
+
if (rc)
527
+
return rc;
528
+
529
+
return 0;
530
+
}
531
+
532
+
int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type)
533
+
{
534
+
int rc;
535
+
u32 val;
536
+
537
+
if (!kvm->arch.aia.aplic_state)
538
+
return -ENODEV;
539
+
540
+
rc = aplic_mmio_read_offset(kvm, type, &val);
541
+
if (rc)
542
+
return rc;
543
+
544
+
return 0;
545
+
}
546
+
547
+
int kvm_riscv_aia_aplic_init(struct kvm *kvm)
548
+
{
549
+
int i, ret = 0;
550
+
struct aplic *aplic;
551
+
552
+
/* Do nothing if we have zero sources */
553
+
if (!kvm->arch.aia.nr_sources)
554
+
return 0;
555
+
556
+
/* Allocate APLIC global state */
557
+
aplic = kzalloc(sizeof(*aplic), GFP_KERNEL);
558
+
if (!aplic)
559
+
return -ENOMEM;
560
+
kvm->arch.aia.aplic_state = aplic;
561
+
562
+
/* Setup APLIC IRQs */
563
+
aplic->nr_irqs = kvm->arch.aia.nr_sources + 1;
564
+
aplic->nr_words = DIV_ROUND_UP(aplic->nr_irqs, 32);
565
+
aplic->irqs = kcalloc(aplic->nr_irqs,
566
+
sizeof(*aplic->irqs), GFP_KERNEL);
567
+
if (!aplic->irqs) {
568
+
ret = -ENOMEM;
569
+
goto fail_free_aplic;
570
+
}
571
+
for (i = 0; i < aplic->nr_irqs; i++)
572
+
raw_spin_lock_init(&aplic->irqs[i].lock);
573
+
574
+
/* Setup IO device */
575
+
kvm_iodevice_init(&aplic->iodev, &aplic_iodoev_ops);
576
+
mutex_lock(&kvm->slots_lock);
577
+
ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
578
+
kvm->arch.aia.aplic_addr,
579
+
KVM_DEV_RISCV_APLIC_SIZE,
580
+
&aplic->iodev);
581
+
mutex_unlock(&kvm->slots_lock);
582
+
if (ret)
583
+
goto fail_free_aplic_irqs;
584
+
585
+
/* Setup default IRQ routing */
586
+
ret = kvm_riscv_setup_default_irq_routing(kvm, aplic->nr_irqs);
587
+
if (ret)
588
+
goto fail_unreg_iodev;
589
+
590
+
return 0;
591
+
592
+
fail_unreg_iodev:
593
+
mutex_lock(&kvm->slots_lock);
594
+
kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
595
+
mutex_unlock(&kvm->slots_lock);
596
+
fail_free_aplic_irqs:
597
+
kfree(aplic->irqs);
598
+
fail_free_aplic:
599
+
kvm->arch.aia.aplic_state = NULL;
600
+
kfree(aplic);
601
+
return ret;
602
+
}
603
+
604
+
void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm)
605
+
{
606
+
struct aplic *aplic = kvm->arch.aia.aplic_state;
607
+
608
+
if (!aplic)
609
+
return;
610
+
611
+
mutex_lock(&kvm->slots_lock);
612
+
kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
613
+
mutex_unlock(&kvm->slots_lock);
614
+
615
+
kfree(aplic->irqs);
616
+
617
+
kvm->arch.aia.aplic_state = NULL;
618
+
kfree(aplic);
619
+
}
+673
arch/riscv/kvm/aia_device.c
+673
arch/riscv/kvm/aia_device.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* Copyright (C) 2021 Western Digital Corporation or its affiliates.
4
+
* Copyright (C) 2022 Ventana Micro Systems Inc.
5
+
*
6
+
* Authors:
7
+
* Anup Patel <apatel@ventanamicro.com>
8
+
*/
9
+
10
+
#include <linux/bits.h>
11
+
#include <linux/kvm_host.h>
12
+
#include <linux/uaccess.h>
13
+
#include <asm/kvm_aia_imsic.h>
14
+
15
+
static void unlock_vcpus(struct kvm *kvm, int vcpu_lock_idx)
16
+
{
17
+
struct kvm_vcpu *tmp_vcpu;
18
+
19
+
for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
20
+
tmp_vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
21
+
mutex_unlock(&tmp_vcpu->mutex);
22
+
}
23
+
}
24
+
25
+
static void unlock_all_vcpus(struct kvm *kvm)
26
+
{
27
+
unlock_vcpus(kvm, atomic_read(&kvm->online_vcpus) - 1);
28
+
}
29
+
30
+
static bool lock_all_vcpus(struct kvm *kvm)
31
+
{
32
+
struct kvm_vcpu *tmp_vcpu;
33
+
unsigned long c;
34
+
35
+
kvm_for_each_vcpu(c, tmp_vcpu, kvm) {
36
+
if (!mutex_trylock(&tmp_vcpu->mutex)) {
37
+
unlock_vcpus(kvm, c - 1);
38
+
return false;
39
+
}
40
+
}
41
+
42
+
return true;
43
+
}
44
+
45
+
static int aia_create(struct kvm_device *dev, u32 type)
46
+
{
47
+
int ret;
48
+
unsigned long i;
49
+
struct kvm *kvm = dev->kvm;
50
+
struct kvm_vcpu *vcpu;
51
+
52
+
if (irqchip_in_kernel(kvm))
53
+
return -EEXIST;
54
+
55
+
ret = -EBUSY;
56
+
if (!lock_all_vcpus(kvm))
57
+
return ret;
58
+
59
+
kvm_for_each_vcpu(i, vcpu, kvm) {
60
+
if (vcpu->arch.ran_atleast_once)
61
+
goto out_unlock;
62
+
}
63
+
ret = 0;
64
+
65
+
kvm->arch.aia.in_kernel = true;
66
+
67
+
out_unlock:
68
+
unlock_all_vcpus(kvm);
69
+
return ret;
70
+
}
71
+
72
+
static void aia_destroy(struct kvm_device *dev)
73
+
{
74
+
kfree(dev);
75
+
}
76
+
77
+
static int aia_config(struct kvm *kvm, unsigned long type,
78
+
u32 *nr, bool write)
79
+
{
80
+
struct kvm_aia *aia = &kvm->arch.aia;
81
+
82
+
/* Writes can only be done before irqchip is initialized */
83
+
if (write && kvm_riscv_aia_initialized(kvm))
84
+
return -EBUSY;
85
+
86
+
switch (type) {
87
+
case KVM_DEV_RISCV_AIA_CONFIG_MODE:
88
+
if (write) {
89
+
switch (*nr) {
90
+
case KVM_DEV_RISCV_AIA_MODE_EMUL:
91
+
break;
92
+
case KVM_DEV_RISCV_AIA_MODE_HWACCEL:
93
+
case KVM_DEV_RISCV_AIA_MODE_AUTO:
94
+
/*
95
+
* HW Acceleration and Auto modes only
96
+
* supported on host with non-zero guest
97
+
* external interrupts (i.e. non-zero
98
+
* VS-level IMSIC pages).
99
+
*/
100
+
if (!kvm_riscv_aia_nr_hgei)
101
+
return -EINVAL;
102
+
break;
103
+
default:
104
+
return -EINVAL;
105
+
}
106
+
aia->mode = *nr;
107
+
} else
108
+
*nr = aia->mode;
109
+
break;
110
+
case KVM_DEV_RISCV_AIA_CONFIG_IDS:
111
+
if (write) {
112
+
if ((*nr < KVM_DEV_RISCV_AIA_IDS_MIN) ||
113
+
(*nr >= KVM_DEV_RISCV_AIA_IDS_MAX) ||
114
+
((*nr & KVM_DEV_RISCV_AIA_IDS_MIN) !=
115
+
KVM_DEV_RISCV_AIA_IDS_MIN) ||
116
+
(kvm_riscv_aia_max_ids <= *nr))
117
+
return -EINVAL;
118
+
aia->nr_ids = *nr;
119
+
} else
120
+
*nr = aia->nr_ids;
121
+
break;
122
+
case KVM_DEV_RISCV_AIA_CONFIG_SRCS:
123
+
if (write) {
124
+
if ((*nr >= KVM_DEV_RISCV_AIA_SRCS_MAX) ||
125
+
(*nr >= kvm_riscv_aia_max_ids))
126
+
return -EINVAL;
127
+
aia->nr_sources = *nr;
128
+
} else
129
+
*nr = aia->nr_sources;
130
+
break;
131
+
case KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS:
132
+
if (write) {
133
+
if (*nr >= KVM_DEV_RISCV_AIA_GROUP_BITS_MAX)
134
+
return -EINVAL;
135
+
aia->nr_group_bits = *nr;
136
+
} else
137
+
*nr = aia->nr_group_bits;
138
+
break;
139
+
case KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT:
140
+
if (write) {
141
+
if ((*nr < KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN) ||
142
+
(*nr >= KVM_DEV_RISCV_AIA_GROUP_SHIFT_MAX))
143
+
return -EINVAL;
144
+
aia->nr_group_shift = *nr;
145
+
} else
146
+
*nr = aia->nr_group_shift;
147
+
break;
148
+
case KVM_DEV_RISCV_AIA_CONFIG_HART_BITS:
149
+
if (write) {
150
+
if (*nr >= KVM_DEV_RISCV_AIA_HART_BITS_MAX)
151
+
return -EINVAL;
152
+
aia->nr_hart_bits = *nr;
153
+
} else
154
+
*nr = aia->nr_hart_bits;
155
+
break;
156
+
case KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS:
157
+
if (write) {
158
+
if (*nr >= KVM_DEV_RISCV_AIA_GUEST_BITS_MAX)
159
+
return -EINVAL;
160
+
aia->nr_guest_bits = *nr;
161
+
} else
162
+
*nr = aia->nr_guest_bits;
163
+
break;
164
+
default:
165
+
return -ENXIO;
166
+
}
167
+
168
+
return 0;
169
+
}
170
+
171
+
static int aia_aplic_addr(struct kvm *kvm, u64 *addr, bool write)
172
+
{
173
+
struct kvm_aia *aia = &kvm->arch.aia;
174
+
175
+
if (write) {
176
+
/* Writes can only be done before irqchip is initialized */
177
+
if (kvm_riscv_aia_initialized(kvm))
178
+
return -EBUSY;
179
+
180
+
if (*addr & (KVM_DEV_RISCV_APLIC_ALIGN - 1))
181
+
return -EINVAL;
182
+
183
+
aia->aplic_addr = *addr;
184
+
} else
185
+
*addr = aia->aplic_addr;
186
+
187
+
return 0;
188
+
}
189
+
190
+
static int aia_imsic_addr(struct kvm *kvm, u64 *addr,
191
+
unsigned long vcpu_idx, bool write)
192
+
{
193
+
struct kvm_vcpu *vcpu;
194
+
struct kvm_vcpu_aia *vcpu_aia;
195
+
196
+
vcpu = kvm_get_vcpu(kvm, vcpu_idx);
197
+
if (!vcpu)
198
+
return -EINVAL;
199
+
vcpu_aia = &vcpu->arch.aia_context;
200
+
201
+
if (write) {
202
+
/* Writes can only be done before irqchip is initialized */
203
+
if (kvm_riscv_aia_initialized(kvm))
204
+
return -EBUSY;
205
+
206
+
if (*addr & (KVM_DEV_RISCV_IMSIC_ALIGN - 1))
207
+
return -EINVAL;
208
+
}
209
+
210
+
mutex_lock(&vcpu->mutex);
211
+
if (write)
212
+
vcpu_aia->imsic_addr = *addr;
213
+
else
214
+
*addr = vcpu_aia->imsic_addr;
215
+
mutex_unlock(&vcpu->mutex);
216
+
217
+
return 0;
218
+
}
219
+
220
+
static gpa_t aia_imsic_ppn(struct kvm_aia *aia, gpa_t addr)
221
+
{
222
+
u32 h, l;
223
+
gpa_t mask = 0;
224
+
225
+
h = aia->nr_hart_bits + aia->nr_guest_bits +
226
+
IMSIC_MMIO_PAGE_SHIFT - 1;
227
+
mask = GENMASK_ULL(h, 0);
228
+
229
+
if (aia->nr_group_bits) {
230
+
h = aia->nr_group_bits + aia->nr_group_shift - 1;
231
+
l = aia->nr_group_shift;
232
+
mask |= GENMASK_ULL(h, l);
233
+
}
234
+
235
+
return (addr & ~mask) >> IMSIC_MMIO_PAGE_SHIFT;
236
+
}
237
+
238
+
static u32 aia_imsic_hart_index(struct kvm_aia *aia, gpa_t addr)
239
+
{
240
+
u32 hart, group = 0;
241
+
242
+
hart = (addr >> (aia->nr_guest_bits + IMSIC_MMIO_PAGE_SHIFT)) &
243
+
GENMASK_ULL(aia->nr_hart_bits - 1, 0);
244
+
if (aia->nr_group_bits)
245
+
group = (addr >> aia->nr_group_shift) &
246
+
GENMASK_ULL(aia->nr_group_bits - 1, 0);
247
+
248
+
return (group << aia->nr_hart_bits) | hart;
249
+
}
250
+
251
+
static int aia_init(struct kvm *kvm)
252
+
{
253
+
int ret, i;
254
+
unsigned long idx;
255
+
struct kvm_vcpu *vcpu;
256
+
struct kvm_vcpu_aia *vaia;
257
+
struct kvm_aia *aia = &kvm->arch.aia;
258
+
gpa_t base_ppn = KVM_RISCV_AIA_UNDEF_ADDR;
259
+
260
+
/* Irqchip can be initialized only once */
261
+
if (kvm_riscv_aia_initialized(kvm))
262
+
return -EBUSY;
263
+
264
+
/* We might be in the middle of creating a VCPU? */
265
+
if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
266
+
return -EBUSY;
267
+
268
+
/* Number of sources should be less than or equals number of IDs */
269
+
if (aia->nr_ids < aia->nr_sources)
270
+
return -EINVAL;
271
+
272
+
/* APLIC base is required for non-zero number of sources */
273
+
if (aia->nr_sources && aia->aplic_addr == KVM_RISCV_AIA_UNDEF_ADDR)
274
+
return -EINVAL;
275
+
276
+
/* Initialize APLIC */
277
+
ret = kvm_riscv_aia_aplic_init(kvm);
278
+
if (ret)
279
+
return ret;
280
+
281
+
/* Iterate over each VCPU */
282
+
kvm_for_each_vcpu(idx, vcpu, kvm) {
283
+
vaia = &vcpu->arch.aia_context;
284
+
285
+
/* IMSIC base is required */
286
+
if (vaia->imsic_addr == KVM_RISCV_AIA_UNDEF_ADDR) {
287
+
ret = -EINVAL;
288
+
goto fail_cleanup_imsics;
289
+
}
290
+
291
+
/* All IMSICs should have matching base PPN */
292
+
if (base_ppn == KVM_RISCV_AIA_UNDEF_ADDR)
293
+
base_ppn = aia_imsic_ppn(aia, vaia->imsic_addr);
294
+
if (base_ppn != aia_imsic_ppn(aia, vaia->imsic_addr)) {
295
+
ret = -EINVAL;
296
+
goto fail_cleanup_imsics;
297
+
}
298
+
299
+
/* Update HART index of the IMSIC based on IMSIC base */
300
+
vaia->hart_index = aia_imsic_hart_index(aia,
301
+
vaia->imsic_addr);
302
+
303
+
/* Initialize IMSIC for this VCPU */
304
+
ret = kvm_riscv_vcpu_aia_imsic_init(vcpu);
305
+
if (ret)
306
+
goto fail_cleanup_imsics;
307
+
}
308
+
309
+
/* Set the initialized flag */
310
+
kvm->arch.aia.initialized = true;
311
+
312
+
return 0;
313
+
314
+
fail_cleanup_imsics:
315
+
for (i = idx - 1; i >= 0; i--) {
316
+
vcpu = kvm_get_vcpu(kvm, i);
317
+
if (!vcpu)
318
+
continue;
319
+
kvm_riscv_vcpu_aia_imsic_cleanup(vcpu);
320
+
}
321
+
kvm_riscv_aia_aplic_cleanup(kvm);
322
+
return ret;
323
+
}
324
+
325
+
static int aia_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
326
+
{
327
+
u32 nr;
328
+
u64 addr;
329
+
int nr_vcpus, r = -ENXIO;
330
+
unsigned long v, type = (unsigned long)attr->attr;
331
+
void __user *uaddr = (void __user *)(long)attr->addr;
332
+
333
+
switch (attr->group) {
334
+
case KVM_DEV_RISCV_AIA_GRP_CONFIG:
335
+
if (copy_from_user(&nr, uaddr, sizeof(nr)))
336
+
return -EFAULT;
337
+
338
+
mutex_lock(&dev->kvm->lock);
339
+
r = aia_config(dev->kvm, type, &nr, true);
340
+
mutex_unlock(&dev->kvm->lock);
341
+
342
+
break;
343
+
344
+
case KVM_DEV_RISCV_AIA_GRP_ADDR:
345
+
if (copy_from_user(&addr, uaddr, sizeof(addr)))
346
+
return -EFAULT;
347
+
348
+
nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
349
+
mutex_lock(&dev->kvm->lock);
350
+
if (type == KVM_DEV_RISCV_AIA_ADDR_APLIC)
351
+
r = aia_aplic_addr(dev->kvm, &addr, true);
352
+
else if (type < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
353
+
r = aia_imsic_addr(dev->kvm, &addr,
354
+
type - KVM_DEV_RISCV_AIA_ADDR_IMSIC(0), true);
355
+
mutex_unlock(&dev->kvm->lock);
356
+
357
+
break;
358
+
359
+
case KVM_DEV_RISCV_AIA_GRP_CTRL:
360
+
switch (type) {
361
+
case KVM_DEV_RISCV_AIA_CTRL_INIT:
362
+
mutex_lock(&dev->kvm->lock);
363
+
r = aia_init(dev->kvm);
364
+
mutex_unlock(&dev->kvm->lock);
365
+
break;
366
+
}
367
+
368
+
break;
369
+
case KVM_DEV_RISCV_AIA_GRP_APLIC:
370
+
if (copy_from_user(&nr, uaddr, sizeof(nr)))
371
+
return -EFAULT;
372
+
373
+
mutex_lock(&dev->kvm->lock);
374
+
r = kvm_riscv_aia_aplic_set_attr(dev->kvm, type, nr);
375
+
mutex_unlock(&dev->kvm->lock);
376
+
377
+
break;
378
+
case KVM_DEV_RISCV_AIA_GRP_IMSIC:
379
+
if (copy_from_user(&v, uaddr, sizeof(v)))
380
+
return -EFAULT;
381
+
382
+
mutex_lock(&dev->kvm->lock);
383
+
r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, true, &v);
384
+
mutex_unlock(&dev->kvm->lock);
385
+
386
+
break;
387
+
}
388
+
389
+
return r;
390
+
}
391
+
392
+
static int aia_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
393
+
{
394
+
u32 nr;
395
+
u64 addr;
396
+
int nr_vcpus, r = -ENXIO;
397
+
void __user *uaddr = (void __user *)(long)attr->addr;
398
+
unsigned long v, type = (unsigned long)attr->attr;
399
+
400
+
switch (attr->group) {
401
+
case KVM_DEV_RISCV_AIA_GRP_CONFIG:
402
+
if (copy_from_user(&nr, uaddr, sizeof(nr)))
403
+
return -EFAULT;
404
+
405
+
mutex_lock(&dev->kvm->lock);
406
+
r = aia_config(dev->kvm, type, &nr, false);
407
+
mutex_unlock(&dev->kvm->lock);
408
+
if (r)
409
+
return r;
410
+
411
+
if (copy_to_user(uaddr, &nr, sizeof(nr)))
412
+
return -EFAULT;
413
+
414
+
break;
415
+
case KVM_DEV_RISCV_AIA_GRP_ADDR:
416
+
if (copy_from_user(&addr, uaddr, sizeof(addr)))
417
+
return -EFAULT;
418
+
419
+
nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
420
+
mutex_lock(&dev->kvm->lock);
421
+
if (type == KVM_DEV_RISCV_AIA_ADDR_APLIC)
422
+
r = aia_aplic_addr(dev->kvm, &addr, false);
423
+
else if (type < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
424
+
r = aia_imsic_addr(dev->kvm, &addr,
425
+
type - KVM_DEV_RISCV_AIA_ADDR_IMSIC(0), false);
426
+
mutex_unlock(&dev->kvm->lock);
427
+
if (r)
428
+
return r;
429
+
430
+
if (copy_to_user(uaddr, &addr, sizeof(addr)))
431
+
return -EFAULT;
432
+
433
+
break;
434
+
case KVM_DEV_RISCV_AIA_GRP_APLIC:
435
+
if (copy_from_user(&nr, uaddr, sizeof(nr)))
436
+
return -EFAULT;
437
+
438
+
mutex_lock(&dev->kvm->lock);
439
+
r = kvm_riscv_aia_aplic_get_attr(dev->kvm, type, &nr);
440
+
mutex_unlock(&dev->kvm->lock);
441
+
if (r)
442
+
return r;
443
+
444
+
if (copy_to_user(uaddr, &nr, sizeof(nr)))
445
+
return -EFAULT;
446
+
447
+
break;
448
+
case KVM_DEV_RISCV_AIA_GRP_IMSIC:
449
+
if (copy_from_user(&v, uaddr, sizeof(v)))
450
+
return -EFAULT;
451
+
452
+
mutex_lock(&dev->kvm->lock);
453
+
r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, false, &v);
454
+
mutex_unlock(&dev->kvm->lock);
455
+
if (r)
456
+
return r;
457
+
458
+
if (copy_to_user(uaddr, &v, sizeof(v)))
459
+
return -EFAULT;
460
+
461
+
break;
462
+
}
463
+
464
+
return r;
465
+
}
466
+
467
+
static int aia_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
468
+
{
469
+
int nr_vcpus;
470
+
471
+
switch (attr->group) {
472
+
case KVM_DEV_RISCV_AIA_GRP_CONFIG:
473
+
switch (attr->attr) {
474
+
case KVM_DEV_RISCV_AIA_CONFIG_MODE:
475
+
case KVM_DEV_RISCV_AIA_CONFIG_IDS:
476
+
case KVM_DEV_RISCV_AIA_CONFIG_SRCS:
477
+
case KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS:
478
+
case KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT:
479
+
case KVM_DEV_RISCV_AIA_CONFIG_HART_BITS:
480
+
case KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS:
481
+
return 0;
482
+
}
483
+
break;
484
+
case KVM_DEV_RISCV_AIA_GRP_ADDR:
485
+
nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
486
+
if (attr->attr == KVM_DEV_RISCV_AIA_ADDR_APLIC)
487
+
return 0;
488
+
else if (attr->attr < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
489
+
return 0;
490
+
break;
491
+
case KVM_DEV_RISCV_AIA_GRP_CTRL:
492
+
switch (attr->attr) {
493
+
case KVM_DEV_RISCV_AIA_CTRL_INIT:
494
+
return 0;
495
+
}
496
+
break;
497
+
case KVM_DEV_RISCV_AIA_GRP_APLIC:
498
+
return kvm_riscv_aia_aplic_has_attr(dev->kvm, attr->attr);
499
+
case KVM_DEV_RISCV_AIA_GRP_IMSIC:
500
+
return kvm_riscv_aia_imsic_has_attr(dev->kvm, attr->attr);
501
+
}
502
+
503
+
return -ENXIO;
504
+
}
505
+
506
+
struct kvm_device_ops kvm_riscv_aia_device_ops = {
507
+
.name = "kvm-riscv-aia",
508
+
.create = aia_create,
509
+
.destroy = aia_destroy,
510
+
.set_attr = aia_set_attr,
511
+
.get_attr = aia_get_attr,
512
+
.has_attr = aia_has_attr,
513
+
};
514
+
515
+
int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu)
516
+
{
517
+
/* Proceed only if AIA was initialized successfully */
518
+
if (!kvm_riscv_aia_initialized(vcpu->kvm))
519
+
return 1;
520
+
521
+
/* Update the IMSIC HW state before entering guest mode */
522
+
return kvm_riscv_vcpu_aia_imsic_update(vcpu);
523
+
}
524
+
525
+
void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu)
526
+
{
527
+
struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
528
+
struct kvm_vcpu_aia_csr *reset_csr =
529
+
&vcpu->arch.aia_context.guest_reset_csr;
530
+
531
+
if (!kvm_riscv_aia_available())
532
+
return;
533
+
memcpy(csr, reset_csr, sizeof(*csr));
534
+
535
+
/* Proceed only if AIA was initialized successfully */
536
+
if (!kvm_riscv_aia_initialized(vcpu->kvm))
537
+
return;
538
+
539
+
/* Reset the IMSIC context */
540
+
kvm_riscv_vcpu_aia_imsic_reset(vcpu);
541
+
}
542
+
543
+
int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu)
544
+
{
545
+
struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context;
546
+
547
+
if (!kvm_riscv_aia_available())
548
+
return 0;
549
+
550
+
/*
551
+
* We don't do any memory allocations over here because these
552
+
* will be done after AIA device is initialized by the user-space.
553
+
*
554
+
* Refer, aia_init() implementation for more details.
555
+
*/
556
+
557
+
/* Initialize default values in AIA vcpu context */
558
+
vaia->imsic_addr = KVM_RISCV_AIA_UNDEF_ADDR;
559
+
vaia->hart_index = vcpu->vcpu_idx;
560
+
561
+
return 0;
562
+
}
563
+
564
+
void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu)
565
+
{
566
+
/* Proceed only if AIA was initialized successfully */
567
+
if (!kvm_riscv_aia_initialized(vcpu->kvm))
568
+
return;
569
+
570
+
/* Cleanup IMSIC context */
571
+
kvm_riscv_vcpu_aia_imsic_cleanup(vcpu);
572
+
}
573
+
574
+
int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index,
575
+
u32 guest_index, u32 iid)
576
+
{
577
+
unsigned long idx;
578
+
struct kvm_vcpu *vcpu;
579
+
580
+
/* Proceed only if AIA was initialized successfully */
581
+
if (!kvm_riscv_aia_initialized(kvm))
582
+
return -EBUSY;
583
+
584
+
/* Inject MSI to matching VCPU */
585
+
kvm_for_each_vcpu(idx, vcpu, kvm) {
586
+
if (vcpu->arch.aia_context.hart_index == hart_index)
587
+
return kvm_riscv_vcpu_aia_imsic_inject(vcpu,
588
+
guest_index,
589
+
0, iid);
590
+
}
591
+
592
+
return 0;
593
+
}
594
+
595
+
int kvm_riscv_aia_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
596
+
{
597
+
gpa_t tppn, ippn;
598
+
unsigned long idx;
599
+
struct kvm_vcpu *vcpu;
600
+
u32 g, toff, iid = msi->data;
601
+
struct kvm_aia *aia = &kvm->arch.aia;
602
+
gpa_t target = (((gpa_t)msi->address_hi) << 32) | msi->address_lo;
603
+
604
+
/* Proceed only if AIA was initialized successfully */
605
+
if (!kvm_riscv_aia_initialized(kvm))
606
+
return -EBUSY;
607
+
608
+
/* Convert target address to target PPN */
609
+
tppn = target >> IMSIC_MMIO_PAGE_SHIFT;
610
+
611
+
/* Extract and clear Guest ID from target PPN */
612
+
g = tppn & (BIT(aia->nr_guest_bits) - 1);
613
+
tppn &= ~((gpa_t)(BIT(aia->nr_guest_bits) - 1));
614
+
615
+
/* Inject MSI to matching VCPU */
616
+
kvm_for_each_vcpu(idx, vcpu, kvm) {
617
+
ippn = vcpu->arch.aia_context.imsic_addr >>
618
+
IMSIC_MMIO_PAGE_SHIFT;
619
+
if (ippn == tppn) {
620
+
toff = target & (IMSIC_MMIO_PAGE_SZ - 1);
621
+
return kvm_riscv_vcpu_aia_imsic_inject(vcpu, g,
622
+
toff, iid);
623
+
}
624
+
}
625
+
626
+
return 0;
627
+
}
628
+
629
+
int kvm_riscv_aia_inject_irq(struct kvm *kvm, unsigned int irq, bool level)
630
+
{
631
+
/* Proceed only if AIA was initialized successfully */
632
+
if (!kvm_riscv_aia_initialized(kvm))
633
+
return -EBUSY;
634
+
635
+
/* Inject interrupt level change in APLIC */
636
+
return kvm_riscv_aia_aplic_inject(kvm, irq, level);
637
+
}
638
+
639
+
void kvm_riscv_aia_init_vm(struct kvm *kvm)
640
+
{
641
+
struct kvm_aia *aia = &kvm->arch.aia;
642
+
643
+
if (!kvm_riscv_aia_available())
644
+
return;
645
+
646
+
/*
647
+
* We don't do any memory allocations over here because these
648
+
* will be done after AIA device is initialized by the user-space.
649
+
*
650
+
* Refer, aia_init() implementation for more details.
651
+
*/
652
+
653
+
/* Initialize default values in AIA global context */
654
+
aia->mode = (kvm_riscv_aia_nr_hgei) ?
655
+
KVM_DEV_RISCV_AIA_MODE_AUTO : KVM_DEV_RISCV_AIA_MODE_EMUL;
656
+
aia->nr_ids = kvm_riscv_aia_max_ids - 1;
657
+
aia->nr_sources = 0;
658
+
aia->nr_group_bits = 0;
659
+
aia->nr_group_shift = KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN;
660
+
aia->nr_hart_bits = 0;
661
+
aia->nr_guest_bits = 0;
662
+
aia->aplic_addr = KVM_RISCV_AIA_UNDEF_ADDR;
663
+
}
664
+
665
+
void kvm_riscv_aia_destroy_vm(struct kvm *kvm)
666
+
{
667
+
/* Proceed only if AIA was initialized successfully */
668
+
if (!kvm_riscv_aia_initialized(kvm))
669
+
return;
670
+
671
+
/* Cleanup APLIC context */
672
+
kvm_riscv_aia_aplic_cleanup(kvm);
673
+
}
+1084
arch/riscv/kvm/aia_imsic.c
+1084
arch/riscv/kvm/aia_imsic.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* Copyright (C) 2021 Western Digital Corporation or its affiliates.
4
+
* Copyright (C) 2022 Ventana Micro Systems Inc.
5
+
*
6
+
* Authors:
7
+
* Anup Patel <apatel@ventanamicro.com>
8
+
*/
9
+
10
+
#include <linux/atomic.h>
11
+
#include <linux/bitmap.h>
12
+
#include <linux/kvm_host.h>
13
+
#include <linux/math.h>
14
+
#include <linux/spinlock.h>
15
+
#include <linux/swab.h>
16
+
#include <kvm/iodev.h>
17
+
#include <asm/csr.h>
18
+
#include <asm/kvm_aia_imsic.h>
19
+
20
+
#define IMSIC_MAX_EIX (IMSIC_MAX_ID / BITS_PER_TYPE(u64))
21
+
22
+
struct imsic_mrif_eix {
23
+
unsigned long eip[BITS_PER_TYPE(u64) / BITS_PER_LONG];
24
+
unsigned long eie[BITS_PER_TYPE(u64) / BITS_PER_LONG];
25
+
};
26
+
27
+
struct imsic_mrif {
28
+
struct imsic_mrif_eix eix[IMSIC_MAX_EIX];
29
+
unsigned long eithreshold;
30
+
unsigned long eidelivery;
31
+
};
32
+
33
+
struct imsic {
34
+
struct kvm_io_device iodev;
35
+
36
+
u32 nr_msis;
37
+
u32 nr_eix;
38
+
u32 nr_hw_eix;
39
+
40
+
/*
41
+
* At any point in time, the register state is in
42
+
* one of the following places:
43
+
*
44
+
* 1) Hardware: IMSIC VS-file (vsfile_cpu >= 0)
45
+
* 2) Software: IMSIC SW-file (vsfile_cpu < 0)
46
+
*/
47
+
48
+
/* IMSIC VS-file */
49
+
rwlock_t vsfile_lock;
50
+
int vsfile_cpu;
51
+
int vsfile_hgei;
52
+
void __iomem *vsfile_va;
53
+
phys_addr_t vsfile_pa;
54
+
55
+
/* IMSIC SW-file */
56
+
struct imsic_mrif *swfile;
57
+
phys_addr_t swfile_pa;
58
+
};
59
+
60
+
#define imsic_vs_csr_read(__c) \
61
+
({ \
62
+
unsigned long __r; \
63
+
csr_write(CSR_VSISELECT, __c); \
64
+
__r = csr_read(CSR_VSIREG); \
65
+
__r; \
66
+
})
67
+
68
+
#define imsic_read_switchcase(__ireg) \
69
+
case __ireg: \
70
+
return imsic_vs_csr_read(__ireg);
71
+
#define imsic_read_switchcase_2(__ireg) \
72
+
imsic_read_switchcase(__ireg + 0) \
73
+
imsic_read_switchcase(__ireg + 1)
74
+
#define imsic_read_switchcase_4(__ireg) \
75
+
imsic_read_switchcase_2(__ireg + 0) \
76
+
imsic_read_switchcase_2(__ireg + 2)
77
+
#define imsic_read_switchcase_8(__ireg) \
78
+
imsic_read_switchcase_4(__ireg + 0) \
79
+
imsic_read_switchcase_4(__ireg + 4)
80
+
#define imsic_read_switchcase_16(__ireg) \
81
+
imsic_read_switchcase_8(__ireg + 0) \
82
+
imsic_read_switchcase_8(__ireg + 8)
83
+
#define imsic_read_switchcase_32(__ireg) \
84
+
imsic_read_switchcase_16(__ireg + 0) \
85
+
imsic_read_switchcase_16(__ireg + 16)
86
+
#define imsic_read_switchcase_64(__ireg) \
87
+
imsic_read_switchcase_32(__ireg + 0) \
88
+
imsic_read_switchcase_32(__ireg + 32)
89
+
90
+
static unsigned long imsic_eix_read(int ireg)
91
+
{
92
+
switch (ireg) {
93
+
imsic_read_switchcase_64(IMSIC_EIP0)
94
+
imsic_read_switchcase_64(IMSIC_EIE0)
95
+
}
96
+
97
+
return 0;
98
+
}
99
+
100
+
#define imsic_vs_csr_swap(__c, __v) \
101
+
({ \
102
+
unsigned long __r; \
103
+
csr_write(CSR_VSISELECT, __c); \
104
+
__r = csr_swap(CSR_VSIREG, __v); \
105
+
__r; \
106
+
})
107
+
108
+
#define imsic_swap_switchcase(__ireg, __v) \
109
+
case __ireg: \
110
+
return imsic_vs_csr_swap(__ireg, __v);
111
+
#define imsic_swap_switchcase_2(__ireg, __v) \
112
+
imsic_swap_switchcase(__ireg + 0, __v) \
113
+
imsic_swap_switchcase(__ireg + 1, __v)
114
+
#define imsic_swap_switchcase_4(__ireg, __v) \
115
+
imsic_swap_switchcase_2(__ireg + 0, __v) \
116
+
imsic_swap_switchcase_2(__ireg + 2, __v)
117
+
#define imsic_swap_switchcase_8(__ireg, __v) \
118
+
imsic_swap_switchcase_4(__ireg + 0, __v) \
119
+
imsic_swap_switchcase_4(__ireg + 4, __v)
120
+
#define imsic_swap_switchcase_16(__ireg, __v) \
121
+
imsic_swap_switchcase_8(__ireg + 0, __v) \
122
+
imsic_swap_switchcase_8(__ireg + 8, __v)
123
+
#define imsic_swap_switchcase_32(__ireg, __v) \
124
+
imsic_swap_switchcase_16(__ireg + 0, __v) \
125
+
imsic_swap_switchcase_16(__ireg + 16, __v)
126
+
#define imsic_swap_switchcase_64(__ireg, __v) \
127
+
imsic_swap_switchcase_32(__ireg + 0, __v) \
128
+
imsic_swap_switchcase_32(__ireg + 32, __v)
129
+
130
+
static unsigned long imsic_eix_swap(int ireg, unsigned long val)
131
+
{
132
+
switch (ireg) {
133
+
imsic_swap_switchcase_64(IMSIC_EIP0, val)
134
+
imsic_swap_switchcase_64(IMSIC_EIE0, val)
135
+
}
136
+
137
+
return 0;
138
+
}
139
+
140
+
#define imsic_vs_csr_write(__c, __v) \
141
+
do { \
142
+
csr_write(CSR_VSISELECT, __c); \
143
+
csr_write(CSR_VSIREG, __v); \
144
+
} while (0)
145
+
146
+
#define imsic_write_switchcase(__ireg, __v) \
147
+
case __ireg: \
148
+
imsic_vs_csr_write(__ireg, __v); \
149
+
break;
150
+
#define imsic_write_switchcase_2(__ireg, __v) \
151
+
imsic_write_switchcase(__ireg + 0, __v) \
152
+
imsic_write_switchcase(__ireg + 1, __v)
153
+
#define imsic_write_switchcase_4(__ireg, __v) \
154
+
imsic_write_switchcase_2(__ireg + 0, __v) \
155
+
imsic_write_switchcase_2(__ireg + 2, __v)
156
+
#define imsic_write_switchcase_8(__ireg, __v) \
157
+
imsic_write_switchcase_4(__ireg + 0, __v) \
158
+
imsic_write_switchcase_4(__ireg + 4, __v)
159
+
#define imsic_write_switchcase_16(__ireg, __v) \
160
+
imsic_write_switchcase_8(__ireg + 0, __v) \
161
+
imsic_write_switchcase_8(__ireg + 8, __v)
162
+
#define imsic_write_switchcase_32(__ireg, __v) \
163
+
imsic_write_switchcase_16(__ireg + 0, __v) \
164
+
imsic_write_switchcase_16(__ireg + 16, __v)
165
+
#define imsic_write_switchcase_64(__ireg, __v) \
166
+
imsic_write_switchcase_32(__ireg + 0, __v) \
167
+
imsic_write_switchcase_32(__ireg + 32, __v)
168
+
169
+
static void imsic_eix_write(int ireg, unsigned long val)
170
+
{
171
+
switch (ireg) {
172
+
imsic_write_switchcase_64(IMSIC_EIP0, val)
173
+
imsic_write_switchcase_64(IMSIC_EIE0, val)
174
+
}
175
+
}
176
+
177
+
#define imsic_vs_csr_set(__c, __v) \
178
+
do { \
179
+
csr_write(CSR_VSISELECT, __c); \
180
+
csr_set(CSR_VSIREG, __v); \
181
+
} while (0)
182
+
183
+
#define imsic_set_switchcase(__ireg, __v) \
184
+
case __ireg: \
185
+
imsic_vs_csr_set(__ireg, __v); \
186
+
break;
187
+
#define imsic_set_switchcase_2(__ireg, __v) \
188
+
imsic_set_switchcase(__ireg + 0, __v) \
189
+
imsic_set_switchcase(__ireg + 1, __v)
190
+
#define imsic_set_switchcase_4(__ireg, __v) \
191
+
imsic_set_switchcase_2(__ireg + 0, __v) \
192
+
imsic_set_switchcase_2(__ireg + 2, __v)
193
+
#define imsic_set_switchcase_8(__ireg, __v) \
194
+
imsic_set_switchcase_4(__ireg + 0, __v) \
195
+
imsic_set_switchcase_4(__ireg + 4, __v)
196
+
#define imsic_set_switchcase_16(__ireg, __v) \
197
+
imsic_set_switchcase_8(__ireg + 0, __v) \
198
+
imsic_set_switchcase_8(__ireg + 8, __v)
199
+
#define imsic_set_switchcase_32(__ireg, __v) \
200
+
imsic_set_switchcase_16(__ireg + 0, __v) \
201
+
imsic_set_switchcase_16(__ireg + 16, __v)
202
+
#define imsic_set_switchcase_64(__ireg, __v) \
203
+
imsic_set_switchcase_32(__ireg + 0, __v) \
204
+
imsic_set_switchcase_32(__ireg + 32, __v)
205
+
206
+
static void imsic_eix_set(int ireg, unsigned long val)
207
+
{
208
+
switch (ireg) {
209
+
imsic_set_switchcase_64(IMSIC_EIP0, val)
210
+
imsic_set_switchcase_64(IMSIC_EIE0, val)
211
+
}
212
+
}
213
+
214
+
static unsigned long imsic_mrif_atomic_rmw(struct imsic_mrif *mrif,
215
+
unsigned long *ptr,
216
+
unsigned long new_val,
217
+
unsigned long wr_mask)
218
+
{
219
+
unsigned long old_val = 0, tmp = 0;
220
+
221
+
__asm__ __volatile__ (
222
+
"0: lr.w.aq %1, %0\n"
223
+
" and %2, %1, %3\n"
224
+
" or %2, %2, %4\n"
225
+
" sc.w.rl %2, %2, %0\n"
226
+
" bnez %2, 0b"
227
+
: "+A" (*ptr), "+r" (old_val), "+r" (tmp)
228
+
: "r" (~wr_mask), "r" (new_val & wr_mask)
229
+
: "memory");
230
+
231
+
return old_val;
232
+
}
233
+
234
+
static unsigned long imsic_mrif_atomic_or(struct imsic_mrif *mrif,
235
+
unsigned long *ptr,
236
+
unsigned long val)
237
+
{
238
+
return atomic_long_fetch_or(val, (atomic_long_t *)ptr);
239
+
}
240
+
241
+
#define imsic_mrif_atomic_write(__mrif, __ptr, __new_val) \
242
+
imsic_mrif_atomic_rmw(__mrif, __ptr, __new_val, -1UL)
243
+
#define imsic_mrif_atomic_read(__mrif, __ptr) \
244
+
imsic_mrif_atomic_or(__mrif, __ptr, 0)
245
+
246
+
static u32 imsic_mrif_topei(struct imsic_mrif *mrif, u32 nr_eix, u32 nr_msis)
247
+
{
248
+
struct imsic_mrif_eix *eix;
249
+
u32 i, imin, imax, ei, max_msi;
250
+
unsigned long eipend[BITS_PER_TYPE(u64) / BITS_PER_LONG];
251
+
unsigned long eithreshold = imsic_mrif_atomic_read(mrif,
252
+
&mrif->eithreshold);
253
+
254
+
max_msi = (eithreshold && (eithreshold <= nr_msis)) ?
255
+
eithreshold : nr_msis;
256
+
for (ei = 0; ei < nr_eix; ei++) {
257
+
eix = &mrif->eix[ei];
258
+
eipend[0] = imsic_mrif_atomic_read(mrif, &eix->eie[0]) &
259
+
imsic_mrif_atomic_read(mrif, &eix->eip[0]);
260
+
#ifdef CONFIG_32BIT
261
+
eipend[1] = imsic_mrif_atomic_read(mrif, &eix->eie[1]) &
262
+
imsic_mrif_atomic_read(mrif, &eix->eip[1]);
263
+
if (!eipend[0] && !eipend[1])
264
+
#else
265
+
if (!eipend[0])
266
+
#endif
267
+
continue;
268
+
269
+
imin = ei * BITS_PER_TYPE(u64);
270
+
imax = ((imin + BITS_PER_TYPE(u64)) < max_msi) ?
271
+
imin + BITS_PER_TYPE(u64) : max_msi;
272
+
for (i = (!imin) ? 1 : imin; i < imax; i++) {
273
+
if (test_bit(i - imin, eipend))
274
+
return (i << TOPEI_ID_SHIFT) | i;
275
+
}
276
+
}
277
+
278
+
return 0;
279
+
}
280
+
281
+
static int imsic_mrif_isel_check(u32 nr_eix, unsigned long isel)
282
+
{
283
+
u32 num = 0;
284
+
285
+
switch (isel) {
286
+
case IMSIC_EIDELIVERY:
287
+
case IMSIC_EITHRESHOLD:
288
+
break;
289
+
case IMSIC_EIP0 ... IMSIC_EIP63:
290
+
num = isel - IMSIC_EIP0;
291
+
break;
292
+
case IMSIC_EIE0 ... IMSIC_EIE63:
293
+
num = isel - IMSIC_EIE0;
294
+
break;
295
+
default:
296
+
return -ENOENT;
297
+
}
298
+
#ifndef CONFIG_32BIT
299
+
if (num & 0x1)
300
+
return -EINVAL;
301
+
#endif
302
+
if ((num / 2) >= nr_eix)
303
+
return -EINVAL;
304
+
305
+
return 0;
306
+
}
307
+
308
+
static int imsic_mrif_rmw(struct imsic_mrif *mrif, u32 nr_eix,
309
+
unsigned long isel, unsigned long *val,
310
+
unsigned long new_val, unsigned long wr_mask)
311
+
{
312
+
bool pend;
313
+
struct imsic_mrif_eix *eix;
314
+
unsigned long *ei, num, old_val = 0;
315
+
316
+
switch (isel) {
317
+
case IMSIC_EIDELIVERY:
318
+
old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eidelivery,
319
+
new_val, wr_mask & 0x1);
320
+
break;
321
+
case IMSIC_EITHRESHOLD:
322
+
old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eithreshold,
323
+
new_val, wr_mask & (IMSIC_MAX_ID - 1));
324
+
break;
325
+
case IMSIC_EIP0 ... IMSIC_EIP63:
326
+
case IMSIC_EIE0 ... IMSIC_EIE63:
327
+
if (isel >= IMSIC_EIP0 && isel <= IMSIC_EIP63) {
328
+
pend = true;
329
+
num = isel - IMSIC_EIP0;
330
+
} else {
331
+
pend = false;
332
+
num = isel - IMSIC_EIE0;
333
+
}
334
+
335
+
if ((num / 2) >= nr_eix)
336
+
return -EINVAL;
337
+
eix = &mrif->eix[num / 2];
338
+
339
+
#ifndef CONFIG_32BIT
340
+
if (num & 0x1)
341
+
return -EINVAL;
342
+
ei = (pend) ? &eix->eip[0] : &eix->eie[0];
343
+
#else
344
+
ei = (pend) ? &eix->eip[num & 0x1] : &eix->eie[num & 0x1];
345
+
#endif
346
+
347
+
/* Bit0 of EIP0 or EIE0 is read-only */
348
+
if (!num)
349
+
wr_mask &= ~BIT(0);
350
+
351
+
old_val = imsic_mrif_atomic_rmw(mrif, ei, new_val, wr_mask);
352
+
break;
353
+
default:
354
+
return -ENOENT;
355
+
}
356
+
357
+
if (val)
358
+
*val = old_val;
359
+
360
+
return 0;
361
+
}
362
+
363
+
struct imsic_vsfile_read_data {
364
+
int hgei;
365
+
u32 nr_eix;
366
+
bool clear;
367
+
struct imsic_mrif *mrif;
368
+
};
369
+
370
+
static void imsic_vsfile_local_read(void *data)
371
+
{
372
+
u32 i;
373
+
struct imsic_mrif_eix *eix;
374
+
struct imsic_vsfile_read_data *idata = data;
375
+
struct imsic_mrif *mrif = idata->mrif;
376
+
unsigned long new_hstatus, old_hstatus, old_vsiselect;
377
+
378
+
old_vsiselect = csr_read(CSR_VSISELECT);
379
+
old_hstatus = csr_read(CSR_HSTATUS);
380
+
new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
381
+
new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT;
382
+
csr_write(CSR_HSTATUS, new_hstatus);
383
+
384
+
/*
385
+
* We don't use imsic_mrif_atomic_xyz() functions to store
386
+
* values in MRIF because imsic_vsfile_read() is always called
387
+
* with pointer to temporary MRIF on stack.
388
+
*/
389
+
390
+
if (idata->clear) {
391
+
mrif->eidelivery = imsic_vs_csr_swap(IMSIC_EIDELIVERY, 0);
392
+
mrif->eithreshold = imsic_vs_csr_swap(IMSIC_EITHRESHOLD, 0);
393
+
for (i = 0; i < idata->nr_eix; i++) {
394
+
eix = &mrif->eix[i];
395
+
eix->eip[0] = imsic_eix_swap(IMSIC_EIP0 + i * 2, 0);
396
+
eix->eie[0] = imsic_eix_swap(IMSIC_EIE0 + i * 2, 0);
397
+
#ifdef CONFIG_32BIT
398
+
eix->eip[1] = imsic_eix_swap(IMSIC_EIP0 + i * 2 + 1, 0);
399
+
eix->eie[1] = imsic_eix_swap(IMSIC_EIE0 + i * 2 + 1, 0);
400
+
#endif
401
+
}
402
+
} else {
403
+
mrif->eidelivery = imsic_vs_csr_read(IMSIC_EIDELIVERY);
404
+
mrif->eithreshold = imsic_vs_csr_read(IMSIC_EITHRESHOLD);
405
+
for (i = 0; i < idata->nr_eix; i++) {
406
+
eix = &mrif->eix[i];
407
+
eix->eip[0] = imsic_eix_read(IMSIC_EIP0 + i * 2);
408
+
eix->eie[0] = imsic_eix_read(IMSIC_EIE0 + i * 2);
409
+
#ifdef CONFIG_32BIT
410
+
eix->eip[1] = imsic_eix_read(IMSIC_EIP0 + i * 2 + 1);
411
+
eix->eie[1] = imsic_eix_read(IMSIC_EIE0 + i * 2 + 1);
412
+
#endif
413
+
}
414
+
}
415
+
416
+
csr_write(CSR_HSTATUS, old_hstatus);
417
+
csr_write(CSR_VSISELECT, old_vsiselect);
418
+
}
419
+
420
+
static void imsic_vsfile_read(int vsfile_hgei, int vsfile_cpu, u32 nr_eix,
421
+
bool clear, struct imsic_mrif *mrif)
422
+
{
423
+
struct imsic_vsfile_read_data idata;
424
+
425
+
/* We can only read clear if we have a IMSIC VS-file */
426
+
if (vsfile_cpu < 0 || vsfile_hgei <= 0)
427
+
return;
428
+
429
+
/* We can only read clear on local CPU */
430
+
idata.hgei = vsfile_hgei;
431
+
idata.nr_eix = nr_eix;
432
+
idata.clear = clear;
433
+
idata.mrif = mrif;
434
+
on_each_cpu_mask(cpumask_of(vsfile_cpu),
435
+
imsic_vsfile_local_read, &idata, 1);
436
+
}
437
+
438
+
struct imsic_vsfile_rw_data {
439
+
int hgei;
440
+
int isel;
441
+
bool write;
442
+
unsigned long val;
443
+
};
444
+
445
+
static void imsic_vsfile_local_rw(void *data)
446
+
{
447
+
struct imsic_vsfile_rw_data *idata = data;
448
+
unsigned long new_hstatus, old_hstatus, old_vsiselect;
449
+
450
+
old_vsiselect = csr_read(CSR_VSISELECT);
451
+
old_hstatus = csr_read(CSR_HSTATUS);
452
+
new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
453
+
new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT;
454
+
csr_write(CSR_HSTATUS, new_hstatus);
455
+
456
+
switch (idata->isel) {
457
+
case IMSIC_EIDELIVERY:
458
+
if (idata->write)
459
+
imsic_vs_csr_write(IMSIC_EIDELIVERY, idata->val);
460
+
else
461
+
idata->val = imsic_vs_csr_read(IMSIC_EIDELIVERY);
462
+
break;
463
+
case IMSIC_EITHRESHOLD:
464
+
if (idata->write)
465
+
imsic_vs_csr_write(IMSIC_EITHRESHOLD, idata->val);
466
+
else
467
+
idata->val = imsic_vs_csr_read(IMSIC_EITHRESHOLD);
468
+
break;
469
+
case IMSIC_EIP0 ... IMSIC_EIP63:
470
+
case IMSIC_EIE0 ... IMSIC_EIE63:
471
+
#ifndef CONFIG_32BIT
472
+
if (idata->isel & 0x1)
473
+
break;
474
+
#endif
475
+
if (idata->write)
476
+
imsic_eix_write(idata->isel, idata->val);
477
+
else
478
+
idata->val = imsic_eix_read(idata->isel);
479
+
break;
480
+
default:
481
+
break;
482
+
}
483
+
484
+
csr_write(CSR_HSTATUS, old_hstatus);
485
+
csr_write(CSR_VSISELECT, old_vsiselect);
486
+
}
487
+
488
+
static int imsic_vsfile_rw(int vsfile_hgei, int vsfile_cpu, u32 nr_eix,
489
+
unsigned long isel, bool write,
490
+
unsigned long *val)
491
+
{
492
+
int rc;
493
+
struct imsic_vsfile_rw_data rdata;
494
+
495
+
/* We can only access register if we have a IMSIC VS-file */
496
+
if (vsfile_cpu < 0 || vsfile_hgei <= 0)
497
+
return -EINVAL;
498
+
499
+
/* Check IMSIC register iselect */
500
+
rc = imsic_mrif_isel_check(nr_eix, isel);
501
+
if (rc)
502
+
return rc;
503
+
504
+
/* We can only access register on local CPU */
505
+
rdata.hgei = vsfile_hgei;
506
+
rdata.isel = isel;
507
+
rdata.write = write;
508
+
rdata.val = (write) ? *val : 0;
509
+
on_each_cpu_mask(cpumask_of(vsfile_cpu),
510
+
imsic_vsfile_local_rw, &rdata, 1);
511
+
512
+
if (!write)
513
+
*val = rdata.val;
514
+
515
+
return 0;
516
+
}
517
+
518
+
static void imsic_vsfile_local_clear(int vsfile_hgei, u32 nr_eix)
519
+
{
520
+
u32 i;
521
+
unsigned long new_hstatus, old_hstatus, old_vsiselect;
522
+
523
+
/* We can only zero-out if we have a IMSIC VS-file */
524
+
if (vsfile_hgei <= 0)
525
+
return;
526
+
527
+
old_vsiselect = csr_read(CSR_VSISELECT);
528
+
old_hstatus = csr_read(CSR_HSTATUS);
529
+
new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
530
+
new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT;
531
+
csr_write(CSR_HSTATUS, new_hstatus);
532
+
533
+
imsic_vs_csr_write(IMSIC_EIDELIVERY, 0);
534
+
imsic_vs_csr_write(IMSIC_EITHRESHOLD, 0);
535
+
for (i = 0; i < nr_eix; i++) {
536
+
imsic_eix_write(IMSIC_EIP0 + i * 2, 0);
537
+
imsic_eix_write(IMSIC_EIE0 + i * 2, 0);
538
+
#ifdef CONFIG_32BIT
539
+
imsic_eix_write(IMSIC_EIP0 + i * 2 + 1, 0);
540
+
imsic_eix_write(IMSIC_EIE0 + i * 2 + 1, 0);
541
+
#endif
542
+
}
543
+
544
+
csr_write(CSR_HSTATUS, old_hstatus);
545
+
csr_write(CSR_VSISELECT, old_vsiselect);
546
+
}
547
+
548
+
static void imsic_vsfile_local_update(int vsfile_hgei, u32 nr_eix,
549
+
struct imsic_mrif *mrif)
550
+
{
551
+
u32 i;
552
+
struct imsic_mrif_eix *eix;
553
+
unsigned long new_hstatus, old_hstatus, old_vsiselect;
554
+
555
+
/* We can only update if we have a HW IMSIC context */
556
+
if (vsfile_hgei <= 0)
557
+
return;
558
+
559
+
/*
560
+
* We don't use imsic_mrif_atomic_xyz() functions to read values
561
+
* from MRIF in this function because it is always called with
562
+
* pointer to temporary MRIF on stack.
563
+
*/
564
+
565
+
old_vsiselect = csr_read(CSR_VSISELECT);
566
+
old_hstatus = csr_read(CSR_HSTATUS);
567
+
new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
568
+
new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT;
569
+
csr_write(CSR_HSTATUS, new_hstatus);
570
+
571
+
for (i = 0; i < nr_eix; i++) {
572
+
eix = &mrif->eix[i];
573
+
imsic_eix_set(IMSIC_EIP0 + i * 2, eix->eip[0]);
574
+
imsic_eix_set(IMSIC_EIE0 + i * 2, eix->eie[0]);
575
+
#ifdef CONFIG_32BIT
576
+
imsic_eix_set(IMSIC_EIP0 + i * 2 + 1, eix->eip[1]);
577
+
imsic_eix_set(IMSIC_EIE0 + i * 2 + 1, eix->eie[1]);
578
+
#endif
579
+
}
580
+
imsic_vs_csr_write(IMSIC_EITHRESHOLD, mrif->eithreshold);
581
+
imsic_vs_csr_write(IMSIC_EIDELIVERY, mrif->eidelivery);
582
+
583
+
csr_write(CSR_HSTATUS, old_hstatus);
584
+
csr_write(CSR_VSISELECT, old_vsiselect);
585
+
}
586
+
587
+
static void imsic_vsfile_cleanup(struct imsic *imsic)
588
+
{
589
+
int old_vsfile_hgei, old_vsfile_cpu;
590
+
unsigned long flags;
591
+
592
+
/*
593
+
* We don't use imsic_mrif_atomic_xyz() functions to clear the
594
+
* SW-file in this function because it is always called when the
595
+
* VCPU is being destroyed.
596
+
*/
597
+
598
+
write_lock_irqsave(&imsic->vsfile_lock, flags);
599
+
old_vsfile_hgei = imsic->vsfile_hgei;
600
+
old_vsfile_cpu = imsic->vsfile_cpu;
601
+
imsic->vsfile_cpu = imsic->vsfile_hgei = -1;
602
+
imsic->vsfile_va = NULL;
603
+
imsic->vsfile_pa = 0;
604
+
write_unlock_irqrestore(&imsic->vsfile_lock, flags);
605
+
606
+
memset(imsic->swfile, 0, sizeof(*imsic->swfile));
607
+
608
+
if (old_vsfile_cpu >= 0)
609
+
kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei);
610
+
}
611
+
612
+
static void imsic_swfile_extirq_update(struct kvm_vcpu *vcpu)
613
+
{
614
+
struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
615
+
struct imsic_mrif *mrif = imsic->swfile;
616
+
617
+
if (imsic_mrif_atomic_read(mrif, &mrif->eidelivery) &&
618
+
imsic_mrif_topei(mrif, imsic->nr_eix, imsic->nr_msis))
619
+
kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT);
620
+
else
621
+
kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
622
+
}
623
+
624
+
static void imsic_swfile_read(struct kvm_vcpu *vcpu, bool clear,
625
+
struct imsic_mrif *mrif)
626
+
{
627
+
struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
628
+
629
+
/*
630
+
* We don't use imsic_mrif_atomic_xyz() functions to read and
631
+
* write SW-file and MRIF in this function because it is always
632
+
* called when VCPU is not using SW-file and the MRIF points to
633
+
* a temporary MRIF on stack.
634
+
*/
635
+
636
+
memcpy(mrif, imsic->swfile, sizeof(*mrif));
637
+
if (clear) {
638
+
memset(imsic->swfile, 0, sizeof(*imsic->swfile));
639
+
kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
640
+
}
641
+
}
642
+
643
+
static void imsic_swfile_update(struct kvm_vcpu *vcpu,
644
+
struct imsic_mrif *mrif)
645
+
{
646
+
u32 i;
647
+
struct imsic_mrif_eix *seix, *eix;
648
+
struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
649
+
struct imsic_mrif *smrif = imsic->swfile;
650
+
651
+
imsic_mrif_atomic_write(smrif, &smrif->eidelivery, mrif->eidelivery);
652
+
imsic_mrif_atomic_write(smrif, &smrif->eithreshold, mrif->eithreshold);
653
+
for (i = 0; i < imsic->nr_eix; i++) {
654
+
seix = &smrif->eix[i];
655
+
eix = &mrif->eix[i];
656
+
imsic_mrif_atomic_or(smrif, &seix->eip[0], eix->eip[0]);
657
+
imsic_mrif_atomic_or(smrif, &seix->eie[0], eix->eie[0]);
658
+
#ifdef CONFIG_32BIT
659
+
imsic_mrif_atomic_or(smrif, &seix->eip[1], eix->eip[1]);
660
+
imsic_mrif_atomic_or(smrif, &seix->eie[1], eix->eie[1]);
661
+
#endif
662
+
}
663
+
664
+
imsic_swfile_extirq_update(vcpu);
665
+
}
666
+
667
+
void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu)
668
+
{
669
+
unsigned long flags;
670
+
struct imsic_mrif tmrif;
671
+
int old_vsfile_hgei, old_vsfile_cpu;
672
+
struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
673
+
674
+
/* Read and clear IMSIC VS-file details */
675
+
write_lock_irqsave(&imsic->vsfile_lock, flags);
676
+
old_vsfile_hgei = imsic->vsfile_hgei;
677
+
old_vsfile_cpu = imsic->vsfile_cpu;
678
+
imsic->vsfile_cpu = imsic->vsfile_hgei = -1;
679
+
imsic->vsfile_va = NULL;
680
+
imsic->vsfile_pa = 0;
681
+
write_unlock_irqrestore(&imsic->vsfile_lock, flags);
682
+
683
+
/* Do nothing, if no IMSIC VS-file to release */
684
+
if (old_vsfile_cpu < 0)
685
+
return;
686
+
687
+
/*
688
+
* At this point, all interrupt producers are still using
689
+
* the old IMSIC VS-file so we first re-direct all interrupt
690
+
* producers.
691
+
*/
692
+
693
+
/* Purge the G-stage mapping */
694
+
kvm_riscv_gstage_iounmap(vcpu->kvm,
695
+
vcpu->arch.aia_context.imsic_addr,
696
+
IMSIC_MMIO_PAGE_SZ);
697
+
698
+
/* TODO: Purge the IOMMU mapping ??? */
699
+
700
+
/*
701
+
* At this point, all interrupt producers have been re-directed
702
+
* to somewhere else so we move register state from the old IMSIC
703
+
* VS-file to the IMSIC SW-file.
704
+
*/
705
+
706
+
/* Read and clear register state from old IMSIC VS-file */
707
+
memset(&tmrif, 0, sizeof(tmrif));
708
+
imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu, imsic->nr_hw_eix,
709
+
true, &tmrif);
710
+
711
+
/* Update register state in IMSIC SW-file */
712
+
imsic_swfile_update(vcpu, &tmrif);
713
+
714
+
/* Free-up old IMSIC VS-file */
715
+
kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei);
716
+
}
717
+
718
+
int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu)
719
+
{
720
+
unsigned long flags;
721
+
phys_addr_t new_vsfile_pa;
722
+
struct imsic_mrif tmrif;
723
+
void __iomem *new_vsfile_va;
724
+
struct kvm *kvm = vcpu->kvm;
725
+
struct kvm_run *run = vcpu->run;
726
+
struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context;
727
+
struct imsic *imsic = vaia->imsic_state;
728
+
int ret = 0, new_vsfile_hgei = -1, old_vsfile_hgei, old_vsfile_cpu;
729
+
730
+
/* Do nothing for emulation mode */
731
+
if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_EMUL)
732
+
return 1;
733
+
734
+
/* Read old IMSIC VS-file details */
735
+
read_lock_irqsave(&imsic->vsfile_lock, flags);
736
+
old_vsfile_hgei = imsic->vsfile_hgei;
737
+
old_vsfile_cpu = imsic->vsfile_cpu;
738
+
read_unlock_irqrestore(&imsic->vsfile_lock, flags);
739
+
740
+
/* Do nothing if we are continuing on same CPU */
741
+
if (old_vsfile_cpu == vcpu->cpu)
742
+
return 1;
743
+
744
+
/* Allocate new IMSIC VS-file */
745
+
ret = kvm_riscv_aia_alloc_hgei(vcpu->cpu, vcpu,
746
+
&new_vsfile_va, &new_vsfile_pa);
747
+
if (ret <= 0) {
748
+
/* For HW acceleration mode, we can't continue */
749
+
if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_HWACCEL) {
750
+
run->fail_entry.hardware_entry_failure_reason =
751
+
CSR_HSTATUS;
752
+
run->fail_entry.cpu = vcpu->cpu;
753
+
run->exit_reason = KVM_EXIT_FAIL_ENTRY;
754
+
return 0;
755
+
}
756
+
757
+
/* Release old IMSIC VS-file */
758
+
if (old_vsfile_cpu >= 0)
759
+
kvm_riscv_vcpu_aia_imsic_release(vcpu);
760
+
761
+
/* For automatic mode, we continue */
762
+
goto done;
763
+
}
764
+
new_vsfile_hgei = ret;
765
+
766
+
/*
767
+
* At this point, all interrupt producers are still using
768
+
* to the old IMSIC VS-file so we first move all interrupt
769
+
* producers to the new IMSIC VS-file.
770
+
*/
771
+
772
+
/* Zero-out new IMSIC VS-file */
773
+
imsic_vsfile_local_clear(new_vsfile_hgei, imsic->nr_hw_eix);
774
+
775
+
/* Update G-stage mapping for the new IMSIC VS-file */
776
+
ret = kvm_riscv_gstage_ioremap(kvm, vcpu->arch.aia_context.imsic_addr,
777
+
new_vsfile_pa, IMSIC_MMIO_PAGE_SZ,
778
+
true, true);
779
+
if (ret)
780
+
goto fail_free_vsfile_hgei;
781
+
782
+
/* TODO: Update the IOMMU mapping ??? */
783
+
784
+
/* Update new IMSIC VS-file details in IMSIC context */
785
+
write_lock_irqsave(&imsic->vsfile_lock, flags);
786
+
imsic->vsfile_hgei = new_vsfile_hgei;
787
+
imsic->vsfile_cpu = vcpu->cpu;
788
+
imsic->vsfile_va = new_vsfile_va;
789
+
imsic->vsfile_pa = new_vsfile_pa;
790
+
write_unlock_irqrestore(&imsic->vsfile_lock, flags);
791
+
792
+
/*
793
+
* At this point, all interrupt producers have been moved
794
+
* to the new IMSIC VS-file so we move register state from
795
+
* the old IMSIC VS/SW-file to the new IMSIC VS-file.
796
+
*/
797
+
798
+
memset(&tmrif, 0, sizeof(tmrif));
799
+
if (old_vsfile_cpu >= 0) {
800
+
/* Read and clear register state from old IMSIC VS-file */
801
+
imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu,
802
+
imsic->nr_hw_eix, true, &tmrif);
803
+
804
+
/* Free-up old IMSIC VS-file */
805
+
kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei);
806
+
} else {
807
+
/* Read and clear register state from IMSIC SW-file */
808
+
imsic_swfile_read(vcpu, true, &tmrif);
809
+
}
810
+
811
+
/* Restore register state in the new IMSIC VS-file */
812
+
imsic_vsfile_local_update(new_vsfile_hgei, imsic->nr_hw_eix, &tmrif);
813
+
814
+
done:
815
+
/* Set VCPU HSTATUS.VGEIN to new IMSIC VS-file */
816
+
vcpu->arch.guest_context.hstatus &= ~HSTATUS_VGEIN;
817
+
if (new_vsfile_hgei > 0)
818
+
vcpu->arch.guest_context.hstatus |=
819
+
((unsigned long)new_vsfile_hgei) << HSTATUS_VGEIN_SHIFT;
820
+
821
+
/* Continue run-loop */
822
+
return 1;
823
+
824
+
fail_free_vsfile_hgei:
825
+
kvm_riscv_aia_free_hgei(vcpu->cpu, new_vsfile_hgei);
826
+
return ret;
827
+
}
828
+
829
+
int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, unsigned long isel,
830
+
unsigned long *val, unsigned long new_val,
831
+
unsigned long wr_mask)
832
+
{
833
+
u32 topei;
834
+
struct imsic_mrif_eix *eix;
835
+
int r, rc = KVM_INSN_CONTINUE_NEXT_SEPC;
836
+
struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
837
+
838
+
if (isel == KVM_RISCV_AIA_IMSIC_TOPEI) {
839
+
/* Read pending and enabled interrupt with highest priority */
840
+
topei = imsic_mrif_topei(imsic->swfile, imsic->nr_eix,
841
+
imsic->nr_msis);
842
+
if (val)
843
+
*val = topei;
844
+
845
+
/* Writes ignore value and clear top pending interrupt */
846
+
if (topei && wr_mask) {
847
+
topei >>= TOPEI_ID_SHIFT;
848
+
if (topei) {
849
+
eix = &imsic->swfile->eix[topei /
850
+
BITS_PER_TYPE(u64)];
851
+
clear_bit(topei & (BITS_PER_TYPE(u64) - 1),
852
+
eix->eip);
853
+
}
854
+
}
855
+
} else {
856
+
r = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, isel,
857
+
val, new_val, wr_mask);
858
+
/* Forward unknown IMSIC register to user-space */
859
+
if (r)
860
+
rc = (r == -ENOENT) ? 0 : KVM_INSN_ILLEGAL_TRAP;
861
+
}
862
+
863
+
if (wr_mask)
864
+
imsic_swfile_extirq_update(vcpu);
865
+
866
+
return rc;
867
+
}
868
+
869
+
int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type,
870
+
bool write, unsigned long *val)
871
+
{
872
+
u32 isel, vcpu_id;
873
+
unsigned long flags;
874
+
struct imsic *imsic;
875
+
struct kvm_vcpu *vcpu;
876
+
int rc, vsfile_hgei, vsfile_cpu;
877
+
878
+
if (!kvm_riscv_aia_initialized(kvm))
879
+
return -ENODEV;
880
+
881
+
vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type);
882
+
vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
883
+
if (!vcpu)
884
+
return -ENODEV;
885
+
886
+
isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type);
887
+
imsic = vcpu->arch.aia_context.imsic_state;
888
+
889
+
read_lock_irqsave(&imsic->vsfile_lock, flags);
890
+
891
+
rc = 0;
892
+
vsfile_hgei = imsic->vsfile_hgei;
893
+
vsfile_cpu = imsic->vsfile_cpu;
894
+
if (vsfile_cpu < 0) {
895
+
if (write) {
896
+
rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix,
897
+
isel, NULL, *val, -1UL);
898
+
imsic_swfile_extirq_update(vcpu);
899
+
} else
900
+
rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix,
901
+
isel, val, 0, 0);
902
+
}
903
+
904
+
read_unlock_irqrestore(&imsic->vsfile_lock, flags);
905
+
906
+
if (!rc && vsfile_cpu >= 0)
907
+
rc = imsic_vsfile_rw(vsfile_hgei, vsfile_cpu, imsic->nr_eix,
908
+
isel, write, val);
909
+
910
+
return rc;
911
+
}
912
+
913
+
int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type)
914
+
{
915
+
u32 isel, vcpu_id;
916
+
struct imsic *imsic;
917
+
struct kvm_vcpu *vcpu;
918
+
919
+
if (!kvm_riscv_aia_initialized(kvm))
920
+
return -ENODEV;
921
+
922
+
vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type);
923
+
vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
924
+
if (!vcpu)
925
+
return -ENODEV;
926
+
927
+
isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type);
928
+
imsic = vcpu->arch.aia_context.imsic_state;
929
+
return imsic_mrif_isel_check(imsic->nr_eix, isel);
930
+
}
931
+
932
+
void kvm_riscv_vcpu_aia_imsic_reset(struct kvm_vcpu *vcpu)
933
+
{
934
+
struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
935
+
936
+
if (!imsic)
937
+
return;
938
+
939
+
kvm_riscv_vcpu_aia_imsic_release(vcpu);
940
+
941
+
memset(imsic->swfile, 0, sizeof(*imsic->swfile));
942
+
}
943
+
944
+
int kvm_riscv_vcpu_aia_imsic_inject(struct kvm_vcpu *vcpu,
945
+
u32 guest_index, u32 offset, u32 iid)
946
+
{
947
+
unsigned long flags;
948
+
struct imsic_mrif_eix *eix;
949
+
struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
950
+
951
+
/* We only emulate one IMSIC MMIO page for each Guest VCPU */
952
+
if (!imsic || !iid || guest_index ||
953
+
(offset != IMSIC_MMIO_SETIPNUM_LE &&
954
+
offset != IMSIC_MMIO_SETIPNUM_BE))
955
+
return -ENODEV;
956
+
957
+
iid = (offset == IMSIC_MMIO_SETIPNUM_BE) ? __swab32(iid) : iid;
958
+
if (imsic->nr_msis <= iid)
959
+
return -EINVAL;
960
+
961
+
read_lock_irqsave(&imsic->vsfile_lock, flags);
962
+
963
+
if (imsic->vsfile_cpu >= 0) {
964
+
writel(iid, imsic->vsfile_va + IMSIC_MMIO_SETIPNUM_LE);
965
+
kvm_vcpu_kick(vcpu);
966
+
} else {
967
+
eix = &imsic->swfile->eix[iid / BITS_PER_TYPE(u64)];
968
+
set_bit(iid & (BITS_PER_TYPE(u64) - 1), eix->eip);
969
+
imsic_swfile_extirq_update(vcpu);
970
+
}
971
+
972
+
read_unlock_irqrestore(&imsic->vsfile_lock, flags);
973
+
974
+
return 0;
975
+
}
976
+
977
+
static int imsic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
978
+
gpa_t addr, int len, void *val)
979
+
{
980
+
if (len != 4 || (addr & 0x3) != 0)
981
+
return -EOPNOTSUPP;
982
+
983
+
*((u32 *)val) = 0;
984
+
985
+
return 0;
986
+
}
987
+
988
+
static int imsic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
989
+
gpa_t addr, int len, const void *val)
990
+
{
991
+
struct kvm_msi msi = { 0 };
992
+
993
+
if (len != 4 || (addr & 0x3) != 0)
994
+
return -EOPNOTSUPP;
995
+
996
+
msi.address_hi = addr >> 32;
997
+
msi.address_lo = (u32)addr;
998
+
msi.data = *((const u32 *)val);
999
+
kvm_riscv_aia_inject_msi(vcpu->kvm, &msi);
1000
+
1001
+
return 0;
1002
+
};
1003
+
1004
+
static struct kvm_io_device_ops imsic_iodoev_ops = {
1005
+
.read = imsic_mmio_read,
1006
+
.write = imsic_mmio_write,
1007
+
};
1008
+
1009
+
int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu)
1010
+
{
1011
+
int ret = 0;
1012
+
struct imsic *imsic;
1013
+
struct page *swfile_page;
1014
+
struct kvm *kvm = vcpu->kvm;
1015
+
1016
+
/* Fail if we have zero IDs */
1017
+
if (!kvm->arch.aia.nr_ids)
1018
+
return -EINVAL;
1019
+
1020
+
/* Allocate IMSIC context */
1021
+
imsic = kzalloc(sizeof(*imsic), GFP_KERNEL);
1022
+
if (!imsic)
1023
+
return -ENOMEM;
1024
+
vcpu->arch.aia_context.imsic_state = imsic;
1025
+
1026
+
/* Setup IMSIC context */
1027
+
imsic->nr_msis = kvm->arch.aia.nr_ids + 1;
1028
+
rwlock_init(&imsic->vsfile_lock);
1029
+
imsic->nr_eix = BITS_TO_U64(imsic->nr_msis);
1030
+
imsic->nr_hw_eix = BITS_TO_U64(kvm_riscv_aia_max_ids);
1031
+
imsic->vsfile_hgei = imsic->vsfile_cpu = -1;
1032
+
1033
+
/* Setup IMSIC SW-file */
1034
+
swfile_page = alloc_pages(GFP_KERNEL | __GFP_ZERO,
1035
+
get_order(sizeof(*imsic->swfile)));
1036
+
if (!swfile_page) {
1037
+
ret = -ENOMEM;
1038
+
goto fail_free_imsic;
1039
+
}
1040
+
imsic->swfile = page_to_virt(swfile_page);
1041
+
imsic->swfile_pa = page_to_phys(swfile_page);
1042
+
1043
+
/* Setup IO device */
1044
+
kvm_iodevice_init(&imsic->iodev, &imsic_iodoev_ops);
1045
+
mutex_lock(&kvm->slots_lock);
1046
+
ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
1047
+
vcpu->arch.aia_context.imsic_addr,
1048
+
KVM_DEV_RISCV_IMSIC_SIZE,
1049
+
&imsic->iodev);
1050
+
mutex_unlock(&kvm->slots_lock);
1051
+
if (ret)
1052
+
goto fail_free_swfile;
1053
+
1054
+
return 0;
1055
+
1056
+
fail_free_swfile:
1057
+
free_pages((unsigned long)imsic->swfile,
1058
+
get_order(sizeof(*imsic->swfile)));
1059
+
fail_free_imsic:
1060
+
vcpu->arch.aia_context.imsic_state = NULL;
1061
+
kfree(imsic);
1062
+
return ret;
1063
+
}
1064
+
1065
+
void kvm_riscv_vcpu_aia_imsic_cleanup(struct kvm_vcpu *vcpu)
1066
+
{
1067
+
struct kvm *kvm = vcpu->kvm;
1068
+
struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
1069
+
1070
+
if (!imsic)
1071
+
return;
1072
+
1073
+
imsic_vsfile_cleanup(imsic);
1074
+
1075
+
mutex_lock(&kvm->slots_lock);
1076
+
kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &imsic->iodev);
1077
+
mutex_unlock(&kvm->slots_lock);
1078
+
1079
+
free_pages((unsigned long)imsic->swfile,
1080
+
get_order(sizeof(*imsic->swfile)));
1081
+
1082
+
vcpu->arch.aia_context.imsic_state = NULL;
1083
+
kfree(imsic);
1084
+
}
+2
-1
arch/riscv/kvm/main.c
+2
-1
arch/riscv/kvm/main.c
···
116
116
kvm_info("VMID %ld bits available\n", kvm_riscv_gstage_vmid_bits());
117
117
118
118
if (kvm_riscv_aia_available())
119
-
kvm_info("AIA available\n");
119
+
kvm_info("AIA available with %d guest external interrupts\n",
120
+
kvm_riscv_aia_nr_hgei);
120
121
121
122
rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
122
123
if (rc) {
+1
-1
arch/riscv/kvm/tlb.c
+1
-1
arch/riscv/kvm/tlb.c
···
296
296
unsigned int actual_req = req;
297
297
DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS);
298
298
299
-
bitmap_clear(vcpu_mask, 0, KVM_MAX_VCPUS);
299
+
bitmap_zero(vcpu_mask, KVM_MAX_VCPUS);
300
300
kvm_for_each_vcpu(i, vcpu, kvm) {
301
301
if (hbase != -1UL) {
302
302
if (vcpu->vcpu_id < hbase)
+4
arch/riscv/kvm/vcpu.c
+4
arch/riscv/kvm/vcpu.c
···
61
61
KVM_ISA_EXT_ARR(SSAIA),
62
62
KVM_ISA_EXT_ARR(SSTC),
63
63
KVM_ISA_EXT_ARR(SVINVAL),
64
+
KVM_ISA_EXT_ARR(SVNAPOT),
64
65
KVM_ISA_EXT_ARR(SVPBMT),
65
66
KVM_ISA_EXT_ARR(ZBB),
66
67
KVM_ISA_EXT_ARR(ZIHINTPAUSE),
···
103
102
case KVM_RISCV_ISA_EXT_SSAIA:
104
103
case KVM_RISCV_ISA_EXT_SSTC:
105
104
case KVM_RISCV_ISA_EXT_SVINVAL:
105
+
case KVM_RISCV_ISA_EXT_SVNAPOT:
106
106
case KVM_RISCV_ISA_EXT_ZIHINTPAUSE:
107
107
case KVM_RISCV_ISA_EXT_ZBB:
108
108
return false;
···
252
250
253
251
void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
254
252
{
253
+
kvm_riscv_aia_wakeon_hgei(vcpu, true);
255
254
}
256
255
257
256
void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
258
257
{
258
+
kvm_riscv_aia_wakeon_hgei(vcpu, false);
259
259
}
260
260
261
261
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
+2
arch/riscv/kvm/vcpu_exit.c
+2
arch/riscv/kvm/vcpu_exit.c
···
183
183
run->exit_reason = KVM_EXIT_UNKNOWN;
184
184
switch (trap->scause) {
185
185
case EXC_INST_ILLEGAL:
186
+
case EXC_LOAD_MISALIGNED:
187
+
case EXC_STORE_MISALIGNED:
186
188
if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) {
187
189
kvm_riscv_vcpu_trap_redirect(vcpu, trap);
188
190
ret = 1;
+54
-26
arch/riscv/kvm/vcpu_sbi.c
+54
-26
arch/riscv/kvm/vcpu_sbi.c
···
20
20
};
21
21
#endif
22
22
23
-
#ifdef CONFIG_RISCV_PMU_SBI
24
-
extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_pmu;
25
-
#else
23
+
#ifndef CONFIG_RISCV_PMU_SBI
26
24
static const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_pmu = {
27
25
.extid_start = -1UL,
28
26
.extid_end = -1UL,
···
29
31
#endif
30
32
31
33
struct kvm_riscv_sbi_extension_entry {
32
-
enum KVM_RISCV_SBI_EXT_ID dis_idx;
34
+
enum KVM_RISCV_SBI_EXT_ID ext_idx;
33
35
const struct kvm_vcpu_sbi_extension *ext_ptr;
34
36
};
35
37
36
38
static const struct kvm_riscv_sbi_extension_entry sbi_ext[] = {
37
39
{
38
-
.dis_idx = KVM_RISCV_SBI_EXT_V01,
40
+
.ext_idx = KVM_RISCV_SBI_EXT_V01,
39
41
.ext_ptr = &vcpu_sbi_ext_v01,
40
42
},
41
43
{
42
-
.dis_idx = KVM_RISCV_SBI_EXT_MAX, /* Can't be disabled */
44
+
.ext_idx = KVM_RISCV_SBI_EXT_MAX, /* Can't be disabled */
43
45
.ext_ptr = &vcpu_sbi_ext_base,
44
46
},
45
47
{
46
-
.dis_idx = KVM_RISCV_SBI_EXT_TIME,
48
+
.ext_idx = KVM_RISCV_SBI_EXT_TIME,
47
49
.ext_ptr = &vcpu_sbi_ext_time,
48
50
},
49
51
{
50
-
.dis_idx = KVM_RISCV_SBI_EXT_IPI,
52
+
.ext_idx = KVM_RISCV_SBI_EXT_IPI,
51
53
.ext_ptr = &vcpu_sbi_ext_ipi,
52
54
},
53
55
{
54
-
.dis_idx = KVM_RISCV_SBI_EXT_RFENCE,
56
+
.ext_idx = KVM_RISCV_SBI_EXT_RFENCE,
55
57
.ext_ptr = &vcpu_sbi_ext_rfence,
56
58
},
57
59
{
58
-
.dis_idx = KVM_RISCV_SBI_EXT_SRST,
60
+
.ext_idx = KVM_RISCV_SBI_EXT_SRST,
59
61
.ext_ptr = &vcpu_sbi_ext_srst,
60
62
},
61
63
{
62
-
.dis_idx = KVM_RISCV_SBI_EXT_HSM,
64
+
.ext_idx = KVM_RISCV_SBI_EXT_HSM,
63
65
.ext_ptr = &vcpu_sbi_ext_hsm,
64
66
},
65
67
{
66
-
.dis_idx = KVM_RISCV_SBI_EXT_PMU,
68
+
.ext_idx = KVM_RISCV_SBI_EXT_PMU,
67
69
.ext_ptr = &vcpu_sbi_ext_pmu,
68
70
},
69
71
{
70
-
.dis_idx = KVM_RISCV_SBI_EXT_EXPERIMENTAL,
72
+
.ext_idx = KVM_RISCV_SBI_EXT_EXPERIMENTAL,
71
73
.ext_ptr = &vcpu_sbi_ext_experimental,
72
74
},
73
75
{
74
-
.dis_idx = KVM_RISCV_SBI_EXT_VENDOR,
76
+
.ext_idx = KVM_RISCV_SBI_EXT_VENDOR,
75
77
.ext_ptr = &vcpu_sbi_ext_vendor,
76
78
},
77
79
};
···
145
147
return -EINVAL;
146
148
147
149
for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) {
148
-
if (sbi_ext[i].dis_idx == reg_num) {
150
+
if (sbi_ext[i].ext_idx == reg_num) {
149
151
sext = &sbi_ext[i];
150
152
break;
151
153
}
···
153
155
if (!sext)
154
156
return -ENOENT;
155
157
156
-
scontext->extension_disabled[sext->dis_idx] = !reg_val;
158
+
/*
159
+
* We can't set the extension status to available here, since it may
160
+
* have a probe() function which needs to confirm availability first,
161
+
* but it may be too early to call that here. We can set the status to
162
+
* unavailable, though.
163
+
*/
164
+
if (!reg_val)
165
+
scontext->ext_status[sext->ext_idx] =
166
+
KVM_RISCV_SBI_EXT_UNAVAILABLE;
157
167
158
168
return 0;
159
169
}
···
178
172
return -EINVAL;
179
173
180
174
for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) {
181
-
if (sbi_ext[i].dis_idx == reg_num) {
175
+
if (sbi_ext[i].ext_idx == reg_num) {
182
176
sext = &sbi_ext[i];
183
177
break;
184
178
}
···
186
180
if (!sext)
187
181
return -ENOENT;
188
182
189
-
*reg_val = !scontext->extension_disabled[sext->dis_idx];
183
+
/*
184
+
* If the extension status is still uninitialized, then we should probe
185
+
* to determine if it's available, but it may be too early to do that
186
+
* here. The best we can do is report that the extension has not been
187
+
* disabled, i.e. we return 1 when the extension is available and also
188
+
* when it only may be available.
189
+
*/
190
+
*reg_val = scontext->ext_status[sext->ext_idx] !=
191
+
KVM_RISCV_SBI_EXT_UNAVAILABLE;
190
192
191
193
return 0;
192
194
}
···
321
307
const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext(
322
308
struct kvm_vcpu *vcpu, unsigned long extid)
323
309
{
324
-
int i;
325
-
const struct kvm_riscv_sbi_extension_entry *sext;
326
310
struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context;
311
+
const struct kvm_riscv_sbi_extension_entry *entry;
312
+
const struct kvm_vcpu_sbi_extension *ext;
313
+
int i;
327
314
328
315
for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) {
329
-
sext = &sbi_ext[i];
330
-
if (sext->ext_ptr->extid_start <= extid &&
331
-
sext->ext_ptr->extid_end >= extid) {
332
-
if (sext->dis_idx < KVM_RISCV_SBI_EXT_MAX &&
333
-
scontext->extension_disabled[sext->dis_idx])
316
+
entry = &sbi_ext[i];
317
+
ext = entry->ext_ptr;
318
+
319
+
if (ext->extid_start <= extid && ext->extid_end >= extid) {
320
+
if (entry->ext_idx >= KVM_RISCV_SBI_EXT_MAX ||
321
+
scontext->ext_status[entry->ext_idx] ==
322
+
KVM_RISCV_SBI_EXT_AVAILABLE)
323
+
return ext;
324
+
if (scontext->ext_status[entry->ext_idx] ==
325
+
KVM_RISCV_SBI_EXT_UNAVAILABLE)
334
326
return NULL;
335
-
return sbi_ext[i].ext_ptr;
327
+
if (ext->probe && !ext->probe(vcpu)) {
328
+
scontext->ext_status[entry->ext_idx] =
329
+
KVM_RISCV_SBI_EXT_UNAVAILABLE;
330
+
return NULL;
331
+
}
332
+
333
+
scontext->ext_status[entry->ext_idx] =
334
+
KVM_RISCV_SBI_EXT_AVAILABLE;
335
+
return ext;
336
336
}
337
337
}
338
338
+118
arch/riscv/kvm/vm.c
+118
arch/riscv/kvm/vm.c
···
55
55
kvm_riscv_aia_destroy_vm(kvm);
56
56
}
57
57
58
+
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irql,
59
+
bool line_status)
60
+
{
61
+
if (!irqchip_in_kernel(kvm))
62
+
return -ENXIO;
63
+
64
+
return kvm_riscv_aia_inject_irq(kvm, irql->irq, irql->level);
65
+
}
66
+
67
+
int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
68
+
struct kvm *kvm, int irq_source_id,
69
+
int level, bool line_status)
70
+
{
71
+
struct kvm_msi msi;
72
+
73
+
if (!level)
74
+
return -1;
75
+
76
+
msi.address_lo = e->msi.address_lo;
77
+
msi.address_hi = e->msi.address_hi;
78
+
msi.data = e->msi.data;
79
+
msi.flags = e->msi.flags;
80
+
msi.devid = e->msi.devid;
81
+
82
+
return kvm_riscv_aia_inject_msi(kvm, &msi);
83
+
}
84
+
85
+
static int kvm_riscv_set_irq(struct kvm_kernel_irq_routing_entry *e,
86
+
struct kvm *kvm, int irq_source_id,
87
+
int level, bool line_status)
88
+
{
89
+
return kvm_riscv_aia_inject_irq(kvm, e->irqchip.pin, level);
90
+
}
91
+
92
+
int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines)
93
+
{
94
+
struct kvm_irq_routing_entry *ents;
95
+
int i, rc;
96
+
97
+
ents = kcalloc(lines, sizeof(*ents), GFP_KERNEL);
98
+
if (!ents)
99
+
return -ENOMEM;
100
+
101
+
for (i = 0; i < lines; i++) {
102
+
ents[i].gsi = i;
103
+
ents[i].type = KVM_IRQ_ROUTING_IRQCHIP;
104
+
ents[i].u.irqchip.irqchip = 0;
105
+
ents[i].u.irqchip.pin = i;
106
+
}
107
+
rc = kvm_set_irq_routing(kvm, ents, lines, 0);
108
+
kfree(ents);
109
+
110
+
return rc;
111
+
}
112
+
113
+
bool kvm_arch_can_set_irq_routing(struct kvm *kvm)
114
+
{
115
+
return irqchip_in_kernel(kvm);
116
+
}
117
+
118
+
int kvm_set_routing_entry(struct kvm *kvm,
119
+
struct kvm_kernel_irq_routing_entry *e,
120
+
const struct kvm_irq_routing_entry *ue)
121
+
{
122
+
int r = -EINVAL;
123
+
124
+
switch (ue->type) {
125
+
case KVM_IRQ_ROUTING_IRQCHIP:
126
+
e->set = kvm_riscv_set_irq;
127
+
e->irqchip.irqchip = ue->u.irqchip.irqchip;
128
+
e->irqchip.pin = ue->u.irqchip.pin;
129
+
if ((e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS) ||
130
+
(e->irqchip.irqchip >= KVM_NR_IRQCHIPS))
131
+
goto out;
132
+
break;
133
+
case KVM_IRQ_ROUTING_MSI:
134
+
e->set = kvm_set_msi;
135
+
e->msi.address_lo = ue->u.msi.address_lo;
136
+
e->msi.address_hi = ue->u.msi.address_hi;
137
+
e->msi.data = ue->u.msi.data;
138
+
e->msi.flags = ue->flags;
139
+
e->msi.devid = ue->u.msi.devid;
140
+
break;
141
+
default:
142
+
goto out;
143
+
}
144
+
r = 0;
145
+
out:
146
+
return r;
147
+
}
148
+
149
+
int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
150
+
struct kvm *kvm, int irq_source_id, int level,
151
+
bool line_status)
152
+
{
153
+
if (!level)
154
+
return -EWOULDBLOCK;
155
+
156
+
switch (e->type) {
157
+
case KVM_IRQ_ROUTING_MSI:
158
+
return kvm_set_msi(e, kvm, irq_source_id, level, line_status);
159
+
160
+
case KVM_IRQ_ROUTING_IRQCHIP:
161
+
return kvm_riscv_set_irq(e, kvm, irq_source_id,
162
+
level, line_status);
163
+
}
164
+
165
+
return -EWOULDBLOCK;
166
+
}
167
+
168
+
bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
169
+
{
170
+
return irqchip_in_kernel(kvm);
171
+
}
172
+
58
173
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
59
174
{
60
175
int r;
61
176
62
177
switch (ext) {
178
+
case KVM_CAP_IRQCHIP:
179
+
r = kvm_riscv_aia_available();
180
+
break;
63
181
case KVM_CAP_IOEVENTFD:
64
182
case KVM_CAP_DEVICE_CTRL:
65
183
case KVM_CAP_USER_MEMORY:
+2
include/uapi/linux/kvm.h
+2
include/uapi/linux/kvm.h
···
1442
1442
#define KVM_DEV_TYPE_XIVE KVM_DEV_TYPE_XIVE
1443
1443
KVM_DEV_TYPE_ARM_PV_TIME,
1444
1444
#define KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_ARM_PV_TIME
1445
+
KVM_DEV_TYPE_RISCV_AIA,
1446
+
#define KVM_DEV_TYPE_RISCV_AIA KVM_DEV_TYPE_RISCV_AIA
1445
1447
KVM_DEV_TYPE_MAX,
1446
1448
};
1447
1449