Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'kvmarm-6.17' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD
KVM/arm64 changes for 6.17, round #1
- Host driver for GICv5, the next generation interrupt controller for
arm64, including support for interrupt routing, MSIs, interrupt
translation and wired interrupts.
- Use FEAT_GCIE_LEGACY on GICv5 systems to virtualize GICv3 VMs on
GICv5 hardware, leveraging the legacy VGIC interface.
- Userspace control of the 'nASSGIcap' GICv3 feature, allowing
userspace to disable support for SGIs w/o an active state on hardware
that previously advertised it unconditionally.
- Map supporting endpoints with cacheable memory attributes on systems
with FEAT_S2FWB and DIC where KVM no longer needs to perform cache
maintenance on the address range.
- Nested support for FEAT_RAS and FEAT_DoubleFault2, allowing the guest
hypervisor to inject external aborts into an L2 VM and take traps of
masked external aborts to the hypervisor.
- Convert more system register sanitization to the config-driven
implementation.
- Fixes to the visibility of EL2 registers, namely making VGICv3 system
registers accessible through the VGIC device instead of the ONE_REG
vCPU ioctls.
- Various cleanups and minor fixes.
+7589
-681
+41
Documentation/arch/arm64/booting.rst
+41
Documentation/arch/arm64/booting.rst
···
223
223
224
224
- SCR_EL3.HCE (bit 8) must be initialised to 0b1.
225
225
226
+
For systems with a GICv5 interrupt controller to be used in v5 mode:
227
+
228
+
- If the kernel is entered at EL1 and EL2 is present:
229
+
230
+
- ICH_HFGRTR_EL2.ICC_PPI_ACTIVERn_EL1 (bit 20) must be initialised to 0b1.
231
+
- ICH_HFGRTR_EL2.ICC_PPI_PRIORITYRn_EL1 (bit 19) must be initialised to 0b1.
232
+
- ICH_HFGRTR_EL2.ICC_PPI_PENDRn_EL1 (bit 18) must be initialised to 0b1.
233
+
- ICH_HFGRTR_EL2.ICC_PPI_ENABLERn_EL1 (bit 17) must be initialised to 0b1.
234
+
- ICH_HFGRTR_EL2.ICC_PPI_HMRn_EL1 (bit 16) must be initialised to 0b1.
235
+
- ICH_HFGRTR_EL2.ICC_IAFFIDR_EL1 (bit 7) must be initialised to 0b1.
236
+
- ICH_HFGRTR_EL2.ICC_ICSR_EL1 (bit 6) must be initialised to 0b1.
237
+
- ICH_HFGRTR_EL2.ICC_PCR_EL1 (bit 5) must be initialised to 0b1.
238
+
- ICH_HFGRTR_EL2.ICC_HPPIR_EL1 (bit 4) must be initialised to 0b1.
239
+
- ICH_HFGRTR_EL2.ICC_HAPR_EL1 (bit 3) must be initialised to 0b1.
240
+
- ICH_HFGRTR_EL2.ICC_CR0_EL1 (bit 2) must be initialised to 0b1.
241
+
- ICH_HFGRTR_EL2.ICC_IDRn_EL1 (bit 1) must be initialised to 0b1.
242
+
- ICH_HFGRTR_EL2.ICC_APR_EL1 (bit 0) must be initialised to 0b1.
243
+
244
+
- ICH_HFGWTR_EL2.ICC_PPI_ACTIVERn_EL1 (bit 20) must be initialised to 0b1.
245
+
- ICH_HFGWTR_EL2.ICC_PPI_PRIORITYRn_EL1 (bit 19) must be initialised to 0b1.
246
+
- ICH_HFGWTR_EL2.ICC_PPI_PENDRn_EL1 (bit 18) must be initialised to 0b1.
247
+
- ICH_HFGWTR_EL2.ICC_PPI_ENABLERn_EL1 (bit 17) must be initialised to 0b1.
248
+
- ICH_HFGWTR_EL2.ICC_ICSR_EL1 (bit 6) must be initialised to 0b1.
249
+
- ICH_HFGWTR_EL2.ICC_PCR_EL1 (bit 5) must be initialised to 0b1.
250
+
- ICH_HFGWTR_EL2.ICC_CR0_EL1 (bit 2) must be initialised to 0b1.
251
+
- ICH_HFGWTR_EL2.ICC_APR_EL1 (bit 0) must be initialised to 0b1.
252
+
253
+
- ICH_HFGITR_EL2.GICRCDNMIA (bit 10) must be initialised to 0b1.
254
+
- ICH_HFGITR_EL2.GICRCDIA (bit 9) must be initialised to 0b1.
255
+
- ICH_HFGITR_EL2.GICCDDI (bit 8) must be initialised to 0b1.
256
+
- ICH_HFGITR_EL2.GICCDEOI (bit 7) must be initialised to 0b1.
257
+
- ICH_HFGITR_EL2.GICCDHM (bit 6) must be initialised to 0b1.
258
+
- ICH_HFGITR_EL2.GICCDRCFG (bit 5) must be initialised to 0b1.
259
+
- ICH_HFGITR_EL2.GICCDPEND (bit 4) must be initialised to 0b1.
260
+
- ICH_HFGITR_EL2.GICCDAFF (bit 3) must be initialised to 0b1.
261
+
- ICH_HFGITR_EL2.GICCDPRI (bit 2) must be initialised to 0b1.
262
+
- ICH_HFGITR_EL2.GICCDDIS (bit 1) must be initialised to 0b1.
263
+
- ICH_HFGITR_EL2.GICCDEN (bit 0) must be initialised to 0b1.
264
+
265
+
- The DT or ACPI tables must describe a GICv5 interrupt controller.
266
+
226
267
For systems with a GICv3 interrupt controller to be used in v3 mode:
227
268
- If EL3 is present:
228
269
+78
Documentation/devicetree/bindings/interrupt-controller/arm,gic-v5-iwb.yaml
+78
Documentation/devicetree/bindings/interrupt-controller/arm,gic-v5-iwb.yaml
···
1
+
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2
+
%YAML 1.2
3
+
---
4
+
$id: http://devicetree.org/schemas/interrupt-controller/arm,gic-v5-iwb.yaml#
5
+
$schema: http://devicetree.org/meta-schemas/core.yaml#
6
+
7
+
title: ARM Generic Interrupt Controller, version 5 Interrupt Wire Bridge (IWB)
8
+
9
+
maintainers:
10
+
- Lorenzo Pieralisi <lpieralisi@kernel.org>
11
+
- Marc Zyngier <maz@kernel.org>
12
+
13
+
description: |
14
+
The GICv5 architecture defines the guidelines to implement GICv5
15
+
compliant interrupt controllers for AArch64 systems.
16
+
17
+
The GICv5 specification can be found at
18
+
https://developer.arm.com/documentation/aes0070
19
+
20
+
GICv5 has zero or more Interrupt Wire Bridges (IWB) that are responsible
21
+
for translating wire signals into interrupt messages to the GICv5 ITS.
22
+
23
+
allOf:
24
+
- $ref: /schemas/interrupt-controller.yaml#
25
+
26
+
properties:
27
+
compatible:
28
+
const: arm,gic-v5-iwb
29
+
30
+
reg:
31
+
items:
32
+
- description: IWB control frame
33
+
34
+
"#address-cells":
35
+
const: 0
36
+
37
+
"#interrupt-cells":
38
+
description: |
39
+
The 1st cell corresponds to the IWB wire.
40
+
41
+
The 2nd cell is the flags, encoded as follows:
42
+
bits[3:0] trigger type and level flags.
43
+
44
+
1 = low-to-high edge triggered
45
+
2 = high-to-low edge triggered
46
+
4 = active high level-sensitive
47
+
8 = active low level-sensitive
48
+
49
+
const: 2
50
+
51
+
interrupt-controller: true
52
+
53
+
msi-parent:
54
+
maxItems: 1
55
+
56
+
required:
57
+
- compatible
58
+
- reg
59
+
- "#interrupt-cells"
60
+
- interrupt-controller
61
+
- msi-parent
62
+
63
+
additionalProperties: false
64
+
65
+
examples:
66
+
- |
67
+
interrupt-controller@2f000000 {
68
+
compatible = "arm,gic-v5-iwb";
69
+
reg = <0x2f000000 0x10000>;
70
+
71
+
#address-cells = <0>;
72
+
73
+
#interrupt-cells = <2>;
74
+
interrupt-controller;
75
+
76
+
msi-parent = <&its0 64>;
77
+
};
78
+
...
+267
Documentation/devicetree/bindings/interrupt-controller/arm,gic-v5.yaml
+267
Documentation/devicetree/bindings/interrupt-controller/arm,gic-v5.yaml
···
1
+
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2
+
%YAML 1.2
3
+
---
4
+
$id: http://devicetree.org/schemas/interrupt-controller/arm,gic-v5.yaml#
5
+
$schema: http://devicetree.org/meta-schemas/core.yaml#
6
+
7
+
title: ARM Generic Interrupt Controller, version 5
8
+
9
+
maintainers:
10
+
- Lorenzo Pieralisi <lpieralisi@kernel.org>
11
+
- Marc Zyngier <maz@kernel.org>
12
+
13
+
description: |
14
+
The GICv5 architecture defines the guidelines to implement GICv5
15
+
compliant interrupt controllers for AArch64 systems.
16
+
17
+
The GICv5 specification can be found at
18
+
https://developer.arm.com/documentation/aes0070
19
+
20
+
The GICv5 architecture is composed of multiple components:
21
+
- one or more IRS (Interrupt Routing Service)
22
+
- zero or more ITS (Interrupt Translation Service)
23
+
24
+
The architecture defines:
25
+
- PE-Private Peripheral Interrupts (PPI)
26
+
- Shared Peripheral Interrupts (SPI)
27
+
- Logical Peripheral Interrupts (LPI)
28
+
29
+
allOf:
30
+
- $ref: /schemas/interrupt-controller.yaml#
31
+
32
+
properties:
33
+
compatible:
34
+
const: arm,gic-v5
35
+
36
+
"#address-cells":
37
+
enum: [ 1, 2 ]
38
+
39
+
"#size-cells":
40
+
enum: [ 1, 2 ]
41
+
42
+
ranges: true
43
+
44
+
"#interrupt-cells":
45
+
description: |
46
+
The 1st cell corresponds to the INTID.Type field in the INTID; 1 for PPI,
47
+
3 for SPI. LPI interrupts must not be described in the bindings since
48
+
they are allocated dynamically by the software component managing them.
49
+
50
+
The 2nd cell contains the interrupt INTID.ID field.
51
+
52
+
The 3rd cell is the flags, encoded as follows:
53
+
bits[3:0] trigger type and level flags.
54
+
55
+
1 = low-to-high edge triggered
56
+
2 = high-to-low edge triggered
57
+
4 = active high level-sensitive
58
+
8 = active low level-sensitive
59
+
60
+
const: 3
61
+
62
+
interrupt-controller: true
63
+
64
+
interrupts:
65
+
description:
66
+
The VGIC maintenance interrupt.
67
+
maxItems: 1
68
+
69
+
required:
70
+
- compatible
71
+
- "#address-cells"
72
+
- "#size-cells"
73
+
- ranges
74
+
- "#interrupt-cells"
75
+
- interrupt-controller
76
+
77
+
patternProperties:
78
+
"^irs@[0-9a-f]+$":
79
+
type: object
80
+
description:
81
+
GICv5 has one or more Interrupt Routing Services (IRS) that are
82
+
responsible for handling IRQ state and routing.
83
+
84
+
additionalProperties: false
85
+
86
+
properties:
87
+
compatible:
88
+
const: arm,gic-v5-irs
89
+
90
+
reg:
91
+
minItems: 1
92
+
items:
93
+
- description: IRS config frames
94
+
- description: IRS setlpi frames
95
+
96
+
reg-names:
97
+
description:
98
+
Describe config and setlpi frames that are present.
99
+
"ns-" stands for non-secure, "s-" for secure, "realm-" for realm
100
+
and "el3-" for EL3.
101
+
minItems: 1
102
+
maxItems: 8
103
+
items:
104
+
enum: [ ns-config, s-config, realm-config, el3-config, ns-setlpi,
105
+
s-setlpi, realm-setlpi, el3-setlpi ]
106
+
107
+
"#address-cells":
108
+
enum: [ 1, 2 ]
109
+
110
+
"#size-cells":
111
+
enum: [ 1, 2 ]
112
+
113
+
ranges: true
114
+
115
+
dma-noncoherent:
116
+
description:
117
+
Present if the GIC IRS permits programming shareability and
118
+
cacheability attributes but is connected to a non-coherent
119
+
downstream interconnect.
120
+
121
+
cpus:
122
+
description:
123
+
CPUs managed by the IRS.
124
+
125
+
arm,iaffids:
126
+
$ref: /schemas/types.yaml#/definitions/uint16-array
127
+
description:
128
+
Interrupt AFFinity ID (IAFFID) associated with the CPU whose
129
+
CPU node phandle is at the same index in the cpus array.
130
+
131
+
patternProperties:
132
+
"^its@[0-9a-f]+$":
133
+
type: object
134
+
description:
135
+
GICv5 has zero or more Interrupt Translation Services (ITS) that are
136
+
used to route Message Signalled Interrupts (MSI) to the CPUs. Each
137
+
ITS is connected to an IRS.
138
+
additionalProperties: false
139
+
140
+
properties:
141
+
compatible:
142
+
const: arm,gic-v5-its
143
+
144
+
reg:
145
+
items:
146
+
- description: ITS config frames
147
+
148
+
reg-names:
149
+
description:
150
+
Describe config frames that are present.
151
+
"ns-" stands for non-secure, "s-" for secure, "realm-" for realm
152
+
and "el3-" for EL3.
153
+
minItems: 1
154
+
maxItems: 4
155
+
items:
156
+
enum: [ ns-config, s-config, realm-config, el3-config ]
157
+
158
+
"#address-cells":
159
+
enum: [ 1, 2 ]
160
+
161
+
"#size-cells":
162
+
enum: [ 1, 2 ]
163
+
164
+
ranges: true
165
+
166
+
dma-noncoherent:
167
+
description:
168
+
Present if the GIC ITS permits programming shareability and
169
+
cacheability attributes but is connected to a non-coherent
170
+
downstream interconnect.
171
+
172
+
patternProperties:
173
+
"^msi-controller@[0-9a-f]+$":
174
+
type: object
175
+
description:
176
+
GICv5 ITS has one or more translate register frames.
177
+
additionalProperties: false
178
+
179
+
properties:
180
+
reg:
181
+
items:
182
+
- description: ITS translate frames
183
+
184
+
reg-names:
185
+
description:
186
+
Describe translate frames that are present.
187
+
"ns-" stands for non-secure, "s-" for secure, "realm-" for realm
188
+
and "el3-" for EL3.
189
+
minItems: 1
190
+
maxItems: 4
191
+
items:
192
+
enum: [ ns-translate, s-translate, realm-translate, el3-translate ]
193
+
194
+
"#msi-cells":
195
+
description:
196
+
The single msi-cell is the DeviceID of the device which will
197
+
generate the MSI.
198
+
const: 1
199
+
200
+
msi-controller: true
201
+
202
+
required:
203
+
- reg
204
+
- reg-names
205
+
- "#msi-cells"
206
+
- msi-controller
207
+
208
+
required:
209
+
- compatible
210
+
- reg
211
+
- reg-names
212
+
213
+
required:
214
+
- compatible
215
+
- reg
216
+
- reg-names
217
+
- cpus
218
+
- arm,iaffids
219
+
220
+
additionalProperties: false
221
+
222
+
examples:
223
+
- |
224
+
interrupt-controller {
225
+
compatible = "arm,gic-v5";
226
+
227
+
#interrupt-cells = <3>;
228
+
interrupt-controller;
229
+
230
+
#address-cells = <1>;
231
+
#size-cells = <1>;
232
+
ranges;
233
+
234
+
interrupts = <1 25 4>;
235
+
236
+
irs@2f1a0000 {
237
+
compatible = "arm,gic-v5-irs";
238
+
reg = <0x2f1a0000 0x10000>; // IRS_CONFIG_FRAME
239
+
reg-names = "ns-config";
240
+
241
+
#address-cells = <1>;
242
+
#size-cells = <1>;
243
+
ranges;
244
+
245
+
cpus = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>, <&cpu4>, <&cpu5>, <&cpu6>, <&cpu7>;
246
+
arm,iaffids = /bits/ 16 <0 1 2 3 4 5 6 7>;
247
+
248
+
its@2f120000 {
249
+
compatible = "arm,gic-v5-its";
250
+
reg = <0x2f120000 0x10000>; // ITS_CONFIG_FRAME
251
+
reg-names = "ns-config";
252
+
253
+
#address-cells = <1>;
254
+
#size-cells = <1>;
255
+
ranges;
256
+
257
+
msi-controller@2f130000 {
258
+
reg = <0x2f130000 0x10000>; // ITS_TRANSLATE_FRAME
259
+
reg-names = "ns-translate";
260
+
261
+
#msi-cells = <1>;
262
+
msi-controller;
263
+
};
264
+
};
265
+
};
266
+
};
267
+
...
+12
-1
Documentation/virt/kvm/api.rst
+12
-1
Documentation/virt/kvm/api.rst
···
8622
8622
When enabled, KVM will exit to userspace with KVM_EXIT_SYSTEM_EVENT of
8623
8623
type KVM_SYSTEM_EVENT_SUSPEND to process the guest suspend request.
8624
8624
8625
-
7.37 KVM_CAP_ARM_WRITABLE_IMP_ID_REGS
8625
+
7.42 KVM_CAP_ARM_WRITABLE_IMP_ID_REGS
8626
8626
-------------------------------------
8627
8627
8628
8628
:Architectures: arm64
···
8650
8650
8651
8651
When this capability is enabled, KVM resets the VCPU when setting
8652
8652
MP_STATE_INIT_RECEIVED through IOCTL. The original MP_STATE is preserved.
8653
+
8654
+
7.43 KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED
8655
+
-------------------------------------------
8656
+
8657
+
:Architectures: arm64
8658
+
:Target: VM
8659
+
:Parameters: None
8660
+
8661
+
This capability indicate to the userspace whether a PFNMAP memory region
8662
+
can be safely mapped as cacheable. This relies on the presence of
8663
+
force write back (FWB) feature support on the hardware.
8653
8664
8654
8665
8. Other capabilities.
8655
8666
======================
+75
-5
Documentation/virt/kvm/devices/arm-vgic-v3.rst
+75
-5
Documentation/virt/kvm/devices/arm-vgic-v3.rst
···
78
78
-ENXIO The group or attribute is unknown/unsupported for this device
79
79
or hardware support is missing.
80
80
-EFAULT Invalid user pointer for attr->addr.
81
+
-EBUSY Attempt to write a register that is read-only after
82
+
initialization
81
83
======= =============================================================
82
84
83
85
···
122
120
Note that distributor fields are not banked, but return the same value
123
121
regardless of the mpidr used to access the register.
124
122
123
+
Userspace is allowed to write the following register fields prior to
124
+
initialization of the VGIC:
125
+
126
+
=====================
127
+
GICD_IIDR.Revision
128
+
GICD_TYPER2.nASSGIcap
129
+
=====================
130
+
131
+
125
132
GICD_IIDR.Revision is updated when the KVM implementation is changed in a
126
133
way directly observable by the guest or userspace. Userspace should read
127
134
GICD_IIDR from KVM and write back the read value to confirm its expected
128
135
behavior is aligned with the KVM implementation. Userspace should set
129
136
GICD_IIDR before setting any other registers to ensure the expected
130
137
behavior.
138
+
139
+
140
+
GICD_TYPER2.nASSGIcap allows userspace to control the support of SGIs
141
+
without an active state. At VGIC creation the field resets to the
142
+
maximum capability of the system. Userspace is expected to read the field
143
+
to determine the supported value(s) before writing to the field.
131
144
132
145
133
146
The GICD_STATUSR and GICR_STATUSR registers are architecturally defined such
···
219
202
KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS accesses the CPU interface registers for the
220
203
CPU specified by the mpidr field.
221
204
222
-
CPU interface registers access is not implemented for AArch32 mode.
223
-
Error -ENXIO is returned when accessed in AArch32 mode.
205
+
The available registers are:
206
+
207
+
=============== ====================================================
208
+
ICC_PMR_EL1
209
+
ICC_BPR0_EL1
210
+
ICC_AP0R0_EL1
211
+
ICC_AP0R1_EL1 when the host implements at least 6 bits of priority
212
+
ICC_AP0R2_EL1 when the host implements 7 bits of priority
213
+
ICC_AP0R3_EL1 when the host implements 7 bits of priority
214
+
ICC_AP1R0_EL1
215
+
ICC_AP1R1_EL1 when the host implements at least 6 bits of priority
216
+
ICC_AP1R2_EL1 when the host implements 7 bits of priority
217
+
ICC_AP1R3_EL1 when the host implements 7 bits of priority
218
+
ICC_BPR1_EL1
219
+
ICC_CTLR_EL1
220
+
ICC_SRE_EL1
221
+
ICC_IGRPEN0_EL1
222
+
ICC_IGRPEN1_EL1
223
+
=============== ====================================================
224
+
225
+
When EL2 is available for the guest, these registers are also available:
226
+
227
+
============= ====================================================
228
+
ICH_AP0R0_EL2
229
+
ICH_AP0R1_EL2 when the host implements at least 6 bits of priority
230
+
ICH_AP0R2_EL2 when the host implements 7 bits of priority
231
+
ICH_AP0R3_EL2 when the host implements 7 bits of priority
232
+
ICH_AP1R0_EL2
233
+
ICH_AP1R1_EL2 when the host implements at least 6 bits of priority
234
+
ICH_AP1R2_EL2 when the host implements 7 bits of priority
235
+
ICH_AP1R3_EL2 when the host implements 7 bits of priority
236
+
ICH_HCR_EL2
237
+
ICC_SRE_EL2
238
+
ICH_VTR_EL2
239
+
ICH_VMCR_EL2
240
+
ICH_LR0_EL2
241
+
ICH_LR1_EL2
242
+
ICH_LR2_EL2
243
+
ICH_LR3_EL2
244
+
ICH_LR4_EL2
245
+
ICH_LR5_EL2
246
+
ICH_LR6_EL2
247
+
ICH_LR7_EL2
248
+
ICH_LR8_EL2
249
+
ICH_LR9_EL2
250
+
ICH_LR10_EL2
251
+
ICH_LR11_EL2
252
+
ICH_LR12_EL2
253
+
ICH_LR13_EL2
254
+
ICH_LR14_EL2
255
+
ICH_LR15_EL2
256
+
============= ====================================================
257
+
258
+
CPU interface registers are only described using the AArch64
259
+
encoding.
224
260
225
261
Errors:
226
262
227
-
======= =====================================================
228
-
-ENXIO Getting or setting this register is not yet supported
263
+
======= =================================================
264
+
-ENXIO Getting or setting this register is not supported
229
265
-EBUSY VCPU is running
230
266
-EINVAL Invalid mpidr or register value supplied
231
-
======= =====================================================
267
+
======= =================================================
232
268
233
269
234
270
KVM_DEV_ARM_VGIC_GRP_NR_IRQS
+10
MAINTAINERS
+10
MAINTAINERS
···
1964
1964
F: include/linux/irqchip/arm-gic*.h
1965
1965
F: include/linux/irqchip/arm-vgic-info.h
1966
1966
1967
+
ARM GENERIC INTERRUPT CONTROLLER V5 DRIVERS
1968
+
M: Lorenzo Pieralisi <lpieralisi@kernel.org>
1969
+
M: Marc Zyngier <maz@kernel.org>
1970
+
L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
1971
+
S: Maintained
1972
+
F: Documentation/devicetree/bindings/interrupt-controller/arm,gic-v5*.yaml
1973
+
F: drivers/irqchip/irq-gic-its-msi-parent.[ch]
1974
+
F: drivers/irqchip/irq-gic-v5*.[ch]
1975
+
F: include/linux/irqchip/arm-gic-v5.h
1976
+
1967
1977
ARM HDLCD DRM DRIVER
1968
1978
M: Liviu Dudau <liviu.dudau@arm.com>
1969
1979
S: Supported
+1
arch/arm64/Kconfig
+1
arch/arm64/Kconfig
+3
arch/arm64/include/asm/barrier.h
+3
arch/arm64/include/asm/barrier.h
+45
arch/arm64/include/asm/el2_setup.h
+45
arch/arm64/include/asm/el2_setup.h
···
165
165
.Lskip_gicv3_\@:
166
166
.endm
167
167
168
+
/* GICv5 system register access */
169
+
.macro __init_el2_gicv5
170
+
mrs_s x0, SYS_ID_AA64PFR2_EL1
171
+
ubfx x0, x0, #ID_AA64PFR2_EL1_GCIE_SHIFT, #4
172
+
cbz x0, .Lskip_gicv5_\@
173
+
174
+
mov x0, #(ICH_HFGITR_EL2_GICRCDNMIA | \
175
+
ICH_HFGITR_EL2_GICRCDIA | \
176
+
ICH_HFGITR_EL2_GICCDDI | \
177
+
ICH_HFGITR_EL2_GICCDEOI | \
178
+
ICH_HFGITR_EL2_GICCDHM | \
179
+
ICH_HFGITR_EL2_GICCDRCFG | \
180
+
ICH_HFGITR_EL2_GICCDPEND | \
181
+
ICH_HFGITR_EL2_GICCDAFF | \
182
+
ICH_HFGITR_EL2_GICCDPRI | \
183
+
ICH_HFGITR_EL2_GICCDDIS | \
184
+
ICH_HFGITR_EL2_GICCDEN)
185
+
msr_s SYS_ICH_HFGITR_EL2, x0 // Disable instruction traps
186
+
mov_q x0, (ICH_HFGRTR_EL2_ICC_PPI_ACTIVERn_EL1 | \
187
+
ICH_HFGRTR_EL2_ICC_PPI_PRIORITYRn_EL1 | \
188
+
ICH_HFGRTR_EL2_ICC_PPI_PENDRn_EL1 | \
189
+
ICH_HFGRTR_EL2_ICC_PPI_ENABLERn_EL1 | \
190
+
ICH_HFGRTR_EL2_ICC_PPI_HMRn_EL1 | \
191
+
ICH_HFGRTR_EL2_ICC_IAFFIDR_EL1 | \
192
+
ICH_HFGRTR_EL2_ICC_ICSR_EL1 | \
193
+
ICH_HFGRTR_EL2_ICC_PCR_EL1 | \
194
+
ICH_HFGRTR_EL2_ICC_HPPIR_EL1 | \
195
+
ICH_HFGRTR_EL2_ICC_HAPR_EL1 | \
196
+
ICH_HFGRTR_EL2_ICC_CR0_EL1 | \
197
+
ICH_HFGRTR_EL2_ICC_IDRn_EL1 | \
198
+
ICH_HFGRTR_EL2_ICC_APR_EL1)
199
+
msr_s SYS_ICH_HFGRTR_EL2, x0 // Disable reg read traps
200
+
mov_q x0, (ICH_HFGWTR_EL2_ICC_PPI_ACTIVERn_EL1 | \
201
+
ICH_HFGWTR_EL2_ICC_PPI_PRIORITYRn_EL1 | \
202
+
ICH_HFGWTR_EL2_ICC_PPI_PENDRn_EL1 | \
203
+
ICH_HFGWTR_EL2_ICC_PPI_ENABLERn_EL1 | \
204
+
ICH_HFGWTR_EL2_ICC_ICSR_EL1 | \
205
+
ICH_HFGWTR_EL2_ICC_PCR_EL1 | \
206
+
ICH_HFGWTR_EL2_ICC_CR0_EL1 | \
207
+
ICH_HFGWTR_EL2_ICC_APR_EL1)
208
+
msr_s SYS_ICH_HFGWTR_EL2, x0 // Disable reg write traps
209
+
.Lskip_gicv5_\@:
210
+
.endm
211
+
168
212
.macro __init_el2_hstr
169
213
msr hstr_el2, xzr // Disable CP15 traps to EL2
170
214
.endm
···
347
303
__init_el2_lor
348
304
__init_el2_stage2
349
305
__init_el2_gicv3
306
+
__init_el2_gicv5
350
307
__init_el2_hstr
351
308
__init_el2_nvhe_idregs
352
309
__init_el2_cptr
+48
-3
arch/arm64/include/asm/kvm_emulate.h
+48
-3
arch/arm64/include/asm/kvm_emulate.h
···
45
45
void kvm_skip_instr32(struct kvm_vcpu *vcpu);
46
46
47
47
void kvm_inject_undefined(struct kvm_vcpu *vcpu);
48
-
void kvm_inject_vabt(struct kvm_vcpu *vcpu);
49
-
void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);
50
-
void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);
48
+
int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr);
49
+
int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr);
51
50
void kvm_inject_size_fault(struct kvm_vcpu *vcpu);
51
+
52
+
static inline int kvm_inject_sea_dabt(struct kvm_vcpu *vcpu, u64 addr)
53
+
{
54
+
return kvm_inject_sea(vcpu, false, addr);
55
+
}
56
+
57
+
static inline int kvm_inject_sea_iabt(struct kvm_vcpu *vcpu, u64 addr)
58
+
{
59
+
return kvm_inject_sea(vcpu, true, addr);
60
+
}
61
+
62
+
static inline int kvm_inject_serror(struct kvm_vcpu *vcpu)
63
+
{
64
+
/*
65
+
* ESR_ELx.ISV (later renamed to IDS) indicates whether or not
66
+
* ESR_ELx.ISS contains IMPLEMENTATION DEFINED syndrome information.
67
+
*
68
+
* Set the bit when injecting an SError w/o an ESR to indicate ISS
69
+
* does not follow the architected format.
70
+
*/
71
+
return kvm_inject_serror_esr(vcpu, ESR_ELx_ISV);
72
+
}
52
73
53
74
void kvm_vcpu_wfi(struct kvm_vcpu *vcpu);
54
75
55
76
void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu);
56
77
int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2);
57
78
int kvm_inject_nested_irq(struct kvm_vcpu *vcpu);
79
+
int kvm_inject_nested_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr);
80
+
int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr);
58
81
59
82
static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu)
60
83
{
···
218
195
return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_TGE;
219
196
}
220
197
198
+
static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu)
199
+
{
200
+
return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_AMO;
201
+
}
202
+
221
203
static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)
222
204
{
223
205
bool e2h, tge;
···
250
222
static inline bool vcpu_is_host_el0(const struct kvm_vcpu *vcpu)
251
223
{
252
224
return is_hyp_ctxt(vcpu) && !vcpu_is_el2(vcpu);
225
+
}
226
+
227
+
static inline bool is_nested_ctxt(struct kvm_vcpu *vcpu)
228
+
{
229
+
return vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu);
230
+
}
231
+
232
+
static inline bool vserror_state_is_nested(struct kvm_vcpu *vcpu)
233
+
{
234
+
if (!is_nested_ctxt(vcpu))
235
+
return false;
236
+
237
+
return vcpu_el2_amo_is_set(vcpu) ||
238
+
(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA);
253
239
}
254
240
255
241
/*
···
669
627
670
628
if (kvm_has_fpmr(kvm))
671
629
vcpu->arch.hcrx_el2 |= HCRX_EL2_EnFPM;
630
+
631
+
if (kvm_has_sctlr2(kvm))
632
+
vcpu->arch.hcrx_el2 |= HCRX_EL2_SCTLR2En;
672
633
}
673
634
}
674
635
#endif /* __ARM64_KVM_EMULATE_H__ */
+33
-3
arch/arm64/include/asm/kvm_host.h
+33
-3
arch/arm64/include/asm/kvm_host.h
···
523
523
/* Anything from this can be RES0/RES1 sanitised */
524
524
MARKER(__SANITISED_REG_START__),
525
525
TCR2_EL2, /* Extended Translation Control Register (EL2) */
526
+
SCTLR2_EL2, /* System Control Register 2 (EL2) */
526
527
MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */
527
528
CNTHCTL_EL2, /* Counter-timer Hypervisor Control register */
528
529
···
538
537
VNCR(TTBR1_EL1),/* Translation Table Base Register 1 */
539
538
VNCR(TCR_EL1), /* Translation Control Register */
540
539
VNCR(TCR2_EL1), /* Extended Translation Control Register */
540
+
VNCR(SCTLR2_EL1), /* System Control Register 2 */
541
541
VNCR(ESR_EL1), /* Exception Syndrome Register */
542
542
VNCR(AFSR0_EL1),/* Auxiliary Fault Status Register 0 */
543
543
VNCR(AFSR1_EL1),/* Auxiliary Fault Status Register 1 */
···
566
564
VNCR(PIRE0_EL1), /* Permission Indirection Register 0 (EL1) */
567
565
568
566
VNCR(POR_EL1), /* Permission Overlay Register 1 (EL1) */
567
+
568
+
/* FEAT_RAS registers */
569
+
VNCR(VDISR_EL2),
570
+
VNCR(VSESR_EL2),
569
571
570
572
VNCR(HFGRTR_EL2),
571
573
VNCR(HFGWTR_EL2),
···
823
817
u8 iflags;
824
818
825
819
/* State flags for kernel bookkeeping, unused by the hypervisor code */
826
-
u8 sflags;
820
+
u16 sflags;
827
821
828
822
/*
829
823
* Don't run the guest (internal implementation need).
···
959
953
__vcpu_flags_preempt_enable(); \
960
954
} while (0)
961
955
956
+
#define __vcpu_test_and_clear_flag(v, flagset, f, m) \
957
+
({ \
958
+
typeof(v->arch.flagset) set; \
959
+
\
960
+
set = __vcpu_get_flag(v, flagset, f, m); \
961
+
__vcpu_clear_flag(v, flagset, f, m); \
962
+
\
963
+
set; \
964
+
})
965
+
962
966
#define vcpu_get_flag(v, ...) __vcpu_get_flag((v), __VA_ARGS__)
963
967
#define vcpu_set_flag(v, ...) __vcpu_set_flag((v), __VA_ARGS__)
964
968
#define vcpu_clear_flag(v, ...) __vcpu_clear_flag((v), __VA_ARGS__)
969
+
#define vcpu_test_and_clear_flag(v, ...) \
970
+
__vcpu_test_and_clear_flag((v), __VA_ARGS__)
965
971
966
972
/* KVM_ARM_VCPU_INIT completed */
967
973
#define VCPU_INITIALIZED __vcpu_single_flag(cflags, BIT(0))
···
1033
1015
#define IN_WFI __vcpu_single_flag(sflags, BIT(6))
1034
1016
/* KVM is currently emulating a nested ERET */
1035
1017
#define IN_NESTED_ERET __vcpu_single_flag(sflags, BIT(7))
1018
+
/* SError pending for nested guest */
1019
+
#define NESTED_SERROR_PENDING __vcpu_single_flag(sflags, BIT(8))
1036
1020
1037
1021
1038
1022
/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */
···
1169
1149
* System registers listed in the switch are not saved on every
1170
1150
* exit from the guest but are only saved on vcpu_put.
1171
1151
*
1152
+
* SYSREGS_ON_CPU *MUST* be checked before using this helper.
1153
+
*
1172
1154
* Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
1173
1155
* should never be listed below, because the guest cannot modify its
1174
1156
* own MPIDR_EL1 and MPIDR_EL1 is accessed for VCPU A from VCPU B's
···
1208
1186
case IFSR32_EL2: *val = read_sysreg_s(SYS_IFSR32_EL2); break;
1209
1187
case DBGVCR32_EL2: *val = read_sysreg_s(SYS_DBGVCR32_EL2); break;
1210
1188
case ZCR_EL1: *val = read_sysreg_s(SYS_ZCR_EL12); break;
1189
+
case SCTLR2_EL1: *val = read_sysreg_s(SYS_SCTLR2_EL12); break;
1211
1190
default: return false;
1212
1191
}
1213
1192
···
1222
1199
*
1223
1200
* System registers listed in the switch are not restored on every
1224
1201
* entry to the guest but are only restored on vcpu_load.
1202
+
*
1203
+
* SYSREGS_ON_CPU *MUST* be checked before using this helper.
1225
1204
*
1226
1205
* Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
1227
1206
* should never be listed below, because the MPIDR should only be set
···
1261
1236
case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break;
1262
1237
case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); break;
1263
1238
case ZCR_EL1: write_sysreg_s(val, SYS_ZCR_EL12); break;
1239
+
case SCTLR2_EL1: write_sysreg_s(val, SYS_SCTLR2_EL12); break;
1264
1240
default: return false;
1265
1241
}
1266
1242
···
1412
1386
{
1413
1387
return (vcpu_arch->steal.base != INVALID_GPA);
1414
1388
}
1415
-
1416
-
void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 syndrome);
1417
1389
1418
1390
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
1419
1391
···
1688
1664
1689
1665
#define kvm_has_s1poe(k) \
1690
1666
(kvm_has_feat((k), ID_AA64MMFR3_EL1, S1POE, IMP))
1667
+
1668
+
#define kvm_has_ras(k) \
1669
+
(kvm_has_feat((k), ID_AA64PFR0_EL1, RAS, IMP))
1670
+
1671
+
#define kvm_has_sctlr2(k) \
1672
+
(kvm_has_feat((k), ID_AA64MMFR3_EL1, SCTLRX, IMP))
1691
1673
1692
1674
static inline bool kvm_arch_has_irq_bypass(void)
1693
1675
{
+18
arch/arm64/include/asm/kvm_mmu.h
+18
arch/arm64/include/asm/kvm_mmu.h
···
371
371
read_unlock(&kvm->mmu_lock);
372
372
}
373
373
374
+
/*
375
+
* ARM64 KVM relies on a simple conversion from physaddr to a kernel
376
+
* virtual address (KVA) when it does cache maintenance as the CMO
377
+
* instructions work on virtual addresses. This is incompatible with
378
+
* VM_PFNMAP VMAs which may not have a kernel direct mapping to a
379
+
* virtual address.
380
+
*
381
+
* With S2FWB and CACHE DIC features, KVM need not do cache flushing
382
+
* and CMOs are NOP'd. This has the effect of no longer requiring a
383
+
* KVA for addresses mapped into the S2. The presence of these features
384
+
* are thus necessary to support cacheable S2 mapping of VM_PFNMAP.
385
+
*/
386
+
static inline bool kvm_supports_cacheable_pfnmap(void)
387
+
{
388
+
return cpus_have_final_cap(ARM64_HAS_STAGE2_FWB) &&
389
+
cpus_have_final_cap(ARM64_HAS_CACHE_DIC);
390
+
}
391
+
374
392
#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS
375
393
void kvm_s2_ptdump_create_debugfs(struct kvm *kvm);
376
394
#else
+2
arch/arm64/include/asm/kvm_nested.h
+2
arch/arm64/include/asm/kvm_nested.h
···
80
80
extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);
81
81
82
82
extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu);
83
+
extern void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu);
84
+
extern void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu);
83
85
84
86
struct kvm_s2_trans {
85
87
phys_addr_t output;
+23
-1
arch/arm64/include/asm/smp.h
+23
-1
arch/arm64/include/asm/smp.h
···
50
50
*/
51
51
extern void smp_init_cpus(void);
52
52
53
+
enum ipi_msg_type {
54
+
IPI_RESCHEDULE,
55
+
IPI_CALL_FUNC,
56
+
IPI_CPU_STOP,
57
+
IPI_CPU_STOP_NMI,
58
+
IPI_TIMER,
59
+
IPI_IRQ_WORK,
60
+
NR_IPI,
61
+
/*
62
+
* Any enum >= NR_IPI and < MAX_IPI is special and not tracable
63
+
* with trace_ipi_*
64
+
*/
65
+
IPI_CPU_BACKTRACE = NR_IPI,
66
+
IPI_KGDB_ROUNDUP,
67
+
MAX_IPI
68
+
};
69
+
53
70
/*
54
71
* Register IPI interrupts with the arch SMP code
55
72
*/
56
-
extern void set_smp_ipi_range(int ipi_base, int nr_ipi);
73
+
extern void set_smp_ipi_range_percpu(int ipi_base, int nr_ipi, int ncpus);
74
+
75
+
static inline void set_smp_ipi_range(int ipi_base, int n)
76
+
{
77
+
set_smp_ipi_range_percpu(ipi_base, n, 0);
78
+
}
57
79
58
80
/*
59
81
* Called from the secondary holding pen, this is the secondary CPU entry point.
+68
-3
arch/arm64/include/asm/sysreg.h
+68
-3
arch/arm64/include/asm/sysreg.h
···
113
113
/* Register-based PAN access, for save/restore purposes */
114
114
#define SYS_PSTATE_PAN sys_reg(3, 0, 4, 2, 3)
115
115
116
-
#define __SYS_BARRIER_INSN(CRm, op2, Rt) \
117
-
__emit_inst(0xd5000000 | sys_insn(0, 3, 3, (CRm), (op2)) | ((Rt) & 0x1f))
116
+
#define __SYS_BARRIER_INSN(op0, op1, CRn, CRm, op2, Rt) \
117
+
__emit_inst(0xd5000000 | \
118
+
sys_insn((op0), (op1), (CRn), (CRm), (op2)) | \
119
+
((Rt) & 0x1f))
118
120
119
-
#define SB_BARRIER_INSN __SYS_BARRIER_INSN(0, 7, 31)
121
+
#define SB_BARRIER_INSN __SYS_BARRIER_INSN(0, 3, 3, 0, 7, 31)
122
+
#define GSB_SYS_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 0, 31)
123
+
#define GSB_ACK_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 1, 31)
120
124
121
125
/* Data cache zero operations */
122
126
#define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2)
···
1082
1078
1083
1079
#define GCS_CAP(x) ((((unsigned long)x) & GCS_CAP_ADDR_MASK) | \
1084
1080
GCS_CAP_VALID_TOKEN)
1081
+
/*
1082
+
* Definitions for GICv5 instructions
1083
+
*/
1084
+
#define GICV5_OP_GIC_CDAFF sys_insn(1, 0, 12, 1, 3)
1085
+
#define GICV5_OP_GIC_CDDI sys_insn(1, 0, 12, 2, 0)
1086
+
#define GICV5_OP_GIC_CDDIS sys_insn(1, 0, 12, 1, 0)
1087
+
#define GICV5_OP_GIC_CDHM sys_insn(1, 0, 12, 2, 1)
1088
+
#define GICV5_OP_GIC_CDEN sys_insn(1, 0, 12, 1, 1)
1089
+
#define GICV5_OP_GIC_CDEOI sys_insn(1, 0, 12, 1, 7)
1090
+
#define GICV5_OP_GIC_CDPEND sys_insn(1, 0, 12, 1, 4)
1091
+
#define GICV5_OP_GIC_CDPRI sys_insn(1, 0, 12, 1, 2)
1092
+
#define GICV5_OP_GIC_CDRCFG sys_insn(1, 0, 12, 1, 5)
1093
+
#define GICV5_OP_GICR_CDIA sys_insn(1, 0, 12, 3, 0)
1094
+
1095
+
/* Definitions for GIC CDAFF */
1096
+
#define GICV5_GIC_CDAFF_IAFFID_MASK GENMASK_ULL(47, 32)
1097
+
#define GICV5_GIC_CDAFF_TYPE_MASK GENMASK_ULL(31, 29)
1098
+
#define GICV5_GIC_CDAFF_IRM_MASK BIT_ULL(28)
1099
+
#define GICV5_GIC_CDAFF_ID_MASK GENMASK_ULL(23, 0)
1100
+
1101
+
/* Definitions for GIC CDDI */
1102
+
#define GICV5_GIC_CDDI_TYPE_MASK GENMASK_ULL(31, 29)
1103
+
#define GICV5_GIC_CDDI_ID_MASK GENMASK_ULL(23, 0)
1104
+
1105
+
/* Definitions for GIC CDDIS */
1106
+
#define GICV5_GIC_CDDIS_TYPE_MASK GENMASK_ULL(31, 29)
1107
+
#define GICV5_GIC_CDDIS_TYPE(r) FIELD_GET(GICV5_GIC_CDDIS_TYPE_MASK, r)
1108
+
#define GICV5_GIC_CDDIS_ID_MASK GENMASK_ULL(23, 0)
1109
+
#define GICV5_GIC_CDDIS_ID(r) FIELD_GET(GICV5_GIC_CDDIS_ID_MASK, r)
1110
+
1111
+
/* Definitions for GIC CDEN */
1112
+
#define GICV5_GIC_CDEN_TYPE_MASK GENMASK_ULL(31, 29)
1113
+
#define GICV5_GIC_CDEN_ID_MASK GENMASK_ULL(23, 0)
1114
+
1115
+
/* Definitions for GIC CDHM */
1116
+
#define GICV5_GIC_CDHM_HM_MASK BIT_ULL(32)
1117
+
#define GICV5_GIC_CDHM_TYPE_MASK GENMASK_ULL(31, 29)
1118
+
#define GICV5_GIC_CDHM_ID_MASK GENMASK_ULL(23, 0)
1119
+
1120
+
/* Definitions for GIC CDPEND */
1121
+
#define GICV5_GIC_CDPEND_PENDING_MASK BIT_ULL(32)
1122
+
#define GICV5_GIC_CDPEND_TYPE_MASK GENMASK_ULL(31, 29)
1123
+
#define GICV5_GIC_CDPEND_ID_MASK GENMASK_ULL(23, 0)
1124
+
1125
+
/* Definitions for GIC CDPRI */
1126
+
#define GICV5_GIC_CDPRI_PRIORITY_MASK GENMASK_ULL(39, 35)
1127
+
#define GICV5_GIC_CDPRI_TYPE_MASK GENMASK_ULL(31, 29)
1128
+
#define GICV5_GIC_CDPRI_ID_MASK GENMASK_ULL(23, 0)
1129
+
1130
+
/* Definitions for GIC CDRCFG */
1131
+
#define GICV5_GIC_CDRCFG_TYPE_MASK GENMASK_ULL(31, 29)
1132
+
#define GICV5_GIC_CDRCFG_ID_MASK GENMASK_ULL(23, 0)
1133
+
1134
+
/* Definitions for GICR CDIA */
1135
+
#define GICV5_GIC_CDIA_VALID_MASK BIT_ULL(32)
1136
+
#define GICV5_GICR_CDIA_VALID(r) FIELD_GET(GICV5_GIC_CDIA_VALID_MASK, r)
1137
+
#define GICV5_GIC_CDIA_TYPE_MASK GENMASK_ULL(31, 29)
1138
+
#define GICV5_GIC_CDIA_ID_MASK GENMASK_ULL(23, 0)
1139
+
1140
+
#define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn)
1141
+
#define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn)
1085
1142
1086
1143
#define ARM64_FEATURE_FIELD_BITS 4
1087
1144
+2
arch/arm64/include/asm/vncr_mapping.h
+2
arch/arm64/include/asm/vncr_mapping.h
···
51
51
#define VNCR_SP_EL1 0x240
52
52
#define VNCR_VBAR_EL1 0x250
53
53
#define VNCR_TCR2_EL1 0x270
54
+
#define VNCR_SCTLR2_EL1 0x278
54
55
#define VNCR_PIRE0_EL1 0x290
55
56
#define VNCR_PIR_EL1 0x2A0
56
57
#define VNCR_POR_EL1 0x2A8
···
85
84
#define VNCR_ICH_HCR_EL2 0x4C0
86
85
#define VNCR_ICH_VMCR_EL2 0x4C8
87
86
#define VNCR_VDISR_EL2 0x500
87
+
#define VNCR_VSESR_EL2 0x508
88
88
#define VNCR_PMBLIMITR_EL1 0x800
89
89
#define VNCR_PMBPTR_EL1 0x810
90
90
#define VNCR_PMBSR_EL1 0x820
+21
-5
arch/arm64/kernel/cpufeature.c
+21
-5
arch/arm64/kernel/cpufeature.c
···
303
303
};
304
304
305
305
static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
306
+
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_DF2_SHIFT, 4, 0),
306
307
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_GCS),
307
308
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_GCS_SHIFT, 4, 0),
308
309
S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MTE_frac_SHIFT, 4, 0),
···
501
500
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_POE),
502
501
FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1POE_SHIFT, 4, 0),
503
502
ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1PIE_SHIFT, 4, 0),
503
+
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_SCTLRX_SHIFT, 4, 0),
504
504
ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_TCRX_SHIFT, 4, 0),
505
505
ARM64_FTR_END,
506
506
};
···
2298
2296
int scope)
2299
2297
{
2300
2298
/*
2301
-
* ARM64_HAS_GIC_CPUIF_SYSREGS has a lower index, and is a boot CPU
2299
+
* ARM64_HAS_GICV3_CPUIF has a lower index, and is a boot CPU
2302
2300
* feature, so will be detected earlier.
2303
2301
*/
2304
-
BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_MASKING <= ARM64_HAS_GIC_CPUIF_SYSREGS);
2305
-
if (!cpus_have_cap(ARM64_HAS_GIC_CPUIF_SYSREGS))
2302
+
BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_MASKING <= ARM64_HAS_GICV3_CPUIF);
2303
+
if (!cpus_have_cap(ARM64_HAS_GICV3_CPUIF))
2306
2304
return false;
2307
2305
2308
2306
return enable_pseudo_nmi;
···
2498
2496
.matches = has_always,
2499
2497
},
2500
2498
{
2501
-
.desc = "GIC system register CPU interface",
2502
-
.capability = ARM64_HAS_GIC_CPUIF_SYSREGS,
2499
+
.desc = "GICv3 CPU interface",
2500
+
.capability = ARM64_HAS_GICV3_CPUIF,
2503
2501
.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2504
2502
.matches = has_useable_gicv3_cpuif,
2505
2503
ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, GIC, IMP)
···
3063
3061
.matches = has_pmuv3,
3064
3062
},
3065
3063
#endif
3064
+
{
3065
+
.desc = "SCTLR2",
3066
+
.capability = ARM64_HAS_SCTLR2,
3067
+
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3068
+
.matches = has_cpuid_feature,
3069
+
ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, SCTLRX, IMP)
3070
+
},
3071
+
{
3072
+
.desc = "GICv5 CPU interface",
3073
+
.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
3074
+
.capability = ARM64_HAS_GICV5_CPUIF,
3075
+
.matches = has_cpuid_feature,
3076
+
ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, GCIE, IMP)
3077
+
},
3066
3078
{},
3067
3079
};
3068
3080
+93
-51
arch/arm64/kernel/smp.c
+93
-51
arch/arm64/kernel/smp.c
···
64
64
/* Number of CPUs which aren't online, but looping in kernel text. */
65
65
static int cpus_stuck_in_kernel;
66
66
67
-
enum ipi_msg_type {
68
-
IPI_RESCHEDULE,
69
-
IPI_CALL_FUNC,
70
-
IPI_CPU_STOP,
71
-
IPI_CPU_STOP_NMI,
72
-
IPI_TIMER,
73
-
IPI_IRQ_WORK,
74
-
NR_IPI,
75
-
/*
76
-
* Any enum >= NR_IPI and < MAX_IPI is special and not tracable
77
-
* with trace_ipi_*
78
-
*/
79
-
IPI_CPU_BACKTRACE = NR_IPI,
80
-
IPI_KGDB_ROUNDUP,
81
-
MAX_IPI
82
-
};
83
-
84
67
static int ipi_irq_base __ro_after_init;
85
68
static int nr_ipi __ro_after_init = NR_IPI;
86
-
static struct irq_desc *ipi_desc[MAX_IPI] __ro_after_init;
69
+
70
+
struct ipi_descs {
71
+
struct irq_desc *descs[MAX_IPI];
72
+
};
73
+
74
+
static DEFINE_PER_CPU_READ_MOSTLY(struct ipi_descs, pcpu_ipi_desc);
75
+
76
+
#define get_ipi_desc(__cpu, __ipi) (per_cpu_ptr(&pcpu_ipi_desc, __cpu)->descs[__ipi])
77
+
78
+
static bool percpu_ipi_descs __ro_after_init;
87
79
88
80
static bool crash_stop;
89
81
···
836
844
seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
837
845
prec >= 4 ? " " : "");
838
846
for_each_online_cpu(cpu)
839
-
seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu));
847
+
seq_printf(p, "%10u ", irq_desc_kstat_cpu(get_ipi_desc(cpu, i), cpu));
840
848
seq_printf(p, " %s\n", ipi_types[i]);
841
849
}
842
850
···
909
917
#endif
910
918
}
911
919
920
+
static void arm64_send_ipi(const cpumask_t *mask, unsigned int nr)
921
+
{
922
+
unsigned int cpu;
923
+
924
+
if (!percpu_ipi_descs)
925
+
__ipi_send_mask(get_ipi_desc(0, nr), mask);
926
+
else
927
+
for_each_cpu(cpu, mask)
928
+
__ipi_send_single(get_ipi_desc(cpu, nr), cpu);
929
+
}
930
+
912
931
static void arm64_backtrace_ipi(cpumask_t *mask)
913
932
{
914
-
__ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask);
933
+
arm64_send_ipi(mask, IPI_CPU_BACKTRACE);
915
934
}
916
935
917
936
void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu)
···
947
944
if (cpu == this_cpu)
948
945
continue;
949
946
950
-
__ipi_send_single(ipi_desc[IPI_KGDB_ROUNDUP], cpu);
947
+
__ipi_send_single(get_ipi_desc(cpu, IPI_KGDB_ROUNDUP), cpu);
951
948
}
952
949
}
953
950
#endif
···
1016
1013
1017
1014
static irqreturn_t ipi_handler(int irq, void *data)
1018
1015
{
1019
-
do_handle_IPI(irq - ipi_irq_base);
1016
+
unsigned int ipi = (irq - ipi_irq_base) % nr_ipi;
1017
+
1018
+
do_handle_IPI(ipi);
1020
1019
return IRQ_HANDLED;
1021
1020
}
1022
1021
1023
1022
static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
1024
1023
{
1025
1024
trace_ipi_raise(target, ipi_types[ipinr]);
1026
-
__ipi_send_mask(ipi_desc[ipinr], target);
1025
+
arm64_send_ipi(target, ipinr);
1027
1026
}
1028
1027
1029
1028
static bool ipi_should_be_nmi(enum ipi_msg_type ipi)
···
1051
1046
return;
1052
1047
1053
1048
for (i = 0; i < nr_ipi; i++) {
1054
-
if (ipi_should_be_nmi(i)) {
1055
-
prepare_percpu_nmi(ipi_irq_base + i);
1056
-
enable_percpu_nmi(ipi_irq_base + i, 0);
1049
+
if (!percpu_ipi_descs) {
1050
+
if (ipi_should_be_nmi(i)) {
1051
+
prepare_percpu_nmi(ipi_irq_base + i);
1052
+
enable_percpu_nmi(ipi_irq_base + i, 0);
1053
+
} else {
1054
+
enable_percpu_irq(ipi_irq_base + i, 0);
1055
+
}
1057
1056
} else {
1058
-
enable_percpu_irq(ipi_irq_base + i, 0);
1057
+
enable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));
1059
1058
}
1060
1059
}
1061
1060
}
···
1073
1064
return;
1074
1065
1075
1066
for (i = 0; i < nr_ipi; i++) {
1076
-
if (ipi_should_be_nmi(i)) {
1077
-
disable_percpu_nmi(ipi_irq_base + i);
1078
-
teardown_percpu_nmi(ipi_irq_base + i);
1067
+
if (!percpu_ipi_descs) {
1068
+
if (ipi_should_be_nmi(i)) {
1069
+
disable_percpu_nmi(ipi_irq_base + i);
1070
+
teardown_percpu_nmi(ipi_irq_base + i);
1071
+
} else {
1072
+
disable_percpu_irq(ipi_irq_base + i);
1073
+
}
1079
1074
} else {
1080
-
disable_percpu_irq(ipi_irq_base + i);
1075
+
disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));
1081
1076
}
1082
1077
}
1083
1078
}
1084
1079
#endif
1085
1080
1086
-
void __init set_smp_ipi_range(int ipi_base, int n)
1081
+
static void ipi_setup_sgi(int ipi)
1082
+
{
1083
+
int err, irq, cpu;
1084
+
1085
+
irq = ipi_irq_base + ipi;
1086
+
1087
+
if (ipi_should_be_nmi(ipi)) {
1088
+
err = request_percpu_nmi(irq, ipi_handler, "IPI", &irq_stat);
1089
+
WARN(err, "Could not request IRQ %d as NMI, err=%d\n", irq, err);
1090
+
} else {
1091
+
err = request_percpu_irq(irq, ipi_handler, "IPI", &irq_stat);
1092
+
WARN(err, "Could not request IRQ %d as IRQ, err=%d\n", irq, err);
1093
+
}
1094
+
1095
+
for_each_possible_cpu(cpu)
1096
+
get_ipi_desc(cpu, ipi) = irq_to_desc(irq);
1097
+
1098
+
irq_set_status_flags(irq, IRQ_HIDDEN);
1099
+
}
1100
+
1101
+
static void ipi_setup_lpi(int ipi, int ncpus)
1102
+
{
1103
+
for (int cpu = 0; cpu < ncpus; cpu++) {
1104
+
int err, irq;
1105
+
1106
+
irq = ipi_irq_base + (cpu * nr_ipi) + ipi;
1107
+
1108
+
err = irq_force_affinity(irq, cpumask_of(cpu));
1109
+
WARN(err, "Could not force affinity IRQ %d, err=%d\n", irq, err);
1110
+
1111
+
err = request_irq(irq, ipi_handler, IRQF_NO_AUTOEN, "IPI",
1112
+
NULL);
1113
+
WARN(err, "Could not request IRQ %d, err=%d\n", irq, err);
1114
+
1115
+
irq_set_status_flags(irq, (IRQ_HIDDEN | IRQ_NO_BALANCING_MASK));
1116
+
1117
+
get_ipi_desc(cpu, ipi) = irq_to_desc(irq);
1118
+
}
1119
+
}
1120
+
1121
+
void __init set_smp_ipi_range_percpu(int ipi_base, int n, int ncpus)
1087
1122
{
1088
1123
int i;
1089
1124
1090
1125
WARN_ON(n < MAX_IPI);
1091
1126
nr_ipi = min(n, MAX_IPI);
1092
1127
1093
-
for (i = 0; i < nr_ipi; i++) {
1094
-
int err;
1095
-
1096
-
if (ipi_should_be_nmi(i)) {
1097
-
err = request_percpu_nmi(ipi_base + i, ipi_handler,
1098
-
"IPI", &irq_stat);
1099
-
WARN(err, "Could not request IPI %d as NMI, err=%d\n",
1100
-
i, err);
1101
-
} else {
1102
-
err = request_percpu_irq(ipi_base + i, ipi_handler,
1103
-
"IPI", &irq_stat);
1104
-
WARN(err, "Could not request IPI %d as IRQ, err=%d\n",
1105
-
i, err);
1106
-
}
1107
-
1108
-
ipi_desc[i] = irq_to_desc(ipi_base + i);
1109
-
irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);
1110
-
}
1111
-
1128
+
percpu_ipi_descs = !!ncpus;
1112
1129
ipi_irq_base = ipi_base;
1130
+
1131
+
for (i = 0; i < nr_ipi; i++) {
1132
+
if (!percpu_ipi_descs)
1133
+
ipi_setup_sgi(i);
1134
+
else
1135
+
ipi_setup_lpi(i, ncpus);
1136
+
}
1113
1137
1114
1138
/* Setup the boot CPU immediately */
1115
1139
ipi_setup(smp_processor_id());
+2
-1
arch/arm64/kvm/Makefile
+2
-1
arch/arm64/kvm/Makefile
···
23
23
vgic/vgic-v3.o vgic/vgic-v4.o \
24
24
vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \
25
25
vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \
26
-
vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o
26
+
vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \
27
+
vgic/vgic-v5.o
27
28
28
29
kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o
29
30
kvm-$(CONFIG_ARM64_PTR_AUTH) += pauth.o
+1
-1
arch/arm64/kvm/arch_timer.c
+1
-1
arch/arm64/kvm/arch_timer.c
···
830
830
* by the guest (either FEAT_VHE or FEAT_E2H0 is implemented, but
831
831
* not both). This simplifies the handling of the EL1NV* bits.
832
832
*/
833
-
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
833
+
if (is_nested_ctxt(vcpu)) {
834
834
u64 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2);
835
835
836
836
/* Use the VHE format for mental sanity */
+14
-2
arch/arm64/kvm/arm.c
+14
-2
arch/arm64/kvm/arm.c
···
408
408
case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES:
409
409
r = BIT(0);
410
410
break;
411
+
case KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED:
412
+
if (!kvm)
413
+
r = -EINVAL;
414
+
else
415
+
r = kvm_supports_cacheable_pfnmap();
416
+
break;
417
+
411
418
default:
412
419
r = 0;
413
420
}
···
528
521
* Either we're running an L2 guest, and the API/APK bits come
529
522
* from L1's HCR_EL2, or API/APK are both set.
530
523
*/
531
-
if (unlikely(vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))) {
524
+
if (unlikely(is_nested_ctxt(vcpu))) {
532
525
u64 val;
533
526
534
527
val = __vcpu_sys_reg(vcpu, HCR_EL2);
···
747
740
*/
748
741
int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
749
742
{
750
-
bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
743
+
bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE);
744
+
751
745
return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
752
746
&& !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
753
747
}
···
1191
1183
*/
1192
1184
preempt_disable();
1193
1185
1186
+
kvm_nested_flush_hwstate(vcpu);
1187
+
1194
1188
if (kvm_vcpu_has_pmu(vcpu))
1195
1189
kvm_pmu_flush_hwstate(vcpu);
1196
1190
···
1291
1281
1292
1282
/* Exit types that need handling before we can be preempted */
1293
1283
handle_exit_early(vcpu, ret);
1284
+
1285
+
kvm_nested_sync_hwstate(vcpu);
1294
1286
1295
1287
preempt_enable();
1296
1288
+44
-36
arch/arm64/kvm/at.c
+44
-36
arch/arm64/kvm/at.c
···
1047
1047
1048
1048
idx = FIELD_GET(PTE_PO_IDX_MASK, wr->desc);
1049
1049
1050
-
switch (wi->regime) {
1051
-
case TR_EL10:
1052
-
pov_perms = perm_idx(vcpu, POR_EL1, idx);
1053
-
uov_perms = perm_idx(vcpu, POR_EL0, idx);
1054
-
break;
1055
-
case TR_EL20:
1056
-
pov_perms = perm_idx(vcpu, POR_EL2, idx);
1057
-
uov_perms = perm_idx(vcpu, POR_EL0, idx);
1058
-
break;
1059
-
case TR_EL2:
1060
-
pov_perms = perm_idx(vcpu, POR_EL2, idx);
1061
-
uov_perms = 0;
1062
-
break;
1063
-
}
1050
+
if (wr->pov) {
1051
+
switch (wi->regime) {
1052
+
case TR_EL10:
1053
+
pov_perms = perm_idx(vcpu, POR_EL1, idx);
1054
+
break;
1055
+
case TR_EL20:
1056
+
pov_perms = perm_idx(vcpu, POR_EL2, idx);
1057
+
break;
1058
+
case TR_EL2:
1059
+
pov_perms = perm_idx(vcpu, POR_EL2, idx);
1060
+
break;
1061
+
}
1064
1062
1065
-
if (pov_perms & ~POE_RWX)
1066
-
pov_perms = POE_NONE;
1063
+
if (pov_perms & ~POE_RWX)
1064
+
pov_perms = POE_NONE;
1067
1065
1068
-
if (wi->poe && wr->pov) {
1066
+
/* R_QXXPC, S1PrivOverflow enabled */
1067
+
if (wr->pwxn && (pov_perms & POE_X))
1068
+
pov_perms &= ~POE_W;
1069
+
1069
1070
wr->pr &= pov_perms & POE_R;
1070
1071
wr->pw &= pov_perms & POE_W;
1071
1072
wr->px &= pov_perms & POE_X;
1072
1073
}
1073
1074
1074
-
if (uov_perms & ~POE_RWX)
1075
-
uov_perms = POE_NONE;
1075
+
if (wr->uov) {
1076
+
switch (wi->regime) {
1077
+
case TR_EL10:
1078
+
uov_perms = perm_idx(vcpu, POR_EL0, idx);
1079
+
break;
1080
+
case TR_EL20:
1081
+
uov_perms = perm_idx(vcpu, POR_EL0, idx);
1082
+
break;
1083
+
case TR_EL2:
1084
+
uov_perms = 0;
1085
+
break;
1086
+
}
1076
1087
1077
-
if (wi->e0poe && wr->uov) {
1088
+
if (uov_perms & ~POE_RWX)
1089
+
uov_perms = POE_NONE;
1090
+
1091
+
/* R_NPBXC, S1UnprivOverlay enabled */
1092
+
if (wr->uwxn && (uov_perms & POE_X))
1093
+
uov_perms &= ~POE_W;
1094
+
1078
1095
wr->ur &= uov_perms & POE_R;
1079
1096
wr->uw &= uov_perms & POE_W;
1080
1097
wr->ux &= uov_perms & POE_X;
···
1112
1095
if (!wi->hpd)
1113
1096
compute_s1_hierarchical_permissions(vcpu, wi, wr);
1114
1097
1115
-
if (wi->poe || wi->e0poe)
1116
-
compute_s1_overlay_permissions(vcpu, wi, wr);
1098
+
compute_s1_overlay_permissions(vcpu, wi, wr);
1117
1099
1118
-
/* R_QXXPC */
1119
-
if (wr->pwxn) {
1120
-
if (!wr->pov && wr->pw)
1121
-
wr->px = false;
1122
-
if (wr->pov && wr->px)
1123
-
wr->pw = false;
1124
-
}
1100
+
/* R_QXXPC, S1PrivOverlay disabled */
1101
+
if (!wr->pov)
1102
+
wr->px &= !(wr->pwxn && wr->pw);
1125
1103
1126
-
/* R_NPBXC */
1127
-
if (wr->uwxn) {
1128
-
if (!wr->uov && wr->uw)
1129
-
wr->ux = false;
1130
-
if (wr->uov && wr->ux)
1131
-
wr->uw = false;
1132
-
}
1104
+
/* R_NPBXC, S1UnprivOverlay disabled */
1105
+
if (!wr->uov)
1106
+
wr->ux &= !(wr->uwxn && wr->uw);
1133
1107
1134
1108
pan = wi->pan && (wr->ur || wr->uw ||
1135
1109
(pan3_enabled(vcpu, wi->regime) && wr->ux));
+252
-3
arch/arm64/kvm/config.c
+252
-3
arch/arm64/kvm/config.c
···
66
66
#define FEAT_BRBE ID_AA64DFR0_EL1, BRBE, IMP
67
67
#define FEAT_TRC_SR ID_AA64DFR0_EL1, TraceVer, IMP
68
68
#define FEAT_PMUv3 ID_AA64DFR0_EL1, PMUVer, IMP
69
-
#define FEAT_PMUv3p9 ID_AA64DFR0_EL1, PMUVer, V3P9
70
69
#define FEAT_TRBE ID_AA64DFR0_EL1, TraceBuffer, IMP
71
70
#define FEAT_TRBEv1p1 ID_AA64DFR0_EL1, TraceBuffer, TRBE_V1P1
72
71
#define FEAT_DoubleLock ID_AA64DFR0_EL1, DoubleLock, IMP
···
88
89
#define FEAT_RASv2 ID_AA64PFR0_EL1, RAS, V2
89
90
#define FEAT_GICv3 ID_AA64PFR0_EL1, GIC, IMP
90
91
#define FEAT_LOR ID_AA64MMFR1_EL1, LO, IMP
92
+
#define FEAT_SPEv1p2 ID_AA64DFR0_EL1, PMSVer, V1P2
91
93
#define FEAT_SPEv1p4 ID_AA64DFR0_EL1, PMSVer, V1P4
92
94
#define FEAT_SPEv1p5 ID_AA64DFR0_EL1, PMSVer, V1P5
93
95
#define FEAT_ATS1A ID_AA64ISAR2_EL1, ATS1A, IMP
···
131
131
#define FEAT_SPMU ID_AA64DFR1_EL1, SPMU, IMP
132
132
#define FEAT_SPE_nVM ID_AA64DFR2_EL1, SPE_nVM, IMP
133
133
#define FEAT_STEP2 ID_AA64DFR2_EL1, STEP, IMP
134
+
#define FEAT_SYSREG128 ID_AA64ISAR2_EL1, SYSREG_128, IMP
135
+
#define FEAT_CPA2 ID_AA64ISAR3_EL1, CPA, CPA2
136
+
#define FEAT_ASID2 ID_AA64MMFR4_EL1, ASID2, IMP
137
+
#define FEAT_MEC ID_AA64MMFR3_EL1, MEC, IMP
138
+
#define FEAT_HAFT ID_AA64MMFR1_EL1, HAFDBS, HAFT
139
+
#define FEAT_BTI ID_AA64PFR1_EL1, BT, IMP
140
+
#define FEAT_ExS ID_AA64MMFR0_EL1, EXS, IMP
141
+
#define FEAT_IESB ID_AA64MMFR2_EL1, IESB, IMP
142
+
#define FEAT_LSE2 ID_AA64MMFR2_EL1, AT, IMP
143
+
#define FEAT_LSMAOC ID_AA64MMFR2_EL1, LSM, IMP
144
+
#define FEAT_MixedEnd ID_AA64MMFR0_EL1, BIGEND, IMP
145
+
#define FEAT_MixedEndEL0 ID_AA64MMFR0_EL1, BIGENDEL0, IMP
146
+
#define FEAT_MTE2 ID_AA64PFR1_EL1, MTE, MTE2
147
+
#define FEAT_MTE_ASYNC ID_AA64PFR1_EL1, MTE_frac, ASYNC
148
+
#define FEAT_MTE_STORE_ONLY ID_AA64PFR2_EL1, MTESTOREONLY, IMP
149
+
#define FEAT_PAN ID_AA64MMFR1_EL1, PAN, IMP
150
+
#define FEAT_PAN3 ID_AA64MMFR1_EL1, PAN, PAN3
151
+
#define FEAT_SSBS ID_AA64PFR1_EL1, SSBS, IMP
152
+
#define FEAT_TIDCP1 ID_AA64MMFR1_EL1, TIDCP1, IMP
153
+
#define FEAT_FGT ID_AA64MMFR0_EL1, FGT, IMP
154
+
#define FEAT_MTPMU ID_AA64DFR0_EL1, MTPMU, IMP
134
155
135
156
static bool not_feat_aa64el3(struct kvm *kvm)
136
157
{
···
239
218
(read_sysreg_s(SYS_TRBIDR_EL1) & TRBIDR_EL1_MPAM));
240
219
}
241
220
221
+
static bool feat_asid2_e2h1(struct kvm *kvm)
222
+
{
223
+
return kvm_has_feat(kvm, FEAT_ASID2) && !kvm_has_feat(kvm, FEAT_E2H0);
224
+
}
225
+
226
+
static bool feat_d128_e2h1(struct kvm *kvm)
227
+
{
228
+
return kvm_has_feat(kvm, FEAT_D128) && !kvm_has_feat(kvm, FEAT_E2H0);
229
+
}
230
+
231
+
static bool feat_mec_e2h1(struct kvm *kvm)
232
+
{
233
+
return kvm_has_feat(kvm, FEAT_MEC) && !kvm_has_feat(kvm, FEAT_E2H0);
234
+
}
235
+
242
236
static bool feat_ebep_pmuv3_ss(struct kvm *kvm)
243
237
{
244
238
return kvm_has_feat(kvm, FEAT_EBEP) || kvm_has_feat(kvm, FEAT_PMUv3_SS);
239
+
}
240
+
241
+
static bool feat_mixedendel0(struct kvm *kvm)
242
+
{
243
+
return kvm_has_feat(kvm, FEAT_MixedEnd) || kvm_has_feat(kvm, FEAT_MixedEndEL0);
244
+
}
245
+
246
+
static bool feat_mte_async(struct kvm *kvm)
247
+
{
248
+
return kvm_has_feat(kvm, FEAT_MTE2) && kvm_has_feat_enum(kvm, FEAT_MTE_ASYNC);
249
+
}
250
+
251
+
#define check_pmu_revision(k, r) \
252
+
({ \
253
+
(kvm_has_feat((k), ID_AA64DFR0_EL1, PMUVer, r) && \
254
+
!kvm_has_feat((k), ID_AA64DFR0_EL1, PMUVer, IMP_DEF)); \
255
+
})
256
+
257
+
static bool feat_pmuv3p1(struct kvm *kvm)
258
+
{
259
+
return check_pmu_revision(kvm, V3P1);
260
+
}
261
+
262
+
static bool feat_pmuv3p5(struct kvm *kvm)
263
+
{
264
+
return check_pmu_revision(kvm, V3P5);
265
+
}
266
+
267
+
static bool feat_pmuv3p7(struct kvm *kvm)
268
+
{
269
+
return check_pmu_revision(kvm, V3P7);
270
+
}
271
+
272
+
static bool feat_pmuv3p9(struct kvm *kvm)
273
+
{
274
+
return check_pmu_revision(kvm, V3P9);
245
275
}
246
276
247
277
static bool compute_hcr_rw(struct kvm *kvm, u64 *bits)
···
753
681
NEEDS_FEAT(HDFGRTR2_EL2_nPMICFILTR_EL0 |
754
682
HDFGRTR2_EL2_nPMICNTR_EL0,
755
683
FEAT_PMUv3_ICNTR),
756
-
NEEDS_FEAT(HDFGRTR2_EL2_nPMUACR_EL1, FEAT_PMUv3p9),
684
+
NEEDS_FEAT(HDFGRTR2_EL2_nPMUACR_EL1, feat_pmuv3p9),
757
685
NEEDS_FEAT(HDFGRTR2_EL2_nPMSSCR_EL1 |
758
686
HDFGRTR2_EL2_nPMSSDATA,
759
687
FEAT_PMUv3_SS),
···
785
713
FEAT_PMUv3_ICNTR),
786
714
NEEDS_FEAT(HDFGWTR2_EL2_nPMUACR_EL1 |
787
715
HDFGWTR2_EL2_nPMZR_EL0,
788
-
FEAT_PMUv3p9),
716
+
feat_pmuv3p9),
789
717
NEEDS_FEAT(HDFGWTR2_EL2_nPMSSCR_EL1, FEAT_PMUv3_SS),
790
718
NEEDS_FEAT(HDFGWTR2_EL2_nPMIAR_EL1, FEAT_SEBEP),
791
719
NEEDS_FEAT(HDFGWTR2_EL2_nPMSDSFR_EL1, feat_spe_fds),
···
904
832
NEEDS_FEAT_FIXED(HCR_EL2_E2H, compute_hcr_e2h),
905
833
};
906
834
835
+
static const struct reg_bits_to_feat_map sctlr2_feat_map[] = {
836
+
NEEDS_FEAT(SCTLR2_EL1_NMEA |
837
+
SCTLR2_EL1_EASE,
838
+
FEAT_DoubleFault2),
839
+
NEEDS_FEAT(SCTLR2_EL1_EnADERR, feat_aderr),
840
+
NEEDS_FEAT(SCTLR2_EL1_EnANERR, feat_anerr),
841
+
NEEDS_FEAT(SCTLR2_EL1_EnIDCP128, FEAT_SYSREG128),
842
+
NEEDS_FEAT(SCTLR2_EL1_EnPACM |
843
+
SCTLR2_EL1_EnPACM0,
844
+
feat_pauth_lr),
845
+
NEEDS_FEAT(SCTLR2_EL1_CPTA |
846
+
SCTLR2_EL1_CPTA0 |
847
+
SCTLR2_EL1_CPTM |
848
+
SCTLR2_EL1_CPTM0,
849
+
FEAT_CPA2),
850
+
};
851
+
852
+
static const struct reg_bits_to_feat_map tcr2_el2_feat_map[] = {
853
+
NEEDS_FEAT(TCR2_EL2_FNG1 |
854
+
TCR2_EL2_FNG0 |
855
+
TCR2_EL2_A2,
856
+
feat_asid2_e2h1),
857
+
NEEDS_FEAT(TCR2_EL2_DisCH1 |
858
+
TCR2_EL2_DisCH0 |
859
+
TCR2_EL2_D128,
860
+
feat_d128_e2h1),
861
+
NEEDS_FEAT(TCR2_EL2_AMEC1, feat_mec_e2h1),
862
+
NEEDS_FEAT(TCR2_EL2_AMEC0, FEAT_MEC),
863
+
NEEDS_FEAT(TCR2_EL2_HAFT, FEAT_HAFT),
864
+
NEEDS_FEAT(TCR2_EL2_PTTWI |
865
+
TCR2_EL2_PnCH,
866
+
FEAT_THE),
867
+
NEEDS_FEAT(TCR2_EL2_AIE, FEAT_AIE),
868
+
NEEDS_FEAT(TCR2_EL2_POE |
869
+
TCR2_EL2_E0POE,
870
+
FEAT_S1POE),
871
+
NEEDS_FEAT(TCR2_EL2_PIE, FEAT_S1PIE),
872
+
};
873
+
874
+
static const struct reg_bits_to_feat_map sctlr_el1_feat_map[] = {
875
+
NEEDS_FEAT(SCTLR_EL1_CP15BEN |
876
+
SCTLR_EL1_ITD |
877
+
SCTLR_EL1_SED,
878
+
FEAT_AA32EL0),
879
+
NEEDS_FEAT(SCTLR_EL1_BT0 |
880
+
SCTLR_EL1_BT1,
881
+
FEAT_BTI),
882
+
NEEDS_FEAT(SCTLR_EL1_CMOW, FEAT_CMOW),
883
+
NEEDS_FEAT(SCTLR_EL1_TSCXT, feat_csv2_2_csv2_1p2),
884
+
NEEDS_FEAT(SCTLR_EL1_EIS |
885
+
SCTLR_EL1_EOS,
886
+
FEAT_ExS),
887
+
NEEDS_FEAT(SCTLR_EL1_EnFPM, FEAT_FPMR),
888
+
NEEDS_FEAT(SCTLR_EL1_IESB, FEAT_IESB),
889
+
NEEDS_FEAT(SCTLR_EL1_EnALS, FEAT_LS64),
890
+
NEEDS_FEAT(SCTLR_EL1_EnAS0, FEAT_LS64_ACCDATA),
891
+
NEEDS_FEAT(SCTLR_EL1_EnASR, FEAT_LS64_V),
892
+
NEEDS_FEAT(SCTLR_EL1_nAA, FEAT_LSE2),
893
+
NEEDS_FEAT(SCTLR_EL1_LSMAOE |
894
+
SCTLR_EL1_nTLSMD,
895
+
FEAT_LSMAOC),
896
+
NEEDS_FEAT(SCTLR_EL1_EE, FEAT_MixedEnd),
897
+
NEEDS_FEAT(SCTLR_EL1_E0E, feat_mixedendel0),
898
+
NEEDS_FEAT(SCTLR_EL1_MSCEn, FEAT_MOPS),
899
+
NEEDS_FEAT(SCTLR_EL1_ATA0 |
900
+
SCTLR_EL1_ATA |
901
+
SCTLR_EL1_TCF0 |
902
+
SCTLR_EL1_TCF,
903
+
FEAT_MTE2),
904
+
NEEDS_FEAT(SCTLR_EL1_ITFSB, feat_mte_async),
905
+
NEEDS_FEAT(SCTLR_EL1_TCSO0 |
906
+
SCTLR_EL1_TCSO,
907
+
FEAT_MTE_STORE_ONLY),
908
+
NEEDS_FEAT(SCTLR_EL1_NMI |
909
+
SCTLR_EL1_SPINTMASK,
910
+
FEAT_NMI),
911
+
NEEDS_FEAT(SCTLR_EL1_SPAN, FEAT_PAN),
912
+
NEEDS_FEAT(SCTLR_EL1_EPAN, FEAT_PAN3),
913
+
NEEDS_FEAT(SCTLR_EL1_EnDA |
914
+
SCTLR_EL1_EnDB |
915
+
SCTLR_EL1_EnIA |
916
+
SCTLR_EL1_EnIB,
917
+
feat_pauth),
918
+
NEEDS_FEAT(SCTLR_EL1_EnTP2, FEAT_SME),
919
+
NEEDS_FEAT(SCTLR_EL1_EnRCTX, FEAT_SPECRES),
920
+
NEEDS_FEAT(SCTLR_EL1_DSSBS, FEAT_SSBS),
921
+
NEEDS_FEAT(SCTLR_EL1_TIDCP, FEAT_TIDCP1),
922
+
NEEDS_FEAT(SCTLR_EL1_TME0 |
923
+
SCTLR_EL1_TME |
924
+
SCTLR_EL1_TMT0 |
925
+
SCTLR_EL1_TMT,
926
+
FEAT_TME),
927
+
NEEDS_FEAT(SCTLR_EL1_TWEDEL |
928
+
SCTLR_EL1_TWEDEn,
929
+
FEAT_TWED),
930
+
NEEDS_FEAT(SCTLR_EL1_UCI |
931
+
SCTLR_EL1_EE |
932
+
SCTLR_EL1_E0E |
933
+
SCTLR_EL1_WXN |
934
+
SCTLR_EL1_nTWE |
935
+
SCTLR_EL1_nTWI |
936
+
SCTLR_EL1_UCT |
937
+
SCTLR_EL1_DZE |
938
+
SCTLR_EL1_I |
939
+
SCTLR_EL1_UMA |
940
+
SCTLR_EL1_SA0 |
941
+
SCTLR_EL1_SA |
942
+
SCTLR_EL1_C |
943
+
SCTLR_EL1_A |
944
+
SCTLR_EL1_M,
945
+
FEAT_AA64EL1),
946
+
};
947
+
948
+
static const struct reg_bits_to_feat_map mdcr_el2_feat_map[] = {
949
+
NEEDS_FEAT(MDCR_EL2_EBWE, FEAT_Debugv8p9),
950
+
NEEDS_FEAT(MDCR_EL2_TDOSA, FEAT_DoubleLock),
951
+
NEEDS_FEAT(MDCR_EL2_PMEE, FEAT_EBEP),
952
+
NEEDS_FEAT(MDCR_EL2_TDCC, FEAT_FGT),
953
+
NEEDS_FEAT(MDCR_EL2_MTPME, FEAT_MTPMU),
954
+
NEEDS_FEAT(MDCR_EL2_HPME |
955
+
MDCR_EL2_HPMN |
956
+
MDCR_EL2_TPMCR |
957
+
MDCR_EL2_TPM,
958
+
FEAT_PMUv3),
959
+
NEEDS_FEAT(MDCR_EL2_HPMD, feat_pmuv3p1),
960
+
NEEDS_FEAT(MDCR_EL2_HCCD |
961
+
MDCR_EL2_HLP,
962
+
feat_pmuv3p5),
963
+
NEEDS_FEAT(MDCR_EL2_HPMFZO, feat_pmuv3p7),
964
+
NEEDS_FEAT(MDCR_EL2_PMSSE, FEAT_PMUv3_SS),
965
+
NEEDS_FEAT(MDCR_EL2_E2PB |
966
+
MDCR_EL2_TPMS,
967
+
FEAT_SPE),
968
+
NEEDS_FEAT(MDCR_EL2_HPMFZS, FEAT_SPEv1p2),
969
+
NEEDS_FEAT(MDCR_EL2_EnSPM, FEAT_SPMU),
970
+
NEEDS_FEAT(MDCR_EL2_EnSTEPOP, FEAT_STEP2),
971
+
NEEDS_FEAT(MDCR_EL2_E2TB, FEAT_TRBE),
972
+
NEEDS_FEAT(MDCR_EL2_TTRF, FEAT_TRF),
973
+
NEEDS_FEAT(MDCR_EL2_TDA |
974
+
MDCR_EL2_TDE |
975
+
MDCR_EL2_TDRA,
976
+
FEAT_AA64EL1),
977
+
};
978
+
907
979
static void __init check_feat_map(const struct reg_bits_to_feat_map *map,
908
980
int map_size, u64 res0, const char *str)
909
981
{
···
1079
863
__HCRX_EL2_RES0, "HCRX_EL2");
1080
864
check_feat_map(hcr_feat_map, ARRAY_SIZE(hcr_feat_map),
1081
865
HCR_EL2_RES0, "HCR_EL2");
866
+
check_feat_map(sctlr2_feat_map, ARRAY_SIZE(sctlr2_feat_map),
867
+
SCTLR2_EL1_RES0, "SCTLR2_EL1");
868
+
check_feat_map(tcr2_el2_feat_map, ARRAY_SIZE(tcr2_el2_feat_map),
869
+
TCR2_EL2_RES0, "TCR2_EL2");
870
+
check_feat_map(sctlr_el1_feat_map, ARRAY_SIZE(sctlr_el1_feat_map),
871
+
SCTLR_EL1_RES0, "SCTLR_EL1");
872
+
check_feat_map(mdcr_el2_feat_map, ARRAY_SIZE(mdcr_el2_feat_map),
873
+
MDCR_EL2_RES0, "MDCR_EL2");
1082
874
}
1083
875
1084
876
static bool idreg_feat_match(struct kvm *kvm, const struct reg_bits_to_feat_map *map)
···
1300
1076
ARRAY_SIZE(hcr_feat_map), 0, 0);
1301
1077
*res0 |= HCR_EL2_RES0 | (mask & ~fixed);
1302
1078
*res1 = HCR_EL2_RES1 | (mask & fixed);
1079
+
break;
1080
+
case SCTLR2_EL1:
1081
+
case SCTLR2_EL2:
1082
+
*res0 = compute_res0_bits(kvm, sctlr2_feat_map,
1083
+
ARRAY_SIZE(sctlr2_feat_map), 0, 0);
1084
+
*res0 |= SCTLR2_EL1_RES0;
1085
+
*res1 = SCTLR2_EL1_RES1;
1086
+
break;
1087
+
case TCR2_EL2:
1088
+
*res0 = compute_res0_bits(kvm, tcr2_el2_feat_map,
1089
+
ARRAY_SIZE(tcr2_el2_feat_map), 0, 0);
1090
+
*res0 |= TCR2_EL2_RES0;
1091
+
*res1 = TCR2_EL2_RES1;
1092
+
break;
1093
+
case SCTLR_EL1:
1094
+
*res0 = compute_res0_bits(kvm, sctlr_el1_feat_map,
1095
+
ARRAY_SIZE(sctlr_el1_feat_map), 0, 0);
1096
+
*res0 |= SCTLR_EL1_RES0;
1097
+
*res1 = SCTLR_EL1_RES1;
1098
+
break;
1099
+
case MDCR_EL2:
1100
+
*res0 = compute_res0_bits(kvm, mdcr_el2_feat_map,
1101
+
ARRAY_SIZE(mdcr_el2_feat_map), 0, 0);
1102
+
*res0 |= MDCR_EL2_RES0;
1103
+
*res1 = MDCR_EL2_RES1;
1303
1104
break;
1304
1105
default:
1305
1106
WARN_ON_ONCE(1);
+42
-7
arch/arm64/kvm/emulate-nested.c
+42
-7
arch/arm64/kvm/emulate-nested.c
···
88
88
89
89
CGT_HCRX_EnFPM,
90
90
CGT_HCRX_TCR2En,
91
+
CGT_HCRX_SCTLR2En,
91
92
92
93
CGT_CNTHCTL_EL1TVT,
93
94
CGT_CNTHCTL_EL1TVCT,
···
109
108
CGT_HCR_TTLB_TTLBOS,
110
109
CGT_HCR_TVM_TRVM,
111
110
CGT_HCR_TVM_TRVM_HCRX_TCR2En,
111
+
CGT_HCR_TVM_TRVM_HCRX_SCTLR2En,
112
112
CGT_HCR_TPU_TICAB,
113
113
CGT_HCR_TPU_TOCU,
114
114
CGT_HCR_NV1_nNV2_ENSCXT,
···
400
398
.mask = HCRX_EL2_TCR2En,
401
399
.behaviour = BEHAVE_FORWARD_RW,
402
400
},
401
+
[CGT_HCRX_SCTLR2En] = {
402
+
.index = HCRX_EL2,
403
+
.value = 0,
404
+
.mask = HCRX_EL2_SCTLR2En,
405
+
.behaviour = BEHAVE_FORWARD_RW,
406
+
},
403
407
[CGT_CNTHCTL_EL1TVT] = {
404
408
.index = CNTHCTL_EL2,
405
409
.value = CNTHCTL_EL1TVT,
···
457
449
MCB(CGT_HCR_TVM_TRVM, CGT_HCR_TVM, CGT_HCR_TRVM),
458
450
MCB(CGT_HCR_TVM_TRVM_HCRX_TCR2En,
459
451
CGT_HCR_TVM, CGT_HCR_TRVM, CGT_HCRX_TCR2En),
452
+
MCB(CGT_HCR_TVM_TRVM_HCRX_SCTLR2En,
453
+
CGT_HCR_TVM, CGT_HCR_TRVM, CGT_HCRX_SCTLR2En),
460
454
MCB(CGT_HCR_TPU_TICAB, CGT_HCR_TPU, CGT_HCR_TICAB),
461
455
MCB(CGT_HCR_TPU_TOCU, CGT_HCR_TPU, CGT_HCR_TOCU),
462
456
MCB(CGT_HCR_NV1_nNV2_ENSCXT, CGT_HCR_NV1_nNV2, CGT_HCR_ENSCXT),
···
792
782
SR_TRAP(OP_TLBI_RVALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
793
783
SR_TRAP(OP_TLBI_RVAALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
794
784
SR_TRAP(SYS_SCTLR_EL1, CGT_HCR_TVM_TRVM),
785
+
SR_TRAP(SYS_SCTLR2_EL1, CGT_HCR_TVM_TRVM_HCRX_SCTLR2En),
795
786
SR_TRAP(SYS_TTBR0_EL1, CGT_HCR_TVM_TRVM),
796
787
SR_TRAP(SYS_TTBR1_EL1, CGT_HCR_TVM_TRVM),
797
788
SR_TRAP(SYS_TCR_EL1, CGT_HCR_TVM_TRVM),
···
1365
1354
SR_FGT(SYS_SCXTNUM_EL0, HFGRTR, SCXTNUM_EL0, 1),
1366
1355
SR_FGT(SYS_SCXTNUM_EL1, HFGRTR, SCXTNUM_EL1, 1),
1367
1356
SR_FGT(SYS_SCTLR_EL1, HFGRTR, SCTLR_EL1, 1),
1357
+
SR_FGT(SYS_SCTLR2_EL1, HFGRTR, SCTLR_EL1, 1),
1368
1358
SR_FGT(SYS_REVIDR_EL1, HFGRTR, REVIDR_EL1, 1),
1369
1359
SR_FGT(SYS_PAR_EL1, HFGRTR, PAR_EL1, 1),
1370
1360
SR_FGT(SYS_MPIDR_EL1, HFGRTR, MPIDR_EL1, 1),
···
2604
2592
2605
2593
static bool __forward_traps(struct kvm_vcpu *vcpu, unsigned int reg, u64 control_bit)
2606
2594
{
2607
-
bool control_bit_set;
2608
-
2609
-
if (!vcpu_has_nv(vcpu))
2610
-
return false;
2611
-
2612
-
control_bit_set = __vcpu_sys_reg(vcpu, reg) & control_bit;
2613
-
if (!is_hyp_ctxt(vcpu) && control_bit_set) {
2595
+
if (is_nested_ctxt(vcpu) &&
2596
+
(__vcpu_sys_reg(vcpu, reg) & control_bit)) {
2614
2597
kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
2615
2598
return true;
2616
2599
}
···
2726
2719
case except_type_irq:
2727
2720
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_IRQ);
2728
2721
break;
2722
+
case except_type_serror:
2723
+
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR);
2724
+
break;
2729
2725
default:
2730
2726
WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type);
2731
2727
}
···
2825
2815
2826
2816
/* esr_el2 value doesn't matter for exits due to irqs. */
2827
2817
return kvm_inject_nested(vcpu, 0, except_type_irq);
2818
+
}
2819
+
2820
+
int kvm_inject_nested_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
2821
+
{
2822
+
u64 esr = FIELD_PREP(ESR_ELx_EC_MASK,
2823
+
iabt ? ESR_ELx_EC_IABT_LOW : ESR_ELx_EC_DABT_LOW);
2824
+
esr |= ESR_ELx_FSC_EXTABT | ESR_ELx_IL;
2825
+
2826
+
vcpu_write_sys_reg(vcpu, FAR_EL2, addr);
2827
+
2828
+
if (__vcpu_sys_reg(vcpu, SCTLR2_EL2) & SCTLR2_EL1_EASE)
2829
+
return kvm_inject_nested(vcpu, esr, except_type_serror);
2830
+
2831
+
return kvm_inject_nested_sync(vcpu, esr);
2832
+
}
2833
+
2834
+
int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr)
2835
+
{
2836
+
/*
2837
+
* Hardware sets up the EC field when propagating ESR as a result of
2838
+
* vSError injection. Manually populate EC for an emulated SError
2839
+
* exception.
2840
+
*/
2841
+
esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR);
2842
+
return kvm_inject_nested(vcpu, esr, except_type_serror);
2828
2843
}
+48
-14
arch/arm64/kvm/guest.c
+48
-14
arch/arm64/kvm/guest.c
···
818
818
int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
819
819
struct kvm_vcpu_events *events)
820
820
{
821
-
events->exception.serror_pending = !!(vcpu->arch.hcr_el2 & HCR_VSE);
822
821
events->exception.serror_has_esr = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
822
+
events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_VSE) ||
823
+
vcpu_get_flag(vcpu, NESTED_SERROR_PENDING);
823
824
824
825
if (events->exception.serror_pending && events->exception.serror_has_esr)
825
826
events->exception.serror_esr = vcpu_get_vsesr(vcpu);
···
834
833
return 0;
835
834
}
836
835
836
+
static void commit_pending_events(struct kvm_vcpu *vcpu)
837
+
{
838
+
if (!vcpu_get_flag(vcpu, PENDING_EXCEPTION))
839
+
return;
840
+
841
+
/*
842
+
* Reset the MMIO emulation state to avoid stepping PC after emulating
843
+
* the exception entry.
844
+
*/
845
+
vcpu->mmio_needed = false;
846
+
kvm_call_hyp(__kvm_adjust_pc, vcpu);
847
+
}
848
+
837
849
int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
838
850
struct kvm_vcpu_events *events)
839
851
{
840
852
bool serror_pending = events->exception.serror_pending;
841
853
bool has_esr = events->exception.serror_has_esr;
842
854
bool ext_dabt_pending = events->exception.ext_dabt_pending;
855
+
u64 esr = events->exception.serror_esr;
856
+
int ret = 0;
843
857
844
-
if (serror_pending && has_esr) {
845
-
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
846
-
return -EINVAL;
847
-
848
-
if (!((events->exception.serror_esr) & ~ESR_ELx_ISS_MASK))
849
-
kvm_set_sei_esr(vcpu, events->exception.serror_esr);
850
-
else
851
-
return -EINVAL;
852
-
} else if (serror_pending) {
853
-
kvm_inject_vabt(vcpu);
858
+
/*
859
+
* Immediately commit the pending SEA to the vCPU's architectural
860
+
* state which is necessary since we do not return a pending SEA
861
+
* to userspace via KVM_GET_VCPU_EVENTS.
862
+
*/
863
+
if (ext_dabt_pending) {
864
+
ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
865
+
commit_pending_events(vcpu);
854
866
}
855
867
856
-
if (ext_dabt_pending)
857
-
kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
868
+
if (ret < 0)
869
+
return ret;
858
870
859
-
return 0;
871
+
if (!serror_pending)
872
+
return 0;
873
+
874
+
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && has_esr)
875
+
return -EINVAL;
876
+
877
+
if (has_esr && (esr & ~ESR_ELx_ISS_MASK))
878
+
return -EINVAL;
879
+
880
+
if (has_esr)
881
+
ret = kvm_inject_serror_esr(vcpu, esr);
882
+
else
883
+
ret = kvm_inject_serror(vcpu);
884
+
885
+
/*
886
+
* We could've decided that the SError is due for immediate software
887
+
* injection; commit the exception in case userspace decides it wants
888
+
* to inject more exceptions for some strange reason.
889
+
*/
890
+
commit_pending_events(vcpu);
891
+
return (ret < 0) ? ret : 0;
860
892
}
861
893
862
894
u32 __attribute_const__ kvm_target_cpu(void)
+9
-15
arch/arm64/kvm/handle_exit.c
+9
-15
arch/arm64/kvm/handle_exit.c
···
32
32
static void kvm_handle_guest_serror(struct kvm_vcpu *vcpu, u64 esr)
33
33
{
34
34
if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(NULL, esr))
35
-
kvm_inject_vabt(vcpu);
35
+
kvm_inject_serror(vcpu);
36
36
}
37
37
38
38
static int handle_hvc(struct kvm_vcpu *vcpu)
···
252
252
return 1;
253
253
}
254
254
255
-
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
255
+
if (is_nested_ctxt(vcpu)) {
256
256
kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
257
257
return 1;
258
258
}
···
311
311
312
312
static int handle_other(struct kvm_vcpu *vcpu)
313
313
{
314
-
bool is_l2 = vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu);
314
+
bool allowed, fwd = is_nested_ctxt(vcpu);
315
315
u64 hcrx = __vcpu_sys_reg(vcpu, HCRX_EL2);
316
316
u64 esr = kvm_vcpu_get_esr(vcpu);
317
317
u64 iss = ESR_ELx_ISS(esr);
318
318
struct kvm *kvm = vcpu->kvm;
319
-
bool allowed, fwd = false;
320
319
321
320
/*
322
321
* We only trap for two reasons:
···
334
335
switch (iss) {
335
336
case ESR_ELx_ISS_OTHER_ST64BV:
336
337
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V);
337
-
if (is_l2)
338
-
fwd = !(hcrx & HCRX_EL2_EnASR);
338
+
fwd &= !(hcrx & HCRX_EL2_EnASR);
339
339
break;
340
340
case ESR_ELx_ISS_OTHER_ST64BV0:
341
341
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA);
342
-
if (is_l2)
343
-
fwd = !(hcrx & HCRX_EL2_EnAS0);
342
+
fwd &= !(hcrx & HCRX_EL2_EnAS0);
344
343
break;
345
344
case ESR_ELx_ISS_OTHER_LDST64B:
346
345
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64);
347
-
if (is_l2)
348
-
fwd = !(hcrx & HCRX_EL2_EnALS);
346
+
fwd &= !(hcrx & HCRX_EL2_EnALS);
349
347
break;
350
348
case ESR_ELx_ISS_OTHER_TSBCSYNC:
351
349
allowed = kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, TRBE_V1P1);
352
-
if (is_l2)
353
-
fwd = (__vcpu_sys_reg(vcpu, HFGITR2_EL2) & HFGITR2_EL2_TSBCSYNC);
350
+
fwd &= (__vcpu_sys_reg(vcpu, HFGITR2_EL2) & HFGITR2_EL2_TSBCSYNC);
354
351
break;
355
352
case ESR_ELx_ISS_OTHER_PSBCSYNC:
356
353
allowed = kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P5);
357
-
if (is_l2)
358
-
fwd = (__vcpu_sys_reg(vcpu, HFGITR_EL2) & HFGITR_EL2_PSBCSYNC);
354
+
fwd &= (__vcpu_sys_reg(vcpu, HFGITR_EL2) & HFGITR_EL2_PSBCSYNC);
359
355
break;
360
356
default:
361
357
/* Clearly, we're missing something. */
···
490
496
491
497
kvm_handle_guest_serror(vcpu, disr_to_esr(disr));
492
498
} else {
493
-
kvm_inject_vabt(vcpu);
499
+
kvm_inject_serror(vcpu);
494
500
}
495
501
496
502
return;
+13
-3
arch/arm64/kvm/hyp/exception.c
+13
-3
arch/arm64/kvm/hyp/exception.c
···
26
26
27
27
if (unlikely(vcpu_has_nv(vcpu)))
28
28
return vcpu_read_sys_reg(vcpu, reg);
29
-
else if (__vcpu_read_sys_reg_from_cpu(reg, &val))
29
+
else if (vcpu_get_flag(vcpu, SYSREGS_ON_CPU) &&
30
+
__vcpu_read_sys_reg_from_cpu(reg, &val))
30
31
return val;
31
32
32
33
return __vcpu_sys_reg(vcpu, reg);
···
37
36
{
38
37
if (unlikely(vcpu_has_nv(vcpu)))
39
38
vcpu_write_sys_reg(vcpu, val, reg);
40
-
else if (!__vcpu_write_sys_reg_to_cpu(val, reg))
39
+
else if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU) ||
40
+
!__vcpu_write_sys_reg_to_cpu(val, reg))
41
41
__vcpu_assign_sys_reg(vcpu, reg, val);
42
42
}
43
43
···
341
339
enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync);
342
340
break;
343
341
342
+
case unpack_vcpu_flag(EXCEPT_AA64_EL1_SERR):
343
+
enter_exception64(vcpu, PSR_MODE_EL1h, except_type_serror);
344
+
break;
345
+
344
346
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SYNC):
345
347
enter_exception64(vcpu, PSR_MODE_EL2h, except_type_sync);
346
348
break;
···
353
347
enter_exception64(vcpu, PSR_MODE_EL2h, except_type_irq);
354
348
break;
355
349
350
+
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SERR):
351
+
enter_exception64(vcpu, PSR_MODE_EL2h, except_type_serror);
352
+
break;
353
+
356
354
default:
357
355
/*
358
-
* Only EL1_SYNC and EL2_{SYNC,IRQ} makes
356
+
* Only EL1_{SYNC,SERR} and EL2_{SYNC,IRQ,SERR} makes
359
357
* sense so far. Everything else gets silently
360
358
* ignored.
361
359
*/
+44
-9
arch/arm64/kvm/hyp/include/hyp/switch.h
+44
-9
arch/arm64/kvm/hyp/include/hyp/switch.h
···
298
298
u64 val; \
299
299
\
300
300
ctxt_sys_reg(hctxt, reg) = read_sysreg_s(SYS_ ## reg); \
301
-
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) \
301
+
if (is_nested_ctxt(vcpu)) \
302
302
compute_clr_set(vcpu, reg, c, s); \
303
303
\
304
304
compute_undef_clr_set(vcpu, kvm, reg, c, s); \
···
436
436
437
437
if (cpus_have_final_cap(ARM64_HAS_HCX)) {
438
438
u64 hcrx = vcpu->arch.hcrx_el2;
439
-
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
439
+
if (is_nested_ctxt(vcpu)) {
440
440
u64 val = __vcpu_sys_reg(vcpu, HCRX_EL2);
441
441
hcrx |= val & __HCRX_EL2_MASK;
442
442
hcrx &= ~(~val & __HCRX_EL2_nMASK);
···
476
476
477
477
write_sysreg_hcr(hcr);
478
478
479
-
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
480
-
write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);
479
+
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) {
480
+
u64 vsesr;
481
+
482
+
/*
483
+
* When HCR_EL2.AMO is set, physical SErrors are taken to EL2
484
+
* and vSError injection is enabled for EL1. Conveniently, for
485
+
* NV this means that it is never the case where a 'physical'
486
+
* SError (injected by KVM or userspace) and vSError are
487
+
* deliverable to the same context.
488
+
*
489
+
* As such, we can trivially select between the host or guest's
490
+
* VSESR_EL2. Except for the case that FEAT_RAS hasn't been
491
+
* exposed to the guest, where ESR propagation in hardware
492
+
* occurs unconditionally.
493
+
*
494
+
* Paper over the architectural wart and use an IMPLEMENTATION
495
+
* DEFINED ESR value in case FEAT_RAS is hidden from the guest.
496
+
*/
497
+
if (!vserror_state_is_nested(vcpu))
498
+
vsesr = vcpu->arch.vsesr_el2;
499
+
else if (kvm_has_ras(kern_hyp_va(vcpu->kvm)))
500
+
vsesr = __vcpu_sys_reg(vcpu, VSESR_EL2);
501
+
else
502
+
vsesr = ESR_ELx_ISV;
503
+
504
+
write_sysreg_s(vsesr, SYS_VSESR_EL2);
505
+
}
481
506
}
482
507
483
508
static inline void ___deactivate_traps(struct kvm_vcpu *vcpu)
484
509
{
510
+
u64 *hcr;
511
+
512
+
if (vserror_state_is_nested(vcpu))
513
+
hcr = __ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2);
514
+
else
515
+
hcr = &vcpu->arch.hcr_el2;
516
+
485
517
/*
486
518
* If we pended a virtual abort, preserve it until it gets
487
519
* cleared. See D1.14.3 (Virtual Interrupts) for details, but
488
520
* the crucial bit is "On taking a vSError interrupt,
489
521
* HCR_EL2.VSE is cleared to 0."
522
+
*
523
+
* Additionally, when in a nested context we need to propagate the
524
+
* updated state to the guest hypervisor's HCR_EL2.
490
525
*/
491
-
if (vcpu->arch.hcr_el2 & HCR_VSE) {
492
-
vcpu->arch.hcr_el2 &= ~HCR_VSE;
493
-
vcpu->arch.hcr_el2 |= read_sysreg(hcr_el2) & HCR_VSE;
526
+
if (*hcr & HCR_VSE) {
527
+
*hcr &= ~HCR_VSE;
528
+
*hcr |= read_sysreg(hcr_el2) & HCR_VSE;
494
529
}
495
530
}
496
531
···
566
531
* nested guest, as the guest hypervisor could select a smaller VL. Slap
567
532
* that into hardware before wrapping up.
568
533
*/
569
-
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))
534
+
if (is_nested_ctxt(vcpu))
570
535
sve_cond_update_zcr_vq(__vcpu_sys_reg(vcpu, ZCR_EL2), SYS_ZCR_EL2);
571
536
572
537
write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR);
···
592
557
593
558
if (vcpu_has_sve(vcpu)) {
594
559
/* A guest hypervisor may restrict the effective max VL. */
595
-
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))
560
+
if (is_nested_ctxt(vcpu))
596
561
zcr_el2 = __vcpu_sys_reg(vcpu, ZCR_EL2);
597
562
else
598
563
zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
+46
-3
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
+46
-3
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
···
109
109
return kvm_has_s1poe(kern_hyp_va(vcpu->kvm));
110
110
}
111
111
112
+
static inline bool ctxt_has_ras(struct kvm_cpu_context *ctxt)
113
+
{
114
+
struct kvm_vcpu *vcpu;
115
+
116
+
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
117
+
return false;
118
+
119
+
vcpu = ctxt_to_vcpu(ctxt);
120
+
return kvm_has_ras(kern_hyp_va(vcpu->kvm));
121
+
}
122
+
123
+
static inline bool ctxt_has_sctlr2(struct kvm_cpu_context *ctxt)
124
+
{
125
+
struct kvm_vcpu *vcpu;
126
+
127
+
if (!cpus_have_final_cap(ARM64_HAS_SCTLR2))
128
+
return false;
129
+
130
+
vcpu = ctxt_to_vcpu(ctxt);
131
+
return kvm_has_sctlr2(kern_hyp_va(vcpu->kvm));
132
+
}
133
+
112
134
static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
113
135
{
114
136
ctxt_sys_reg(ctxt, SCTLR_EL1) = read_sysreg_el1(SYS_SCTLR);
···
169
147
ctxt_sys_reg(ctxt, SP_EL1) = read_sysreg(sp_el1);
170
148
ctxt_sys_reg(ctxt, ELR_EL1) = read_sysreg_el1(SYS_ELR);
171
149
ctxt_sys_reg(ctxt, SPSR_EL1) = read_sysreg_el1(SYS_SPSR);
150
+
151
+
if (ctxt_has_sctlr2(ctxt))
152
+
ctxt_sys_reg(ctxt, SCTLR2_EL1) = read_sysreg_el1(SYS_SCTLR2);
172
153
}
173
154
174
155
static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
···
184
159
if (!has_vhe() && ctxt->__hyp_running_vcpu)
185
160
ctxt->regs.pstate = read_sysreg_el2(SYS_SPSR);
186
161
187
-
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
162
+
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
163
+
return;
164
+
165
+
if (!vserror_state_is_nested(ctxt_to_vcpu(ctxt)))
188
166
ctxt_sys_reg(ctxt, DISR_EL1) = read_sysreg_s(SYS_VDISR_EL2);
167
+
else if (ctxt_has_ras(ctxt))
168
+
ctxt_sys_reg(ctxt, VDISR_EL2) = read_sysreg_s(SYS_VDISR_EL2);
189
169
}
190
170
191
171
static inline void __sysreg_restore_common_state(struct kvm_cpu_context *ctxt)
···
282
252
write_sysreg(ctxt_sys_reg(ctxt, SP_EL1), sp_el1);
283
253
write_sysreg_el1(ctxt_sys_reg(ctxt, ELR_EL1), SYS_ELR);
284
254
write_sysreg_el1(ctxt_sys_reg(ctxt, SPSR_EL1), SYS_SPSR);
255
+
256
+
if (ctxt_has_sctlr2(ctxt))
257
+
write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR2_EL1), SYS_SCTLR2);
285
258
}
286
259
287
260
/* Read the VCPU state's PSTATE, but translate (v)EL2 to EL1. */
···
308
275
{
309
276
u64 pstate = to_hw_pstate(ctxt);
310
277
u64 mode = pstate & PSR_AA32_MODE_MASK;
278
+
u64 vdisr;
311
279
312
280
/*
313
281
* Safety check to ensure we're setting the CPU up to enter the guest
···
327
293
write_sysreg_el2(ctxt->regs.pc, SYS_ELR);
328
294
write_sysreg_el2(pstate, SYS_SPSR);
329
295
330
-
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
331
-
write_sysreg_s(ctxt_sys_reg(ctxt, DISR_EL1), SYS_VDISR_EL2);
296
+
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
297
+
return;
298
+
299
+
if (!vserror_state_is_nested(ctxt_to_vcpu(ctxt)))
300
+
vdisr = ctxt_sys_reg(ctxt, DISR_EL1);
301
+
else if (ctxt_has_ras(ctxt))
302
+
vdisr = ctxt_sys_reg(ctxt, VDISR_EL2);
303
+
else
304
+
vdisr = 0;
305
+
306
+
write_sysreg_s(vdisr, SYS_VDISR_EL2);
332
307
}
333
308
334
309
static inline void __sysreg32_save_state(struct kvm_vcpu *vcpu)
+44
-9
arch/arm64/kvm/hyp/vgic-v3-sr.c
+44
-9
arch/arm64/kvm/hyp/vgic-v3-sr.c
···
296
296
}
297
297
298
298
/*
299
-
* Prevent the guest from touching the ICC_SRE_EL1 system
300
-
* register. Note that this may not have any effect, as
301
-
* ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
299
+
* GICv5 BET0 FEAT_GCIE_LEGACY doesn't include ICC_SRE_EL2. This is due
300
+
* to be relaxed in a future spec release, at which point this in
301
+
* condition can be dropped.
302
302
*/
303
-
write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
304
-
ICC_SRE_EL2);
303
+
if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) {
304
+
/*
305
+
* Prevent the guest from touching the ICC_SRE_EL1 system
306
+
* register. Note that this may not have any effect, as
307
+
* ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
308
+
*/
309
+
write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
310
+
ICC_SRE_EL2);
311
+
}
305
312
306
313
/*
307
314
* If we need to trap system registers, we must write
···
329
322
cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
330
323
}
331
324
332
-
val = read_gicreg(ICC_SRE_EL2);
333
-
write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
325
+
/*
326
+
* Can be dropped in the future when GICv5 spec is relaxed. See comment
327
+
* above.
328
+
*/
329
+
if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) {
330
+
val = read_gicreg(ICC_SRE_EL2);
331
+
write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
332
+
}
334
333
335
334
if (!cpu_if->vgic_sre) {
336
335
/* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
···
436
423
*/
437
424
u64 __vgic_v3_get_gic_config(void)
438
425
{
439
-
u64 val, sre = read_gicreg(ICC_SRE_EL1);
426
+
u64 val, sre;
440
427
unsigned long flags = 0;
441
428
429
+
/*
430
+
* In compat mode, we cannot access ICC_SRE_EL1 at any EL
431
+
* other than EL1 itself; just return the
432
+
* ICH_VTR_EL2. ICC_IDR0_EL1 is only implemented on a GICv5
433
+
* system, so we first check if we have GICv5 support.
434
+
*/
435
+
if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
436
+
return read_gicreg(ICH_VTR_EL2);
437
+
438
+
sre = read_gicreg(ICC_SRE_EL1);
442
439
/*
443
440
* To check whether we have a MMIO-based (GICv2 compatible)
444
441
* CPU interface, we need to disable the system register
···
494
471
return val;
495
472
}
496
473
474
+
static void __vgic_v3_compat_mode_enable(void)
475
+
{
476
+
if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
477
+
return;
478
+
479
+
sysreg_clear_set_s(SYS_ICH_VCTLR_EL2, 0, ICH_VCTLR_EL2_V3);
480
+
/* Wait for V3 to become enabled */
481
+
isb();
482
+
}
483
+
497
484
static u64 __vgic_v3_read_vmcr(void)
498
485
{
499
486
return read_gicreg(ICH_VMCR_EL2);
···
523
490
524
491
void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
525
492
{
493
+
__vgic_v3_compat_mode_enable();
494
+
526
495
/*
527
496
* If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
528
497
* is dependent on ICC_SRE_EL1.SRE, and we have to perform the
···
1085
1050
{
1086
1051
u64 ich_hcr;
1087
1052
1088
-
if (!vcpu_has_nv(vcpu) || is_hyp_ctxt(vcpu))
1053
+
if (!is_nested_ctxt(vcpu))
1089
1054
return false;
1090
1055
1091
1056
ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);
+12
-2
arch/arm64/kvm/hyp/vhe/switch.c
+12
-2
arch/arm64/kvm/hyp/vhe/switch.c
···
48
48
49
49
static u64 __compute_hcr(struct kvm_vcpu *vcpu)
50
50
{
51
-
u64 guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
52
-
u64 hcr = vcpu->arch.hcr_el2;
51
+
u64 guest_hcr, hcr = vcpu->arch.hcr_el2;
53
52
54
53
if (!vcpu_has_nv(vcpu))
55
54
return hcr;
···
67
68
if (!vcpu_el2_e2h_is_set(vcpu))
68
69
hcr |= HCR_NV1;
69
70
71
+
/*
72
+
* Nothing in HCR_EL2 should impact running in hypervisor
73
+
* context, apart from bits we have defined as RESx (E2H,
74
+
* HCD and co), or that cannot be set directly (the EXCLUDE
75
+
* bits). Given that we OR the guest's view with the host's,
76
+
* we can use the 0 value as the starting point, and only
77
+
* use the config-driven RES1 bits.
78
+
*/
79
+
guest_hcr = kvm_vcpu_apply_reg_masks(vcpu, HCR_EL2, 0);
80
+
70
81
write_sysreg_s(vcpu->arch.ctxt.vncr_array, SYS_VNCR_EL2);
71
82
} else {
72
83
host_data_clear_flag(VCPU_IN_HYP_CONTEXT);
73
84
85
+
guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
74
86
if (guest_hcr & HCR_NV) {
75
87
u64 va = __fix_to_virt(vncr_fixmap(smp_processor_id()));
76
88
+6
arch/arm64/kvm/hyp/vhe/sysreg-sr.c
+6
arch/arm64/kvm/hyp/vhe/sysreg-sr.c
···
77
77
__vcpu_assign_sys_reg(vcpu, SP_EL2, read_sysreg(sp_el1));
78
78
__vcpu_assign_sys_reg(vcpu, ELR_EL2, read_sysreg_el1(SYS_ELR));
79
79
__vcpu_assign_sys_reg(vcpu, SPSR_EL2, read_sysreg_el1(SYS_SPSR));
80
+
81
+
if (ctxt_has_sctlr2(&vcpu->arch.ctxt))
82
+
__vcpu_assign_sys_reg(vcpu, SCTLR2_EL2, read_sysreg_el1(SYS_SCTLR2));
80
83
}
81
84
82
85
static void __sysreg_restore_vel2_state(struct kvm_vcpu *vcpu)
···
142
139
write_sysreg(__vcpu_sys_reg(vcpu, SP_EL2), sp_el1);
143
140
write_sysreg_el1(__vcpu_sys_reg(vcpu, ELR_EL2), SYS_ELR);
144
141
write_sysreg_el1(__vcpu_sys_reg(vcpu, SPSR_EL2), SYS_SPSR);
142
+
143
+
if (ctxt_has_sctlr2(&vcpu->arch.ctxt))
144
+
write_sysreg_el1(__vcpu_sys_reg(vcpu, SCTLR2_EL2), SYS_SCTLR2);
145
145
}
146
146
147
147
/*
+160
-75
arch/arm64/kvm/inject_fault.c
+160
-75
arch/arm64/kvm/inject_fault.c
···
15
15
#include <asm/kvm_nested.h>
16
16
#include <asm/esr.h>
17
17
18
-
static void pend_sync_exception(struct kvm_vcpu *vcpu)
18
+
static unsigned int exception_target_el(struct kvm_vcpu *vcpu)
19
19
{
20
20
/* If not nesting, EL1 is the only possible exception target */
21
-
if (likely(!vcpu_has_nv(vcpu))) {
22
-
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
23
-
return;
24
-
}
21
+
if (likely(!vcpu_has_nv(vcpu)))
22
+
return PSR_MODE_EL1h;
25
23
26
24
/*
27
25
* With NV, we need to pick between EL1 and EL2. Note that we
···
30
32
switch(*vcpu_cpsr(vcpu) & PSR_MODE_MASK) {
31
33
case PSR_MODE_EL2h:
32
34
case PSR_MODE_EL2t:
33
-
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
34
-
break;
35
+
return PSR_MODE_EL2h;
35
36
case PSR_MODE_EL1h:
36
37
case PSR_MODE_EL1t:
37
-
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
38
-
break;
38
+
return PSR_MODE_EL1h;
39
39
case PSR_MODE_EL0t:
40
-
if (vcpu_el2_tge_is_set(vcpu))
41
-
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
42
-
else
43
-
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
44
-
break;
40
+
return vcpu_el2_tge_is_set(vcpu) ? PSR_MODE_EL2h : PSR_MODE_EL1h;
45
41
default:
46
42
BUG();
47
43
}
48
44
}
49
45
50
-
static bool match_target_el(struct kvm_vcpu *vcpu, unsigned long target)
46
+
static enum vcpu_sysreg exception_esr_elx(struct kvm_vcpu *vcpu)
51
47
{
52
-
return (vcpu_get_flag(vcpu, EXCEPT_MASK) == target);
48
+
if (exception_target_el(vcpu) == PSR_MODE_EL2h)
49
+
return ESR_EL2;
50
+
51
+
return ESR_EL1;
52
+
}
53
+
54
+
static enum vcpu_sysreg exception_far_elx(struct kvm_vcpu *vcpu)
55
+
{
56
+
if (exception_target_el(vcpu) == PSR_MODE_EL2h)
57
+
return FAR_EL2;
58
+
59
+
return FAR_EL1;
60
+
}
61
+
62
+
static void pend_sync_exception(struct kvm_vcpu *vcpu)
63
+
{
64
+
if (exception_target_el(vcpu) == PSR_MODE_EL1h)
65
+
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
66
+
else
67
+
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
68
+
}
69
+
70
+
static void pend_serror_exception(struct kvm_vcpu *vcpu)
71
+
{
72
+
if (exception_target_el(vcpu) == PSR_MODE_EL1h)
73
+
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SERR);
74
+
else
75
+
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR);
76
+
}
77
+
78
+
static bool __effective_sctlr2_bit(struct kvm_vcpu *vcpu, unsigned int idx)
79
+
{
80
+
u64 sctlr2;
81
+
82
+
if (!kvm_has_sctlr2(vcpu->kvm))
83
+
return false;
84
+
85
+
if (is_nested_ctxt(vcpu) &&
86
+
!(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_SCTLR2En))
87
+
return false;
88
+
89
+
if (exception_target_el(vcpu) == PSR_MODE_EL1h)
90
+
sctlr2 = vcpu_read_sys_reg(vcpu, SCTLR2_EL1);
91
+
else
92
+
sctlr2 = vcpu_read_sys_reg(vcpu, SCTLR2_EL2);
93
+
94
+
return sctlr2 & BIT(idx);
95
+
}
96
+
97
+
static bool effective_sctlr2_ease(struct kvm_vcpu *vcpu)
98
+
{
99
+
return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_EASE_SHIFT);
100
+
}
101
+
102
+
static bool effective_sctlr2_nmea(struct kvm_vcpu *vcpu)
103
+
{
104
+
return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_NMEA_SHIFT);
53
105
}
54
106
55
107
static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr)
···
108
60
bool is_aarch32 = vcpu_mode_is_32bit(vcpu);
109
61
u64 esr = 0;
110
62
111
-
pend_sync_exception(vcpu);
63
+
/* This delight is brought to you by FEAT_DoubleFault2. */
64
+
if (effective_sctlr2_ease(vcpu))
65
+
pend_serror_exception(vcpu);
66
+
else
67
+
pend_sync_exception(vcpu);
112
68
113
69
/*
114
70
* Build an {i,d}abort, depending on the level and the
···
135
83
136
84
esr |= ESR_ELx_FSC_EXTABT;
137
85
138
-
if (match_target_el(vcpu, unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC))) {
139
-
vcpu_write_sys_reg(vcpu, addr, FAR_EL1);
140
-
vcpu_write_sys_reg(vcpu, esr, ESR_EL1);
141
-
} else {
142
-
vcpu_write_sys_reg(vcpu, addr, FAR_EL2);
143
-
vcpu_write_sys_reg(vcpu, esr, ESR_EL2);
144
-
}
86
+
vcpu_write_sys_reg(vcpu, addr, exception_far_elx(vcpu));
87
+
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
145
88
}
146
89
147
90
static void inject_undef64(struct kvm_vcpu *vcpu)
···
152
105
if (kvm_vcpu_trap_il_is32bit(vcpu))
153
106
esr |= ESR_ELx_IL;
154
107
155
-
if (match_target_el(vcpu, unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC)))
156
-
vcpu_write_sys_reg(vcpu, esr, ESR_EL1);
157
-
else
158
-
vcpu_write_sys_reg(vcpu, esr, ESR_EL2);
108
+
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
159
109
}
160
110
161
111
#define DFSR_FSC_EXTABT_LPAE 0x10
···
199
155
vcpu_write_sys_reg(vcpu, far, FAR_EL1);
200
156
}
201
157
202
-
/**
203
-
* kvm_inject_dabt - inject a data abort into the guest
204
-
* @vcpu: The VCPU to receive the data abort
205
-
* @addr: The address to report in the DFAR
206
-
*
207
-
* It is assumed that this code is called from the VCPU thread and that the
208
-
* VCPU therefore is not currently executing guest code.
209
-
*/
210
-
void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr)
158
+
static void __kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
211
159
{
212
160
if (vcpu_el1_is_32bit(vcpu))
213
-
inject_abt32(vcpu, false, addr);
161
+
inject_abt32(vcpu, iabt, addr);
214
162
else
215
-
inject_abt64(vcpu, false, addr);
163
+
inject_abt64(vcpu, iabt, addr);
216
164
}
217
165
218
-
/**
219
-
* kvm_inject_pabt - inject a prefetch abort into the guest
220
-
* @vcpu: The VCPU to receive the prefetch abort
221
-
* @addr: The address to report in the DFAR
222
-
*
223
-
* It is assumed that this code is called from the VCPU thread and that the
224
-
* VCPU therefore is not currently executing guest code.
225
-
*/
226
-
void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr)
166
+
static bool kvm_sea_target_is_el2(struct kvm_vcpu *vcpu)
227
167
{
228
-
if (vcpu_el1_is_32bit(vcpu))
229
-
inject_abt32(vcpu, true, addr);
230
-
else
231
-
inject_abt64(vcpu, true, addr);
168
+
if (__vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_TGE | HCR_TEA))
169
+
return true;
170
+
171
+
if (!vcpu_mode_priv(vcpu))
172
+
return false;
173
+
174
+
return (*vcpu_cpsr(vcpu) & PSR_A_BIT) &&
175
+
(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA);
176
+
}
177
+
178
+
int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
179
+
{
180
+
lockdep_assert_held(&vcpu->mutex);
181
+
182
+
if (is_nested_ctxt(vcpu) && kvm_sea_target_is_el2(vcpu))
183
+
return kvm_inject_nested_sea(vcpu, iabt, addr);
184
+
185
+
__kvm_inject_sea(vcpu, iabt, addr);
186
+
return 1;
232
187
}
233
188
234
189
void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
···
237
194
addr = kvm_vcpu_get_fault_ipa(vcpu);
238
195
addr |= kvm_vcpu_get_hfar(vcpu) & GENMASK(11, 0);
239
196
240
-
if (kvm_vcpu_trap_is_iabt(vcpu))
241
-
kvm_inject_pabt(vcpu, addr);
242
-
else
243
-
kvm_inject_dabt(vcpu, addr);
197
+
__kvm_inject_sea(vcpu, kvm_vcpu_trap_is_iabt(vcpu), addr);
244
198
245
199
/*
246
200
* If AArch64 or LPAE, set FSC to 0 to indicate an Address
···
250
210
!(vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE))
251
211
return;
252
212
253
-
esr = vcpu_read_sys_reg(vcpu, ESR_EL1);
213
+
esr = vcpu_read_sys_reg(vcpu, exception_esr_elx(vcpu));
254
214
esr &= ~GENMASK_ULL(5, 0);
255
-
vcpu_write_sys_reg(vcpu, esr, ESR_EL1);
215
+
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
256
216
}
257
217
258
218
/**
···
270
230
inject_undef64(vcpu);
271
231
}
272
232
273
-
void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 esr)
233
+
static bool serror_is_masked(struct kvm_vcpu *vcpu)
274
234
{
275
-
vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK);
276
-
*vcpu_hcr(vcpu) |= HCR_VSE;
235
+
return (*vcpu_cpsr(vcpu) & PSR_A_BIT) && !effective_sctlr2_nmea(vcpu);
277
236
}
278
237
279
-
/**
280
-
* kvm_inject_vabt - inject an async abort / SError into the guest
281
-
* @vcpu: The VCPU to receive the exception
282
-
*
283
-
* It is assumed that this code is called from the VCPU thread and that the
284
-
* VCPU therefore is not currently executing guest code.
285
-
*
286
-
* Systems with the RAS Extensions specify an imp-def ESR (ISV/IDS = 1) with
287
-
* the remaining ISS all-zeros so that this error is not interpreted as an
288
-
* uncategorized RAS error. Without the RAS Extensions we can't specify an ESR
289
-
* value, so the CPU generates an imp-def value.
290
-
*/
291
-
void kvm_inject_vabt(struct kvm_vcpu *vcpu)
238
+
static bool kvm_serror_target_is_el2(struct kvm_vcpu *vcpu)
292
239
{
293
-
kvm_set_sei_esr(vcpu, ESR_ELx_ISV);
240
+
if (is_hyp_ctxt(vcpu) || vcpu_el2_amo_is_set(vcpu))
241
+
return true;
242
+
243
+
if (!(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA))
244
+
return false;
245
+
246
+
/*
247
+
* In another example where FEAT_DoubleFault2 is entirely backwards,
248
+
* "masked" as it relates to the routing effects of HCRX_EL2.TMEA
249
+
* doesn't consider SCTLR2_EL1.NMEA. That is to say, even if EL1 asked
250
+
* for non-maskable SErrors, the EL2 bit takes priority if A is set.
251
+
*/
252
+
if (vcpu_mode_priv(vcpu))
253
+
return *vcpu_cpsr(vcpu) & PSR_A_BIT;
254
+
255
+
/*
256
+
* Otherwise SErrors are considered unmasked when taken from EL0 and
257
+
* NMEA is set.
258
+
*/
259
+
return serror_is_masked(vcpu);
260
+
}
261
+
262
+
static bool kvm_serror_undeliverable_at_el2(struct kvm_vcpu *vcpu)
263
+
{
264
+
return !(vcpu_el2_tge_is_set(vcpu) || vcpu_el2_amo_is_set(vcpu));
265
+
}
266
+
267
+
int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr)
268
+
{
269
+
lockdep_assert_held(&vcpu->mutex);
270
+
271
+
if (is_nested_ctxt(vcpu) && kvm_serror_target_is_el2(vcpu))
272
+
return kvm_inject_nested_serror(vcpu, esr);
273
+
274
+
if (vcpu_is_el2(vcpu) && kvm_serror_undeliverable_at_el2(vcpu)) {
275
+
vcpu_set_vsesr(vcpu, esr);
276
+
vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
277
+
return 1;
278
+
}
279
+
280
+
/*
281
+
* Emulate the exception entry if SErrors are unmasked. This is useful if
282
+
* the vCPU is in a nested context w/ vSErrors enabled then we've already
283
+
* delegated he hardware vSError context (i.e. HCR_EL2.VSE, VSESR_EL2,
284
+
* VDISR_EL2) to the guest hypervisor.
285
+
*
286
+
* As we're emulating the SError injection we need to explicitly populate
287
+
* ESR_ELx.EC because hardware will not do it on our behalf.
288
+
*/
289
+
if (!serror_is_masked(vcpu)) {
290
+
pend_serror_exception(vcpu);
291
+
esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR);
292
+
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
293
+
return 1;
294
+
}
295
+
296
+
vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK);
297
+
*vcpu_hcr(vcpu) |= HCR_VSE;
298
+
return 1;
294
299
}
+6
-6
arch/arm64/kvm/mmio.c
+6
-6
arch/arm64/kvm/mmio.c
···
72
72
return data;
73
73
}
74
74
75
-
static bool kvm_pending_sync_exception(struct kvm_vcpu *vcpu)
75
+
static bool kvm_pending_external_abort(struct kvm_vcpu *vcpu)
76
76
{
77
77
if (!vcpu_get_flag(vcpu, PENDING_EXCEPTION))
78
78
return false;
···
90
90
switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
91
91
case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC):
92
92
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SYNC):
93
+
case unpack_vcpu_flag(EXCEPT_AA64_EL1_SERR):
94
+
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SERR):
93
95
return true;
94
96
default:
95
97
return false;
···
115
113
* Detect if the MMIO return was already handled or if userspace aborted
116
114
* the MMIO access.
117
115
*/
118
-
if (unlikely(!vcpu->mmio_needed || kvm_pending_sync_exception(vcpu)))
116
+
if (unlikely(!vcpu->mmio_needed || kvm_pending_external_abort(vcpu)))
119
117
return 1;
120
118
121
119
vcpu->mmio_needed = 0;
···
171
169
trace_kvm_mmio_nisv(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu),
172
170
kvm_vcpu_get_hfar(vcpu), fault_ipa);
173
171
174
-
if (vcpu_is_protected(vcpu)) {
175
-
kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
176
-
return 1;
177
-
}
172
+
if (vcpu_is_protected(vcpu))
173
+
return kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
178
174
179
175
if (test_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
180
176
&vcpu->kvm->arch.flags)) {
+70
-35
arch/arm64/kvm/mmu.c
+70
-35
arch/arm64/kvm/mmu.c
···
193
193
return 0;
194
194
}
195
195
196
-
static bool kvm_is_device_pfn(unsigned long pfn)
197
-
{
198
-
return !pfn_is_map_memory(pfn);
199
-
}
200
-
201
196
static void *stage2_memcache_zalloc_page(void *arg)
202
197
{
203
198
struct kvm_mmu_memory_cache *mc = arg;
···
1465
1470
return vma->vm_flags & VM_MTE_ALLOWED;
1466
1471
}
1467
1472
1473
+
static bool kvm_vma_is_cacheable(struct vm_area_struct *vma)
1474
+
{
1475
+
switch (FIELD_GET(PTE_ATTRINDX_MASK, pgprot_val(vma->vm_page_prot))) {
1476
+
case MT_NORMAL_NC:
1477
+
case MT_DEVICE_nGnRnE:
1478
+
case MT_DEVICE_nGnRE:
1479
+
return false;
1480
+
default:
1481
+
return true;
1482
+
}
1483
+
}
1484
+
1468
1485
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
1469
1486
struct kvm_s2_trans *nested,
1470
1487
struct kvm_memory_slot *memslot, unsigned long hva,
···
1484
1477
{
1485
1478
int ret = 0;
1486
1479
bool write_fault, writable, force_pte = false;
1487
-
bool exec_fault, mte_allowed;
1488
-
bool device = false, vfio_allow_any_uc = false;
1480
+
bool exec_fault, mte_allowed, is_vma_cacheable;
1481
+
bool s2_force_noncacheable = false, vfio_allow_any_uc = false;
1489
1482
unsigned long mmu_seq;
1490
1483
phys_addr_t ipa = fault_ipa;
1491
1484
struct kvm *kvm = vcpu->kvm;
···
1499
1492
enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
1500
1493
struct kvm_pgtable *pgt;
1501
1494
struct page *page;
1495
+
vm_flags_t vm_flags;
1502
1496
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED;
1503
1497
1504
1498
if (fault_is_perm)
···
1627
1619
1628
1620
vfio_allow_any_uc = vma->vm_flags & VM_ALLOW_ANY_UNCACHED;
1629
1621
1622
+
vm_flags = vma->vm_flags;
1623
+
1624
+
is_vma_cacheable = kvm_vma_is_cacheable(vma);
1625
+
1630
1626
/* Don't use the VMA after the unlock -- it may have vanished */
1631
1627
vma = NULL;
1632
1628
···
1654
1642
if (is_error_noslot_pfn(pfn))
1655
1643
return -EFAULT;
1656
1644
1657
-
if (kvm_is_device_pfn(pfn)) {
1658
-
/*
1659
-
* If the page was identified as device early by looking at
1660
-
* the VMA flags, vma_pagesize is already representing the
1661
-
* largest quantity we can map. If instead it was mapped
1662
-
* via __kvm_faultin_pfn(), vma_pagesize is set to PAGE_SIZE
1663
-
* and must not be upgraded.
1664
-
*
1665
-
* In both cases, we don't let transparent_hugepage_adjust()
1666
-
* change things at the last minute.
1667
-
*/
1668
-
device = true;
1645
+
/*
1646
+
* Check if this is non-struct page memory PFN, and cannot support
1647
+
* CMOs. It could potentially be unsafe to access as cachable.
1648
+
*/
1649
+
if (vm_flags & (VM_PFNMAP | VM_MIXEDMAP) && !pfn_is_map_memory(pfn)) {
1650
+
if (is_vma_cacheable) {
1651
+
/*
1652
+
* Whilst the VMA owner expects cacheable mapping to this
1653
+
* PFN, hardware also has to support the FWB and CACHE DIC
1654
+
* features.
1655
+
*
1656
+
* ARM64 KVM relies on kernel VA mapping to the PFN to
1657
+
* perform cache maintenance as the CMO instructions work on
1658
+
* virtual addresses. VM_PFNMAP region are not necessarily
1659
+
* mapped to a KVA and hence the presence of hardware features
1660
+
* S2FWB and CACHE DIC are mandatory to avoid the need for
1661
+
* cache maintenance.
1662
+
*/
1663
+
if (!kvm_supports_cacheable_pfnmap())
1664
+
return -EFAULT;
1665
+
} else {
1666
+
/*
1667
+
* If the page was identified as device early by looking at
1668
+
* the VMA flags, vma_pagesize is already representing the
1669
+
* largest quantity we can map. If instead it was mapped
1670
+
* via __kvm_faultin_pfn(), vma_pagesize is set to PAGE_SIZE
1671
+
* and must not be upgraded.
1672
+
*
1673
+
* In both cases, we don't let transparent_hugepage_adjust()
1674
+
* change things at the last minute.
1675
+
*/
1676
+
s2_force_noncacheable = true;
1677
+
}
1669
1678
} else if (logging_active && !write_fault) {
1670
1679
/*
1671
1680
* Only actually map the page as writable if this was a write
···
1695
1662
writable = false;
1696
1663
}
1697
1664
1698
-
if (exec_fault && device)
1665
+
if (exec_fault && s2_force_noncacheable)
1699
1666
return -ENOEXEC;
1700
1667
1701
1668
/*
···
1728
1695
* If we are not forced to use page mapping, check if we are
1729
1696
* backed by a THP and thus use block mapping if possible.
1730
1697
*/
1731
-
if (vma_pagesize == PAGE_SIZE && !(force_pte || device)) {
1698
+
if (vma_pagesize == PAGE_SIZE && !(force_pte || s2_force_noncacheable)) {
1732
1699
if (fault_is_perm && fault_granule > PAGE_SIZE)
1733
1700
vma_pagesize = fault_granule;
1734
1701
else
···
1742
1709
}
1743
1710
}
1744
1711
1745
-
if (!fault_is_perm && !device && kvm_has_mte(kvm)) {
1712
+
if (!fault_is_perm && !s2_force_noncacheable && kvm_has_mte(kvm)) {
1746
1713
/* Check the VMM hasn't introduced a new disallowed VMA */
1747
1714
if (mte_allowed) {
1748
1715
sanitise_mte_tags(kvm, pfn, vma_pagesize);
···
1758
1725
if (exec_fault)
1759
1726
prot |= KVM_PGTABLE_PROT_X;
1760
1727
1761
-
if (device) {
1728
+
if (s2_force_noncacheable) {
1762
1729
if (vfio_allow_any_uc)
1763
1730
prot |= KVM_PGTABLE_PROT_NORMAL_NC;
1764
1731
else
···
1841
1808
* There is no need to pass the error into the guest.
1842
1809
*/
1843
1810
if (kvm_handle_guest_sea())
1844
-
kvm_inject_vabt(vcpu);
1811
+
return kvm_inject_serror(vcpu);
1845
1812
1846
1813
return 1;
1847
1814
}
···
1869
1836
if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(vcpu->arch.hw_mmu->vtcr))) {
1870
1837
fault_ipa |= kvm_vcpu_get_hfar(vcpu) & GENMASK(11, 0);
1871
1838
1872
-
if (is_iabt)
1873
-
kvm_inject_pabt(vcpu, fault_ipa);
1874
-
else
1875
-
kvm_inject_dabt(vcpu, fault_ipa);
1876
-
return 1;
1839
+
return kvm_inject_sea(vcpu, is_iabt, fault_ipa);
1877
1840
}
1878
1841
}
1879
1842
···
1941
1912
}
1942
1913
1943
1914
if (kvm_vcpu_abt_iss1tw(vcpu)) {
1944
-
kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
1945
-
ret = 1;
1915
+
ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
1946
1916
goto out_unlock;
1947
1917
}
1948
1918
···
1986
1958
if (ret == 0)
1987
1959
ret = 1;
1988
1960
out:
1989
-
if (ret == -ENOEXEC) {
1990
-
kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu));
1991
-
ret = 1;
1992
-
}
1961
+
if (ret == -ENOEXEC)
1962
+
ret = kvm_inject_sea_iabt(vcpu, kvm_vcpu_get_hfar(vcpu));
1993
1963
out_unlock:
1994
1964
srcu_read_unlock(&vcpu->kvm->srcu, idx);
1995
1965
return ret;
···
2244
2218
if (vma->vm_flags & VM_PFNMAP) {
2245
2219
/* IO region dirty page logging not allowed */
2246
2220
if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
2221
+
ret = -EINVAL;
2222
+
break;
2223
+
}
2224
+
2225
+
/*
2226
+
* Cacheable PFNMAP is allowed only if the hardware
2227
+
* supports it.
2228
+
*/
2229
+
if (kvm_vma_is_cacheable(vma) && !kvm_supports_cacheable_pfnmap()) {
2247
2230
ret = -EINVAL;
2248
2231
break;
2249
2232
}
+50
-59
arch/arm64/kvm/nested.c
+50
-59
arch/arm64/kvm/nested.c
···
1441
1441
break;
1442
1442
1443
1443
case SYS_ID_AA64PFR0_EL1:
1444
-
/* No RME, AMU, MPAM, S-EL2, or RAS */
1444
+
/* No RME, AMU, MPAM, or S-EL2 */
1445
1445
val &= ~(ID_AA64PFR0_EL1_RME |
1446
1446
ID_AA64PFR0_EL1_AMU |
1447
1447
ID_AA64PFR0_EL1_MPAM |
1448
1448
ID_AA64PFR0_EL1_SEL2 |
1449
-
ID_AA64PFR0_EL1_RAS |
1450
1449
ID_AA64PFR0_EL1_EL3 |
1451
1450
ID_AA64PFR0_EL1_EL2 |
1452
1451
ID_AA64PFR0_EL1_EL1 |
···
1682
1683
set_sysreg_masks(kvm, HFGITR2_EL2, res0, res1);
1683
1684
1684
1685
/* TCR2_EL2 */
1685
-
res0 = TCR2_EL2_RES0;
1686
-
res1 = TCR2_EL2_RES1;
1687
-
if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP))
1688
-
res0 |= (TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1 | TCR2_EL2_D128);
1689
-
if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, MEC, IMP))
1690
-
res0 |= TCR2_EL2_AMEC1 | TCR2_EL2_AMEC0;
1691
-
if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, HAFDBS, HAFT))
1692
-
res0 |= TCR2_EL2_HAFT;
1693
-
if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP))
1694
-
res0 |= TCR2_EL2_PTTWI | TCR2_EL2_PnCH;
1695
-
if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP))
1696
-
res0 |= TCR2_EL2_AIE;
1697
-
if (!kvm_has_s1poe(kvm))
1698
-
res0 |= TCR2_EL2_POE | TCR2_EL2_E0POE;
1699
-
if (!kvm_has_s1pie(kvm))
1700
-
res0 |= TCR2_EL2_PIE;
1701
-
if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP))
1702
-
res0 |= (TCR2_EL2_E0POE | TCR2_EL2_D128 |
1703
-
TCR2_EL2_AMEC1 | TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1);
1686
+
get_reg_fixed_bits(kvm, TCR2_EL2, &res0, &res1);
1704
1687
set_sysreg_masks(kvm, TCR2_EL2, res0, res1);
1705
1688
1706
1689
/* SCTLR_EL1 */
1707
-
res0 = SCTLR_EL1_RES0;
1708
-
res1 = SCTLR_EL1_RES1;
1709
-
if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN3))
1710
-
res0 |= SCTLR_EL1_EPAN;
1690
+
get_reg_fixed_bits(kvm, SCTLR_EL1, &res0, &res1);
1711
1691
set_sysreg_masks(kvm, SCTLR_EL1, res0, res1);
1712
1692
1693
+
/* SCTLR2_ELx */
1694
+
get_reg_fixed_bits(kvm, SCTLR2_EL1, &res0, &res1);
1695
+
set_sysreg_masks(kvm, SCTLR2_EL1, res0, res1);
1696
+
get_reg_fixed_bits(kvm, SCTLR2_EL2, &res0, &res1);
1697
+
set_sysreg_masks(kvm, SCTLR2_EL2, res0, res1);
1698
+
1713
1699
/* MDCR_EL2 */
1714
-
res0 = MDCR_EL2_RES0;
1715
-
res1 = MDCR_EL2_RES1;
1716
-
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP))
1717
-
res0 |= (MDCR_EL2_HPMN | MDCR_EL2_TPMCR |
1718
-
MDCR_EL2_TPM | MDCR_EL2_HPME);
1719
-
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP))
1720
-
res0 |= MDCR_EL2_E2PB | MDCR_EL2_TPMS;
1721
-
if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, SPMU, IMP))
1722
-
res0 |= MDCR_EL2_EnSPM;
1723
-
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P1))
1724
-
res0 |= MDCR_EL2_HPMD;
1725
-
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP))
1726
-
res0 |= MDCR_EL2_TTRF;
1727
-
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P5))
1728
-
res0 |= MDCR_EL2_HCCD | MDCR_EL2_HLP;
1729
-
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP))
1730
-
res0 |= MDCR_EL2_E2TB;
1731
-
if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, FGT, IMP))
1732
-
res0 |= MDCR_EL2_TDCC;
1733
-
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, MTPMU, IMP) ||
1734
-
kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL3, IMP))
1735
-
res0 |= MDCR_EL2_MTPME;
1736
-
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P7))
1737
-
res0 |= MDCR_EL2_HPMFZO;
1738
-
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSS, IMP))
1739
-
res0 |= MDCR_EL2_PMSSE;
1740
-
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2))
1741
-
res0 |= MDCR_EL2_HPMFZS;
1742
-
if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, EBEP, IMP))
1743
-
res0 |= MDCR_EL2_PMEE;
1744
-
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, V8P9))
1745
-
res0 |= MDCR_EL2_EBWE;
1746
-
if (!kvm_has_feat(kvm, ID_AA64DFR2_EL1, STEP, IMP))
1747
-
res0 |= MDCR_EL2_EnSTEPOP;
1700
+
get_reg_fixed_bits(kvm, MDCR_EL2, &res0, &res1);
1748
1701
set_sysreg_masks(kvm, MDCR_EL2, res0, res1);
1749
1702
1750
1703
/* CNTHCTL_EL2 */
···
1752
1801
/* Must be last, as may switch context! */
1753
1802
if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu))
1754
1803
kvm_inject_nested_irq(vcpu);
1804
+
}
1805
+
1806
+
/*
1807
+
* One of the many architectural bugs in FEAT_NV2 is that the guest hypervisor
1808
+
* can write to HCR_EL2 behind our back, potentially changing the exception
1809
+
* routing / masking for even the host context.
1810
+
*
1811
+
* What follows is some slop to (1) react to exception routing / masking and (2)
1812
+
* preserve the pending SError state across translation regimes.
1813
+
*/
1814
+
void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu)
1815
+
{
1816
+
if (!vcpu_has_nv(vcpu))
1817
+
return;
1818
+
1819
+
if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
1820
+
kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
1821
+
}
1822
+
1823
+
void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu)
1824
+
{
1825
+
unsigned long *hcr = vcpu_hcr(vcpu);
1826
+
1827
+
if (!vcpu_has_nv(vcpu))
1828
+
return;
1829
+
1830
+
/*
1831
+
* We previously decided that an SError was deliverable to the guest.
1832
+
* Reap the pending state from HCR_EL2 and...
1833
+
*/
1834
+
if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr)))
1835
+
vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
1836
+
1837
+
/*
1838
+
* Re-attempt SError injection in case the deliverability has changed,
1839
+
* which is necessary to faithfully emulate WFI the case of a pending
1840
+
* SError being a wakeup condition.
1841
+
*/
1842
+
if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
1843
+
kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
1755
1844
}
+143
-62
arch/arm64/kvm/sys_regs.c
+143
-62
arch/arm64/kvm/sys_regs.c
···
108
108
PURE_EL2_SYSREG( HACR_EL2 );
109
109
PURE_EL2_SYSREG( VTTBR_EL2 );
110
110
PURE_EL2_SYSREG( VTCR_EL2 );
111
-
PURE_EL2_SYSREG( RVBAR_EL2 );
112
111
PURE_EL2_SYSREG( TPIDR_EL2 );
113
112
PURE_EL2_SYSREG( HPFAR_EL2 );
114
113
PURE_EL2_SYSREG( HCRX_EL2 );
···
143
144
MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );
144
145
MAPPED_EL2_SYSREG(ZCR_EL2, ZCR_EL1, NULL );
145
146
MAPPED_EL2_SYSREG(CONTEXTIDR_EL2, CONTEXTIDR_EL1, NULL );
147
+
MAPPED_EL2_SYSREG(SCTLR2_EL2, SCTLR2_EL1, NULL );
146
148
default:
147
149
return false;
148
150
}
···
533
533
return ignore_write(vcpu, p);
534
534
535
535
if (p->Op1 == 4) { /* ICC_SRE_EL2 */
536
-
p->regval = (ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_SRE |
537
-
ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB);
536
+
p->regval = KVM_ICC_SRE_EL2;
538
537
} else { /* ICC_SRE_EL1 */
539
538
p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
540
539
}
···
770
771
vcpu_write_sys_reg(vcpu, mpidr, MPIDR_EL1);
771
772
772
773
return mpidr;
774
+
}
775
+
776
+
static unsigned int hidden_visibility(const struct kvm_vcpu *vcpu,
777
+
const struct sys_reg_desc *r)
778
+
{
779
+
return REG_HIDDEN;
773
780
}
774
781
775
782
static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
···
1617
1612
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_GCS);
1618
1613
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_THE);
1619
1614
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTEX);
1620
-
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_DF2);
1621
1615
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_PFAR);
1622
1616
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MPAM_frac);
1623
1617
break;
···
1647
1643
val &= ~ID_AA64MMFR2_EL1_NV;
1648
1644
break;
1649
1645
case SYS_ID_AA64MMFR3_EL1:
1650
-
val &= ID_AA64MMFR3_EL1_TCRX | ID_AA64MMFR3_EL1_S1POE |
1651
-
ID_AA64MMFR3_EL1_S1PIE;
1646
+
val &= ID_AA64MMFR3_EL1_TCRX |
1647
+
ID_AA64MMFR3_EL1_SCTLRX |
1648
+
ID_AA64MMFR3_EL1_S1POE |
1649
+
ID_AA64MMFR3_EL1_S1PIE;
1652
1650
break;
1653
1651
case SYS_ID_MMFR4_EL1:
1654
1652
val &= ~ARM64_FEATURE_MASK(ID_MMFR4_EL1_CCIDX);
···
1817
1811
val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV3, IMP);
1818
1812
}
1819
1813
1820
-
if (kvm_vgic_global_state.type == VGIC_V3) {
1814
+
if (vgic_is_v3(vcpu->kvm)) {
1821
1815
val &= ~ID_AA64PFR0_EL1_GIC_MASK;
1822
1816
val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP);
1823
1817
}
···
1957
1951
if (!FIELD_GET(ID_AA64PFR0_EL1_EL0, user_val) ||
1958
1952
!FIELD_GET(ID_AA64PFR0_EL1_EL1, user_val) ||
1959
1953
(vcpu_has_nv(vcpu) && !FIELD_GET(ID_AA64PFR0_EL1_EL2, user_val)))
1954
+
return -EINVAL;
1955
+
1956
+
/*
1957
+
* If we are running on a GICv5 host and support FEAT_GCIE_LEGACY, then
1958
+
* we support GICv3. Fail attempts to do anything but set that to IMP.
1959
+
*/
1960
+
if (vgic_is_v3_compat(vcpu->kvm) &&
1961
+
FIELD_GET(ID_AA64PFR0_EL1_GIC_MASK, user_val) != ID_AA64PFR0_EL1_GIC_IMP)
1960
1962
return -EINVAL;
1961
1963
1962
1964
return set_id_reg(vcpu, rd, user_val);
···
2339
2325
EL2_REG_FILTERED(name, acc, rst, v, el2_visibility)
2340
2326
2341
2327
#define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v)
2328
+
#define EL2_REG_VNCR_FILT(name, vis) \
2329
+
EL2_REG_FILTERED(name, bad_vncr_trap, reset_val, 0, vis)
2330
+
#define EL2_REG_VNCR_GICv3(name) \
2331
+
EL2_REG_VNCR_FILT(name, hidden_visibility)
2342
2332
#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
2343
2333
2344
2334
/*
···
2501
2483
return REG_HIDDEN;
2502
2484
}
2503
2485
2486
+
static unsigned int sctlr2_visibility(const struct kvm_vcpu *vcpu,
2487
+
const struct sys_reg_desc *rd)
2488
+
{
2489
+
if (kvm_has_sctlr2(vcpu->kvm))
2490
+
return 0;
2491
+
2492
+
return REG_HIDDEN;
2493
+
}
2494
+
2495
+
static unsigned int sctlr2_el2_visibility(const struct kvm_vcpu *vcpu,
2496
+
const struct sys_reg_desc *rd)
2497
+
{
2498
+
return __el2_visibility(vcpu, rd, sctlr2_visibility);
2499
+
}
2500
+
2504
2501
static bool access_zcr_el2(struct kvm_vcpu *vcpu,
2505
2502
struct sys_reg_params *p,
2506
2503
const struct sys_reg_desc *r)
···
2546
2513
if (p->is_write)
2547
2514
return write_to_read_only(vcpu, p, r);
2548
2515
2549
-
p->regval = kvm_vgic_global_state.ich_vtr_el2;
2550
-
p->regval &= ~(ICH_VTR_EL2_DVIM |
2551
-
ICH_VTR_EL2_A3V |
2552
-
ICH_VTR_EL2_IDbits);
2553
-
p->regval |= ICH_VTR_EL2_nV4;
2516
+
p->regval = kvm_get_guest_vtr_el2();
2554
2517
2555
2518
return true;
2556
2519
}
···
2617
2588
return __el2_visibility(vcpu, rd, tcr2_visibility);
2618
2589
}
2619
2590
2591
+
static unsigned int fgt2_visibility(const struct kvm_vcpu *vcpu,
2592
+
const struct sys_reg_desc *rd)
2593
+
{
2594
+
if (el2_visibility(vcpu, rd) == 0 &&
2595
+
kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, FGT, FGT2))
2596
+
return 0;
2597
+
2598
+
return REG_HIDDEN;
2599
+
}
2600
+
2601
+
static unsigned int fgt_visibility(const struct kvm_vcpu *vcpu,
2602
+
const struct sys_reg_desc *rd)
2603
+
{
2604
+
if (el2_visibility(vcpu, rd) == 0 &&
2605
+
kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, FGT, IMP))
2606
+
return 0;
2607
+
2608
+
return REG_HIDDEN;
2609
+
}
2610
+
2620
2611
static unsigned int s1pie_visibility(const struct kvm_vcpu *vcpu,
2621
2612
const struct sys_reg_desc *rd)
2622
2613
{
···
2686
2637
kvm_make_request(KVM_REQ_RELOAD_PMU, vcpu);
2687
2638
2688
2639
return true;
2640
+
}
2641
+
2642
+
static bool access_ras(struct kvm_vcpu *vcpu,
2643
+
struct sys_reg_params *p,
2644
+
const struct sys_reg_desc *r)
2645
+
{
2646
+
struct kvm *kvm = vcpu->kvm;
2647
+
2648
+
switch(reg_to_encoding(r)) {
2649
+
default:
2650
+
if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) {
2651
+
kvm_inject_undefined(vcpu);
2652
+
return false;
2653
+
}
2654
+
}
2655
+
2656
+
return trap_raz_wi(vcpu, p, r);
2689
2657
}
2690
2658
2691
2659
/*
···
2932
2866
ID_AA64PFR0_EL1_FP)),
2933
2867
ID_FILTERED(ID_AA64PFR1_EL1, id_aa64pfr1_el1,
2934
2868
~(ID_AA64PFR1_EL1_PFAR |
2935
-
ID_AA64PFR1_EL1_DF2 |
2936
2869
ID_AA64PFR1_EL1_MTEX |
2937
2870
ID_AA64PFR1_EL1_THE |
2938
2871
ID_AA64PFR1_EL1_GCS |
···
3010
2945
ID_AA64MMFR2_EL1_NV |
3011
2946
ID_AA64MMFR2_EL1_CCIDX)),
3012
2947
ID_WRITABLE(ID_AA64MMFR3_EL1, (ID_AA64MMFR3_EL1_TCRX |
2948
+
ID_AA64MMFR3_EL1_SCTLRX |
3013
2949
ID_AA64MMFR3_EL1_S1PIE |
3014
2950
ID_AA64MMFR3_EL1_S1POE)),
3015
2951
ID_WRITABLE(ID_AA64MMFR4_EL1, ID_AA64MMFR4_EL1_NV_frac),
···
3021
2955
{ SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
3022
2956
{ SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
3023
2957
{ SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
2958
+
{ SYS_DESC(SYS_SCTLR2_EL1), access_vm_reg, reset_val, SCTLR2_EL1, 0,
2959
+
.visibility = sctlr2_visibility },
3024
2960
3025
2961
MTE_REG(RGSR_EL1),
3026
2962
MTE_REG(GCR_EL1),
···
3052
2984
{ SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
3053
2985
{ SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
3054
2986
3055
-
{ SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
3056
-
{ SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
3057
-
{ SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
3058
-
{ SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
3059
-
{ SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
3060
-
{ SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
3061
-
{ SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
3062
-
{ SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
2987
+
{ SYS_DESC(SYS_ERRIDR_EL1), access_ras },
2988
+
{ SYS_DESC(SYS_ERRSELR_EL1), access_ras },
2989
+
{ SYS_DESC(SYS_ERXFR_EL1), access_ras },
2990
+
{ SYS_DESC(SYS_ERXCTLR_EL1), access_ras },
2991
+
{ SYS_DESC(SYS_ERXSTATUS_EL1), access_ras },
2992
+
{ SYS_DESC(SYS_ERXADDR_EL1), access_ras },
2993
+
{ SYS_DESC(SYS_ERXMISC0_EL1), access_ras },
2994
+
{ SYS_DESC(SYS_ERXMISC1_EL1), access_ras },
3063
2995
3064
2996
MTE_REG(TFSR_EL1),
3065
2997
MTE_REG(TFSRE0_EL1),
···
3370
3302
EL2_REG_VNCR(VMPIDR_EL2, reset_unknown, 0),
3371
3303
EL2_REG(SCTLR_EL2, access_rw, reset_val, SCTLR_EL2_RES1),
3372
3304
EL2_REG(ACTLR_EL2, access_rw, reset_val, 0),
3305
+
EL2_REG_FILTERED(SCTLR2_EL2, access_vm_reg, reset_val, 0,
3306
+
sctlr2_el2_visibility),
3373
3307
EL2_REG_VNCR(HCR_EL2, reset_hcr, 0),
3374
3308
EL2_REG(MDCR_EL2, access_mdcr, reset_mdcr, 0),
3375
3309
EL2_REG(CPTR_EL2, access_rw, reset_val, CPTR_NVHE_EL2_RES1),
3376
3310
EL2_REG_VNCR(HSTR_EL2, reset_val, 0),
3377
-
EL2_REG_VNCR(HFGRTR_EL2, reset_val, 0),
3378
-
EL2_REG_VNCR(HFGWTR_EL2, reset_val, 0),
3311
+
EL2_REG_VNCR_FILT(HFGRTR_EL2, fgt_visibility),
3312
+
EL2_REG_VNCR_FILT(HFGWTR_EL2, fgt_visibility),
3379
3313
EL2_REG_VNCR(HFGITR_EL2, reset_val, 0),
3380
3314
EL2_REG_VNCR(HACR_EL2, reset_val, 0),
3381
3315
···
3397
3327
vncr_el2_visibility),
3398
3328
3399
3329
{ SYS_DESC(SYS_DACR32_EL2), undef_access, reset_unknown, DACR32_EL2 },
3400
-
EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0),
3401
-
EL2_REG_VNCR(HDFGWTR_EL2, reset_val, 0),
3402
-
EL2_REG_VNCR(HAFGRTR_EL2, reset_val, 0),
3330
+
EL2_REG_VNCR_FILT(HDFGRTR2_EL2, fgt2_visibility),
3331
+
EL2_REG_VNCR_FILT(HDFGWTR2_EL2, fgt2_visibility),
3332
+
EL2_REG_VNCR_FILT(HFGRTR2_EL2, fgt2_visibility),
3333
+
EL2_REG_VNCR_FILT(HFGWTR2_EL2, fgt2_visibility),
3334
+
EL2_REG_VNCR_FILT(HDFGRTR_EL2, fgt_visibility),
3335
+
EL2_REG_VNCR_FILT(HDFGWTR_EL2, fgt_visibility),
3336
+
EL2_REG_VNCR_FILT(HAFGRTR_EL2, fgt_visibility),
3337
+
EL2_REG_VNCR_FILT(HFGITR2_EL2, fgt2_visibility),
3403
3338
EL2_REG_REDIR(SPSR_EL2, reset_val, 0),
3404
3339
EL2_REG_REDIR(ELR_EL2, reset_val, 0),
3405
3340
{ SYS_DESC(SYS_SP_EL1), access_sp_el1},
···
3419
3344
EL2_REG(AFSR0_EL2, access_rw, reset_val, 0),
3420
3345
EL2_REG(AFSR1_EL2, access_rw, reset_val, 0),
3421
3346
EL2_REG_REDIR(ESR_EL2, reset_val, 0),
3347
+
EL2_REG_VNCR(VSESR_EL2, reset_unknown, 0),
3422
3348
{ SYS_DESC(SYS_FPEXC32_EL2), undef_access, reset_val, FPEXC32_EL2, 0x700 },
3423
3349
3424
3350
EL2_REG_REDIR(FAR_EL2, reset_val, 0),
···
3446
3370
{ SYS_DESC(SYS_MPAMVPM7_EL2), undef_access },
3447
3371
3448
3372
EL2_REG(VBAR_EL2, access_rw, reset_val, 0),
3449
-
EL2_REG(RVBAR_EL2, access_rw, reset_val, 0),
3373
+
{ SYS_DESC(SYS_RVBAR_EL2), undef_access },
3450
3374
{ SYS_DESC(SYS_RMR_EL2), undef_access },
3375
+
EL2_REG_VNCR(VDISR_EL2, reset_unknown, 0),
3451
3376
3452
-
EL2_REG_VNCR(ICH_AP0R0_EL2, reset_val, 0),
3453
-
EL2_REG_VNCR(ICH_AP0R1_EL2, reset_val, 0),
3454
-
EL2_REG_VNCR(ICH_AP0R2_EL2, reset_val, 0),
3455
-
EL2_REG_VNCR(ICH_AP0R3_EL2, reset_val, 0),
3456
-
EL2_REG_VNCR(ICH_AP1R0_EL2, reset_val, 0),
3457
-
EL2_REG_VNCR(ICH_AP1R1_EL2, reset_val, 0),
3458
-
EL2_REG_VNCR(ICH_AP1R2_EL2, reset_val, 0),
3459
-
EL2_REG_VNCR(ICH_AP1R3_EL2, reset_val, 0),
3377
+
EL2_REG_VNCR_GICv3(ICH_AP0R0_EL2),
3378
+
EL2_REG_VNCR_GICv3(ICH_AP0R1_EL2),
3379
+
EL2_REG_VNCR_GICv3(ICH_AP0R2_EL2),
3380
+
EL2_REG_VNCR_GICv3(ICH_AP0R3_EL2),
3381
+
EL2_REG_VNCR_GICv3(ICH_AP1R0_EL2),
3382
+
EL2_REG_VNCR_GICv3(ICH_AP1R1_EL2),
3383
+
EL2_REG_VNCR_GICv3(ICH_AP1R2_EL2),
3384
+
EL2_REG_VNCR_GICv3(ICH_AP1R3_EL2),
3460
3385
3461
3386
{ SYS_DESC(SYS_ICC_SRE_EL2), access_gic_sre },
3462
3387
3463
-
EL2_REG_VNCR(ICH_HCR_EL2, reset_val, 0),
3388
+
EL2_REG_VNCR_GICv3(ICH_HCR_EL2),
3464
3389
{ SYS_DESC(SYS_ICH_VTR_EL2), access_gic_vtr },
3465
3390
{ SYS_DESC(SYS_ICH_MISR_EL2), access_gic_misr },
3466
3391
{ SYS_DESC(SYS_ICH_EISR_EL2), access_gic_eisr },
3467
3392
{ SYS_DESC(SYS_ICH_ELRSR_EL2), access_gic_elrsr },
3468
-
EL2_REG_VNCR(ICH_VMCR_EL2, reset_val, 0),
3393
+
EL2_REG_VNCR_GICv3(ICH_VMCR_EL2),
3469
3394
3470
-
EL2_REG_VNCR(ICH_LR0_EL2, reset_val, 0),
3471
-
EL2_REG_VNCR(ICH_LR1_EL2, reset_val, 0),
3472
-
EL2_REG_VNCR(ICH_LR2_EL2, reset_val, 0),
3473
-
EL2_REG_VNCR(ICH_LR3_EL2, reset_val, 0),
3474
-
EL2_REG_VNCR(ICH_LR4_EL2, reset_val, 0),
3475
-
EL2_REG_VNCR(ICH_LR5_EL2, reset_val, 0),
3476
-
EL2_REG_VNCR(ICH_LR6_EL2, reset_val, 0),
3477
-
EL2_REG_VNCR(ICH_LR7_EL2, reset_val, 0),
3478
-
EL2_REG_VNCR(ICH_LR8_EL2, reset_val, 0),
3479
-
EL2_REG_VNCR(ICH_LR9_EL2, reset_val, 0),
3480
-
EL2_REG_VNCR(ICH_LR10_EL2, reset_val, 0),
3481
-
EL2_REG_VNCR(ICH_LR11_EL2, reset_val, 0),
3482
-
EL2_REG_VNCR(ICH_LR12_EL2, reset_val, 0),
3483
-
EL2_REG_VNCR(ICH_LR13_EL2, reset_val, 0),
3484
-
EL2_REG_VNCR(ICH_LR14_EL2, reset_val, 0),
3485
-
EL2_REG_VNCR(ICH_LR15_EL2, reset_val, 0),
3395
+
EL2_REG_VNCR_GICv3(ICH_LR0_EL2),
3396
+
EL2_REG_VNCR_GICv3(ICH_LR1_EL2),
3397
+
EL2_REG_VNCR_GICv3(ICH_LR2_EL2),
3398
+
EL2_REG_VNCR_GICv3(ICH_LR3_EL2),
3399
+
EL2_REG_VNCR_GICv3(ICH_LR4_EL2),
3400
+
EL2_REG_VNCR_GICv3(ICH_LR5_EL2),
3401
+
EL2_REG_VNCR_GICv3(ICH_LR6_EL2),
3402
+
EL2_REG_VNCR_GICv3(ICH_LR7_EL2),
3403
+
EL2_REG_VNCR_GICv3(ICH_LR8_EL2),
3404
+
EL2_REG_VNCR_GICv3(ICH_LR9_EL2),
3405
+
EL2_REG_VNCR_GICv3(ICH_LR10_EL2),
3406
+
EL2_REG_VNCR_GICv3(ICH_LR11_EL2),
3407
+
EL2_REG_VNCR_GICv3(ICH_LR12_EL2),
3408
+
EL2_REG_VNCR_GICv3(ICH_LR13_EL2),
3409
+
EL2_REG_VNCR_GICv3(ICH_LR14_EL2),
3410
+
EL2_REG_VNCR_GICv3(ICH_LR15_EL2),
3486
3411
3487
3412
EL2_REG(CONTEXTIDR_EL2, access_rw, reset_val, 0),
3488
3413
EL2_REG(TPIDR_EL2, access_rw, reset_val, 0),
···
4352
4275
};
4353
4276
4354
4277
static bool check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
4355
-
bool is_32)
4278
+
bool reset_check)
4356
4279
{
4357
4280
unsigned int i;
4358
4281
4359
4282
for (i = 0; i < n; i++) {
4360
-
if (!is_32 && table[i].reg && !table[i].reset) {
4283
+
if (reset_check && table[i].reg && !table[i].reset) {
4361
4284
kvm_err("sys_reg table %pS entry %d (%s) lacks reset\n",
4362
4285
&table[i], i, table[i].name);
4363
4286
return false;
···
4552
4475
return true;
4553
4476
4554
4477
kvm_pr_unimpl("Unhandled cp10 register %s: %u\n",
4555
-
params->is_write ? "write" : "read", reg_id);
4478
+
str_write_read(params->is_write), reg_id);
4556
4479
return false;
4557
4480
}
4558
4481
···
5346
5269
5347
5270
int __init kvm_sys_reg_table_init(void)
5348
5271
{
5272
+
const struct sys_reg_desc *gicv3_regs;
5349
5273
bool valid = true;
5350
-
unsigned int i;
5274
+
unsigned int i, sz;
5351
5275
int ret = 0;
5352
5276
5353
5277
/* Make sure tables are unique and in order. */
5354
-
valid &= check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false);
5355
-
valid &= check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true);
5356
-
valid &= check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true);
5357
-
valid &= check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true);
5358
-
valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true);
5278
+
valid &= check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), true);
5279
+
valid &= check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), false);
5280
+
valid &= check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), false);
5281
+
valid &= check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), false);
5282
+
valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), false);
5359
5283
valid &= check_sysreg_table(sys_insn_descs, ARRAY_SIZE(sys_insn_descs), false);
5284
+
5285
+
gicv3_regs = vgic_v3_get_sysreg_table(&sz);
5286
+
valid &= check_sysreg_table(gicv3_regs, sz, false);
5360
5287
5361
5288
if (!valid)
5362
5289
return -EINVAL;
+1
-1
arch/arm64/kvm/sys_regs.h
+1
-1
arch/arm64/kvm/sys_regs.h
···
108
108
/* Look, we even formatted it for you to paste into the table! */
109
109
kvm_pr_unimpl("%pV { Op0(%2u), Op1(%2u), CRn(%2u), CRm(%2u), Op2(%2u), func_%s },\n",
110
110
&(struct va_format){ fmt, &va },
111
-
p->Op0, p->Op1, p->CRn, p->CRm, p->Op2, p->is_write ? "write" : "read");
111
+
p->Op0, p->Op1, p->CRn, p->CRm, p->Op2, str_write_read(p->is_write));
112
112
va_end(va);
113
113
}
114
114
+1
-1
arch/arm64/kvm/trace_handle_exit.h
+1
-1
arch/arm64/kvm/trace_handle_exit.h
+125
-2
arch/arm64/kvm/vgic-sys-reg-v3.c
+125
-2
arch/arm64/kvm/vgic-sys-reg-v3.c
···
297
297
return 0;
298
298
}
299
299
300
+
static int set_gic_ich_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
301
+
u64 val)
302
+
{
303
+
__vcpu_assign_sys_reg(vcpu, r->reg, val);
304
+
return 0;
305
+
}
306
+
307
+
static int get_gic_ich_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
308
+
u64 *val)
309
+
{
310
+
*val = __vcpu_sys_reg(vcpu, r->reg);
311
+
return 0;
312
+
}
313
+
314
+
static int set_gic_ich_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
315
+
u64 val)
316
+
{
317
+
u8 idx = r->Op2 & 3;
318
+
319
+
if (idx > vgic_v3_max_apr_idx(vcpu))
320
+
return -EINVAL;
321
+
322
+
return set_gic_ich_reg(vcpu, r, val);
323
+
}
324
+
325
+
static int get_gic_ich_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
326
+
u64 *val)
327
+
{
328
+
u8 idx = r->Op2 & 3;
329
+
330
+
if (idx > vgic_v3_max_apr_idx(vcpu))
331
+
return -EINVAL;
332
+
333
+
return get_gic_ich_reg(vcpu, r, val);
334
+
}
335
+
336
+
static int set_gic_icc_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
337
+
u64 val)
338
+
{
339
+
if (val != KVM_ICC_SRE_EL2)
340
+
return -EINVAL;
341
+
return 0;
342
+
}
343
+
344
+
static int get_gic_icc_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
345
+
u64 *val)
346
+
{
347
+
*val = KVM_ICC_SRE_EL2;
348
+
return 0;
349
+
}
350
+
351
+
static int set_gic_ich_vtr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
352
+
u64 val)
353
+
{
354
+
if (val != kvm_get_guest_vtr_el2())
355
+
return -EINVAL;
356
+
return 0;
357
+
}
358
+
359
+
static int get_gic_ich_vtr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
360
+
u64 *val)
361
+
{
362
+
*val = kvm_get_guest_vtr_el2();
363
+
return 0;
364
+
}
365
+
366
+
static unsigned int el2_visibility(const struct kvm_vcpu *vcpu,
367
+
const struct sys_reg_desc *rd)
368
+
{
369
+
return vcpu_has_nv(vcpu) ? 0 : REG_HIDDEN;
370
+
}
371
+
372
+
#define __EL2_REG(r, acc, i) \
373
+
{ \
374
+
SYS_DESC(SYS_ ## r), \
375
+
.get_user = get_gic_ ## acc, \
376
+
.set_user = set_gic_ ## acc, \
377
+
.reg = i, \
378
+
.visibility = el2_visibility, \
379
+
}
380
+
381
+
#define EL2_REG(r, acc) __EL2_REG(r, acc, r)
382
+
383
+
#define EL2_REG_RO(r, acc) __EL2_REG(r, acc, 0)
384
+
300
385
static const struct sys_reg_desc gic_v3_icc_reg_descs[] = {
301
386
{ SYS_DESC(SYS_ICC_PMR_EL1),
302
387
.set_user = set_gic_pmr, .get_user = get_gic_pmr, },
···
413
328
.set_user = set_gic_grpen0, .get_user = get_gic_grpen0, },
414
329
{ SYS_DESC(SYS_ICC_IGRPEN1_EL1),
415
330
.set_user = set_gic_grpen1, .get_user = get_gic_grpen1, },
331
+
EL2_REG(ICH_AP0R0_EL2, ich_apr),
332
+
EL2_REG(ICH_AP0R1_EL2, ich_apr),
333
+
EL2_REG(ICH_AP0R2_EL2, ich_apr),
334
+
EL2_REG(ICH_AP0R3_EL2, ich_apr),
335
+
EL2_REG(ICH_AP1R0_EL2, ich_apr),
336
+
EL2_REG(ICH_AP1R1_EL2, ich_apr),
337
+
EL2_REG(ICH_AP1R2_EL2, ich_apr),
338
+
EL2_REG(ICH_AP1R3_EL2, ich_apr),
339
+
EL2_REG_RO(ICC_SRE_EL2, icc_sre),
340
+
EL2_REG(ICH_HCR_EL2, ich_reg),
341
+
EL2_REG_RO(ICH_VTR_EL2, ich_vtr),
342
+
EL2_REG(ICH_VMCR_EL2, ich_reg),
343
+
EL2_REG(ICH_LR0_EL2, ich_reg),
344
+
EL2_REG(ICH_LR1_EL2, ich_reg),
345
+
EL2_REG(ICH_LR2_EL2, ich_reg),
346
+
EL2_REG(ICH_LR3_EL2, ich_reg),
347
+
EL2_REG(ICH_LR4_EL2, ich_reg),
348
+
EL2_REG(ICH_LR5_EL2, ich_reg),
349
+
EL2_REG(ICH_LR6_EL2, ich_reg),
350
+
EL2_REG(ICH_LR7_EL2, ich_reg),
351
+
EL2_REG(ICH_LR8_EL2, ich_reg),
352
+
EL2_REG(ICH_LR9_EL2, ich_reg),
353
+
EL2_REG(ICH_LR10_EL2, ich_reg),
354
+
EL2_REG(ICH_LR11_EL2, ich_reg),
355
+
EL2_REG(ICH_LR12_EL2, ich_reg),
356
+
EL2_REG(ICH_LR13_EL2, ich_reg),
357
+
EL2_REG(ICH_LR14_EL2, ich_reg),
358
+
EL2_REG(ICH_LR15_EL2, ich_reg),
416
359
};
360
+
361
+
const struct sys_reg_desc *vgic_v3_get_sysreg_table(unsigned int *sz)
362
+
{
363
+
*sz = ARRAY_SIZE(gic_v3_icc_reg_descs);
364
+
return gic_v3_icc_reg_descs;
365
+
}
417
366
418
367
static u64 attr_to_id(u64 attr)
419
368
{
···
460
341
461
342
int vgic_v3_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
462
343
{
463
-
if (get_reg_by_id(attr_to_id(attr->attr), gic_v3_icc_reg_descs,
464
-
ARRAY_SIZE(gic_v3_icc_reg_descs)))
344
+
const struct sys_reg_desc *r;
345
+
346
+
r = get_reg_by_id(attr_to_id(attr->attr), gic_v3_icc_reg_descs,
347
+
ARRAY_SIZE(gic_v3_icc_reg_descs));
348
+
349
+
if (r && !sysreg_hidden(vcpu, r))
465
350
return 0;
466
351
467
352
return -ENXIO;
+15
-15
arch/arm64/kvm/vgic/vgic-init.c
+15
-15
arch/arm64/kvm/vgic/vgic-init.c
···
157
157
158
158
kvm->arch.vgic.in_kernel = true;
159
159
kvm->arch.vgic.vgic_model = type;
160
+
kvm->arch.vgic.implementation_rev = KVM_VGIC_IMP_REV_LATEST;
160
161
161
162
kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
162
163
···
165
164
kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
166
165
else
167
166
INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
167
+
168
+
if (type == KVM_DEV_TYPE_ARM_VGIC_V3)
169
+
kvm->arch.vgic.nassgicap = system_supports_direct_sgis();
168
170
169
171
out_unlock:
170
172
mutex_unlock(&kvm->arch.config_lock);
···
395
391
goto out;
396
392
397
393
/*
398
-
* If we have GICv4.1 enabled, unconditionally request enable the
399
-
* v4 support so that we get HW-accelerated vSGIs. Otherwise, only
400
-
* enable it if we present a virtual ITS to the guest.
394
+
* Ensure vPEs are allocated if direct IRQ injection (e.g. vSGIs,
395
+
* vLPIs) is supported.
401
396
*/
402
-
if (vgic_supports_direct_msis(kvm)) {
397
+
if (vgic_supports_direct_irqs(kvm)) {
403
398
ret = vgic_v4_init(kvm);
404
399
if (ret)
405
400
goto out;
···
412
409
goto out;
413
410
414
411
vgic_debug_init(kvm);
415
-
416
-
/*
417
-
* If userspace didn't set the GIC implementation revision,
418
-
* default to the latest and greatest. You know want it.
419
-
*/
420
-
if (!dist->implementation_rev)
421
-
dist->implementation_rev = KVM_VGIC_IMP_REV_LATEST;
422
412
dist->initialized = true;
423
-
424
413
out:
425
414
return ret;
426
415
}
···
438
443
dist->vgic_cpu_base = VGIC_ADDR_UNDEF;
439
444
}
440
445
441
-
if (vgic_supports_direct_msis(kvm))
446
+
if (vgic_supports_direct_irqs(kvm))
442
447
vgic_v4_teardown(kvm);
443
448
444
449
xa_destroy(&dist->lpi_xa);
···
669
674
* We want to make sure the list registers start out clear so that we
670
675
* only have the program the used registers.
671
676
*/
672
-
if (kvm_vgic_global_state.type == VGIC_V2)
677
+
if (kvm_vgic_global_state.type == VGIC_V2) {
673
678
vgic_v2_init_lrs();
674
-
else
679
+
} else if (kvm_vgic_global_state.type == VGIC_V3 ||
680
+
kvm_vgic_global_state.has_gcie_v3_compat) {
675
681
kvm_call_hyp(__vgic_v3_init_lrs);
682
+
}
676
683
}
677
684
678
685
/**
···
718
721
static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
719
722
kvm_info("GIC system register CPU interface enabled\n");
720
723
}
724
+
break;
725
+
case GIC_V5:
726
+
ret = vgic_v5_probe(gic_kvm_info);
721
727
break;
722
728
default:
723
729
ret = -ENODEV;
+3
arch/arm64/kvm/vgic/vgic-its.c
+3
arch/arm64/kvm/vgic/vgic-its.c
+44
-26
arch/arm64/kvm/vgic/vgic-kvm-device.c
+44
-26
arch/arm64/kvm/vgic/vgic-kvm-device.c
···
5
5
* Copyright (C) 2015 ARM Ltd.
6
6
* Author: Marc Zyngier <marc.zyngier@arm.com>
7
7
*/
8
+
#include <linux/irqchip/arm-gic-v3.h>
8
9
#include <linux/kvm_host.h>
9
10
#include <kvm/arm_vgic.h>
10
11
#include <linux/uaccess.h>
···
304
303
VGIC_NR_PRIVATE_IRQS, uaddr);
305
304
break;
306
305
}
307
-
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ: {
308
-
u32 __user *uaddr = (u32 __user *)(long)attr->addr;
309
-
310
-
r = put_user(dev->kvm->arch.vgic.mi_intid, uaddr);
311
-
break;
312
-
}
313
306
}
314
307
315
308
return r;
···
505
510
}
506
511
507
512
/*
513
+
* Allow access to certain ID-like registers prior to VGIC initialization,
514
+
* thereby allowing the VMM to provision the features / sizing of the VGIC.
515
+
*/
516
+
static bool reg_allowed_pre_init(struct kvm_device_attr *attr)
517
+
{
518
+
if (attr->group != KVM_DEV_ARM_VGIC_GRP_DIST_REGS)
519
+
return false;
520
+
521
+
switch (attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK) {
522
+
case GICD_IIDR:
523
+
case GICD_TYPER2:
524
+
return true;
525
+
default:
526
+
return false;
527
+
}
528
+
}
529
+
530
+
/*
508
531
* vgic_v3_attr_regs_access - allows user space to access VGIC v3 state
509
532
*
510
533
* @dev: kvm device handle
···
536
523
struct vgic_reg_attr reg_attr;
537
524
gpa_t addr;
538
525
struct kvm_vcpu *vcpu;
539
-
bool uaccess, post_init = true;
526
+
bool uaccess;
540
527
u32 val;
541
528
int ret;
542
529
···
552
539
/* Sysregs uaccess is performed by the sysreg handling code */
553
540
uaccess = false;
554
541
break;
555
-
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
556
-
post_init = false;
557
-
fallthrough;
558
542
default:
559
543
uaccess = true;
560
544
}
···
571
561
572
562
mutex_lock(&dev->kvm->arch.config_lock);
573
563
574
-
if (post_init != vgic_initialized(dev->kvm)) {
564
+
if (!(vgic_initialized(dev->kvm) || reg_allowed_pre_init(attr))) {
575
565
ret = -EBUSY;
576
566
goto out;
577
567
}
···
601
591
}
602
592
break;
603
593
}
604
-
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
605
-
if (!is_write) {
606
-
val = dev->kvm->arch.vgic.mi_intid;
607
-
ret = 0;
608
-
break;
609
-
}
610
-
611
-
ret = -EINVAL;
612
-
if ((val < VGIC_NR_PRIVATE_IRQS) && (val >= VGIC_NR_SGIS)) {
613
-
dev->kvm->arch.vgic.mi_intid = val;
614
-
ret = 0;
615
-
}
616
-
break;
617
594
default:
618
595
ret = -EINVAL;
619
596
break;
···
627
630
case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:
628
631
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
629
632
case KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO:
630
-
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
631
633
return vgic_v3_attr_regs_access(dev, attr, true);
634
+
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ: {
635
+
u32 __user *uaddr = (u32 __user *)attr->addr;
636
+
u32 val;
637
+
638
+
if (get_user(val, uaddr))
639
+
return -EFAULT;
640
+
641
+
guard(mutex)(&dev->kvm->arch.config_lock);
642
+
if (vgic_initialized(dev->kvm))
643
+
return -EBUSY;
644
+
645
+
if (!irq_is_ppi(val))
646
+
return -EINVAL;
647
+
648
+
dev->kvm->arch.vgic.mi_intid = val;
649
+
return 0;
650
+
}
632
651
default:
633
652
return vgic_set_common_attr(dev, attr);
634
653
}
···
658
645
case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:
659
646
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
660
647
case KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO:
661
-
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
662
648
return vgic_v3_attr_regs_access(dev, attr, false);
649
+
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ: {
650
+
u32 __user *uaddr = (u32 __user *)(long)attr->addr;
651
+
652
+
guard(mutex)(&dev->kvm->arch.config_lock);
653
+
return put_user(dev->kvm->arch.vgic.mi_intid, uaddr);
654
+
}
663
655
default:
664
656
return vgic_get_common_attr(dev, attr);
665
657
}
+26
-7
arch/arm64/kvm/vgic/vgic-mmio-v3.c
+26
-7
arch/arm64/kvm/vgic/vgic-mmio-v3.c
···
50
50
51
51
bool vgic_supports_direct_msis(struct kvm *kvm)
52
52
{
53
-
return (kvm_vgic_global_state.has_gicv4_1 ||
54
-
(kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm)));
53
+
return kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm);
54
+
}
55
+
56
+
bool system_supports_direct_sgis(void)
57
+
{
58
+
return kvm_vgic_global_state.has_gicv4_1 && gic_cpuif_has_vsgi();
59
+
}
60
+
61
+
bool vgic_supports_direct_sgis(struct kvm *kvm)
62
+
{
63
+
return kvm->arch.vgic.nassgicap;
55
64
}
56
65
57
66
/*
···
95
86
}
96
87
break;
97
88
case GICD_TYPER2:
98
-
if (kvm_vgic_global_state.has_gicv4_1 && gic_cpuif_has_vsgi())
89
+
if (vgic_supports_direct_sgis(vcpu->kvm))
99
90
value = GICD_TYPER2_nASSGIcap;
100
91
break;
101
92
case GICD_IIDR:
···
128
119
dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
129
120
130
121
/* Not a GICv4.1? No HW SGIs */
131
-
if (!kvm_vgic_global_state.has_gicv4_1 || !gic_cpuif_has_vsgi())
122
+
if (!vgic_supports_direct_sgis(vcpu->kvm))
132
123
val &= ~GICD_CTLR_nASSGIreq;
133
124
134
125
/* Dist stays enabled? nASSGIreq is RO */
···
142
133
if (is_hwsgi != dist->nassgireq)
143
134
vgic_v4_configure_vsgis(vcpu->kvm);
144
135
145
-
if (kvm_vgic_global_state.has_gicv4_1 &&
136
+
if (vgic_supports_direct_sgis(vcpu->kvm) &&
146
137
was_enabled != dist->enabled)
147
138
kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4);
148
139
else if (!was_enabled && dist->enabled)
···
168
159
169
160
switch (addr & 0x0c) {
170
161
case GICD_TYPER2:
171
-
if (val != vgic_mmio_read_v3_misc(vcpu, addr, len))
162
+
reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
163
+
164
+
if (reg == val)
165
+
return 0;
166
+
if (vgic_initialized(vcpu->kvm))
167
+
return -EBUSY;
168
+
if ((reg ^ val) & ~GICD_TYPER2_nASSGIcap)
172
169
return -EINVAL;
170
+
if (!system_supports_direct_sgis() && val)
171
+
return -EINVAL;
172
+
173
+
dist->nassgicap = val & GICD_TYPER2_nASSGIcap;
173
174
return 0;
174
175
case GICD_IIDR:
175
176
reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
···
197
178
}
198
179
case GICD_CTLR:
199
180
/* Not a GICv4.1? No HW SGIs */
200
-
if (!kvm_vgic_global_state.has_gicv4_1)
181
+
if (!vgic_supports_direct_sgis(vcpu->kvm))
201
182
val &= ~GICD_CTLR_nASSGIreq;
202
183
203
184
dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
+1
-1
arch/arm64/kvm/vgic/vgic-v3-nested.c
+1
-1
arch/arm64/kvm/vgic/vgic-v3-nested.c
···
116
116
{
117
117
u64 xmo;
118
118
119
-
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
119
+
if (is_nested_ctxt(vcpu)) {
120
120
xmo = __vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_IMO | HCR_FMO);
121
121
WARN_ONCE(xmo && xmo != (HCR_IMO | HCR_FMO),
122
122
"Separate virtual IRQ/FIQ settings not supported\n");
+2
-2
arch/arm64/kvm/vgic/vgic-v4.c
+2
-2
arch/arm64/kvm/vgic/vgic-v4.c
···
356
356
{
357
357
struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
358
358
359
-
if (!vgic_supports_direct_msis(vcpu->kvm) || !vpe->resident)
359
+
if (!vgic_supports_direct_irqs(vcpu->kvm) || !vpe->resident)
360
360
return 0;
361
361
362
362
return its_make_vpe_non_resident(vpe, vgic_v4_want_doorbell(vcpu));
···
367
367
struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
368
368
int err;
369
369
370
-
if (!vgic_supports_direct_msis(vcpu->kvm) || vpe->resident)
370
+
if (!vgic_supports_direct_irqs(vcpu->kvm) || vpe->resident)
371
371
return 0;
372
372
373
373
if (vcpu_get_flag(vcpu, IN_WFI))
+52
arch/arm64/kvm/vgic/vgic-v5.c
+52
arch/arm64/kvm/vgic/vgic-v5.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
3
+
#include <kvm/arm_vgic.h>
4
+
#include <linux/irqchip/arm-vgic-info.h>
5
+
6
+
#include "vgic.h"
7
+
8
+
/*
9
+
* Probe for a vGICv5 compatible interrupt controller, returning 0 on success.
10
+
* Currently only supports GICv3-based VMs on a GICv5 host, and hence only
11
+
* registers a VGIC_V3 device.
12
+
*/
13
+
int vgic_v5_probe(const struct gic_kvm_info *info)
14
+
{
15
+
u64 ich_vtr_el2;
16
+
int ret;
17
+
18
+
if (!info->has_gcie_v3_compat)
19
+
return -ENODEV;
20
+
21
+
kvm_vgic_global_state.type = VGIC_V5;
22
+
kvm_vgic_global_state.has_gcie_v3_compat = true;
23
+
24
+
/* We only support v3 compat mode - use vGICv3 limits */
25
+
kvm_vgic_global_state.max_gic_vcpus = VGIC_V3_MAX_CPUS;
26
+
27
+
kvm_vgic_global_state.vcpu_base = 0;
28
+
kvm_vgic_global_state.vctrl_base = NULL;
29
+
kvm_vgic_global_state.can_emulate_gicv2 = false;
30
+
kvm_vgic_global_state.has_gicv4 = false;
31
+
kvm_vgic_global_state.has_gicv4_1 = false;
32
+
33
+
ich_vtr_el2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config);
34
+
kvm_vgic_global_state.ich_vtr_el2 = (u32)ich_vtr_el2;
35
+
36
+
/*
37
+
* The ListRegs field is 5 bits, but there is an architectural
38
+
* maximum of 16 list registers. Just ignore bit 4...
39
+
*/
40
+
kvm_vgic_global_state.nr_lr = (ich_vtr_el2 & 0xf) + 1;
41
+
42
+
ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3);
43
+
if (ret) {
44
+
kvm_err("Cannot register GICv3-legacy KVM device.\n");
45
+
return ret;
46
+
}
47
+
48
+
static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
49
+
kvm_info("GCIE legacy system register CPU interface\n");
50
+
51
+
return 0;
52
+
}
+2
-2
arch/arm64/kvm/vgic/vgic.c
+2
-2
arch/arm64/kvm/vgic/vgic.c
···
951
951
* can be directly injected (GICv4).
952
952
*/
953
953
if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head) &&
954
-
!vgic_supports_direct_msis(vcpu->kvm))
954
+
!vgic_supports_direct_irqs(vcpu->kvm))
955
955
return;
956
956
957
957
DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
···
965
965
if (can_access_vgic_from_kernel())
966
966
vgic_restore_state(vcpu);
967
967
968
-
if (vgic_supports_direct_msis(vcpu->kvm))
968
+
if (vgic_supports_direct_irqs(vcpu->kvm))
969
969
vgic_v4_commit(vcpu);
970
970
}
971
971
+48
arch/arm64/kvm/vgic/vgic.h
+48
arch/arm64/kvm/vgic/vgic.h
···
64
64
KVM_REG_ARM_VGIC_SYSREG_CRM_MASK | \
65
65
KVM_REG_ARM_VGIC_SYSREG_OP2_MASK)
66
66
67
+
#define KVM_ICC_SRE_EL2 (ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_SRE | \
68
+
ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB)
69
+
#define KVM_ICH_VTR_EL2_RES0 (ICH_VTR_EL2_DVIM | \
70
+
ICH_VTR_EL2_A3V | \
71
+
ICH_VTR_EL2_IDbits)
72
+
#define KVM_ICH_VTR_EL2_RES1 ICH_VTR_EL2_nV4
73
+
74
+
static inline u64 kvm_get_guest_vtr_el2(void)
75
+
{
76
+
u64 vtr;
77
+
78
+
vtr = kvm_vgic_global_state.ich_vtr_el2;
79
+
vtr &= ~KVM_ICH_VTR_EL2_RES0;
80
+
vtr |= KVM_ICH_VTR_EL2_RES1;
81
+
82
+
return vtr;
83
+
}
84
+
67
85
/*
68
86
* As per Documentation/virt/kvm/devices/arm-vgic-its.rst,
69
87
* below macros are defined for ITS table entry encoding.
···
315
297
int vgic_v3_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu,
316
298
struct kvm_device_attr *attr, bool is_write);
317
299
int vgic_v3_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
300
+
const struct sys_reg_desc *vgic_v3_get_sysreg_table(unsigned int *sz);
318
301
int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,
319
302
u32 intid, u32 *val);
320
303
int kvm_register_vgic_device(unsigned long type);
···
326
307
327
308
void vgic_debug_init(struct kvm *kvm);
328
309
void vgic_debug_destroy(struct kvm *kvm);
310
+
311
+
int vgic_v5_probe(const struct gic_kvm_info *info);
329
312
330
313
static inline int vgic_v3_max_apr_idx(struct kvm_vcpu *vcpu)
331
314
{
···
390
369
int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq);
391
370
int vgic_its_invall(struct kvm_vcpu *vcpu);
392
371
372
+
bool system_supports_direct_sgis(void);
393
373
bool vgic_supports_direct_msis(struct kvm *kvm);
374
+
bool vgic_supports_direct_sgis(struct kvm *kvm);
375
+
376
+
static inline bool vgic_supports_direct_irqs(struct kvm *kvm)
377
+
{
378
+
/*
379
+
* Deliberately conflate vLPI and vSGI support on GICv4.1 hardware,
380
+
* indirectly allowing userspace to control whether or not vPEs are
381
+
* allocated for the VM.
382
+
*/
383
+
if (system_supports_direct_sgis())
384
+
return vgic_supports_direct_sgis(kvm);
385
+
386
+
return vgic_supports_direct_msis(kvm);
387
+
}
388
+
394
389
int vgic_v4_init(struct kvm *kvm);
395
390
void vgic_v4_teardown(struct kvm *kvm);
396
391
void vgic_v4_configure_vsgis(struct kvm *kvm);
···
425
388
void vgic_v3_put_nested(struct kvm_vcpu *vcpu);
426
389
void vgic_v3_handle_nested_maint_irq(struct kvm_vcpu *vcpu);
427
390
void vgic_v3_nested_update_mi(struct kvm_vcpu *vcpu);
391
+
392
+
static inline bool vgic_is_v3_compat(struct kvm *kvm)
393
+
{
394
+
return cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF) &&
395
+
kvm_vgic_global_state.has_gcie_v3_compat;
396
+
}
397
+
398
+
static inline bool vgic_is_v3(struct kvm *kvm)
399
+
{
400
+
return kvm_vgic_global_state.type == VGIC_V3 || vgic_is_v3_compat(kvm);
401
+
}
428
402
429
403
int vgic_its_debug_init(struct kvm_device *dev);
430
404
void vgic_its_debug_destroy(struct kvm_device *dev);
+3
-1
arch/arm64/tools/cpucaps
+3
-1
arch/arm64/tools/cpucaps
···
35
35
HAS_GENERIC_AUTH_ARCH_QARMA3
36
36
HAS_GENERIC_AUTH_ARCH_QARMA5
37
37
HAS_GENERIC_AUTH_IMP_DEF
38
-
HAS_GIC_CPUIF_SYSREGS
38
+
HAS_GICV3_CPUIF
39
+
HAS_GICV5_CPUIF
39
40
HAS_GIC_PRIO_MASKING
40
41
HAS_GIC_PRIO_RELAXED_SYNC
41
42
HAS_HCR_NV1
···
50
49
HAS_PMUV3
51
50
HAS_S1PIE
52
51
HAS_S1POE
52
+
HAS_SCTLR2
53
53
HAS_RAS_EXTN
54
54
HAS_RNG
55
55
HAS_SB
+511
-3
arch/arm64/tools/sysreg
+511
-3
arch/arm64/tools/sysreg
···
1314
1314
0b0000 NI
1315
1315
0b0001 IMP
1316
1316
EndEnum
1317
-
Res0 15:12
1317
+
UnsignedEnum 15:12 GCIE
1318
+
0b0000 NI
1319
+
0b0001 IMP
1320
+
EndEnum
1318
1321
UnsignedEnum 11:8 MTEFAR
1319
1322
0b0000 NI
1320
1323
0b0001 IMP
···
3024
3021
Field 63:0 ADDRESS
3025
3022
EndSysreg
3026
3023
3024
+
SysregFields ICC_PPI_HMRx_EL1
3025
+
Field 63 HM63
3026
+
Field 62 HM62
3027
+
Field 61 HM61
3028
+
Field 60 HM60
3029
+
Field 59 HM59
3030
+
Field 58 HM58
3031
+
Field 57 HM57
3032
+
Field 56 HM56
3033
+
Field 55 HM55
3034
+
Field 54 HM54
3035
+
Field 53 HM53
3036
+
Field 52 HM52
3037
+
Field 51 HM51
3038
+
Field 50 HM50
3039
+
Field 49 HM49
3040
+
Field 48 HM48
3041
+
Field 47 HM47
3042
+
Field 46 HM46
3043
+
Field 45 HM45
3044
+
Field 44 HM44
3045
+
Field 43 HM43
3046
+
Field 42 HM42
3047
+
Field 41 HM41
3048
+
Field 40 HM40
3049
+
Field 39 HM39
3050
+
Field 38 HM38
3051
+
Field 37 HM37
3052
+
Field 36 HM36
3053
+
Field 35 HM35
3054
+
Field 34 HM34
3055
+
Field 33 HM33
3056
+
Field 32 HM32
3057
+
Field 31 HM31
3058
+
Field 30 HM30
3059
+
Field 29 HM29
3060
+
Field 28 HM28
3061
+
Field 27 HM27
3062
+
Field 26 HM26
3063
+
Field 25 HM25
3064
+
Field 24 HM24
3065
+
Field 23 HM23
3066
+
Field 22 HM22
3067
+
Field 21 HM21
3068
+
Field 20 HM20
3069
+
Field 19 HM19
3070
+
Field 18 HM18
3071
+
Field 17 HM17
3072
+
Field 16 HM16
3073
+
Field 15 HM15
3074
+
Field 14 HM14
3075
+
Field 13 HM13
3076
+
Field 12 HM12
3077
+
Field 11 HM11
3078
+
Field 10 HM10
3079
+
Field 9 HM9
3080
+
Field 8 HM8
3081
+
Field 7 HM7
3082
+
Field 6 HM6
3083
+
Field 5 HM5
3084
+
Field 4 HM4
3085
+
Field 3 HM3
3086
+
Field 2 HM2
3087
+
Field 1 HM1
3088
+
Field 0 HM0
3089
+
EndSysregFields
3090
+
3091
+
Sysreg ICC_PPI_HMR0_EL1 3 0 12 10 0
3092
+
Fields ICC_PPI_HMRx_EL1
3093
+
EndSysreg
3094
+
3095
+
Sysreg ICC_PPI_HMR1_EL1 3 0 12 10 1
3096
+
Fields ICC_PPI_HMRx_EL1
3097
+
EndSysreg
3098
+
3099
+
Sysreg ICC_IDR0_EL1 3 0 12 10 2
3100
+
Res0 63:12
3101
+
UnsignedEnum 11:8 GCIE_LEGACY
3102
+
0b0000 NI
3103
+
0b0001 IMP
3104
+
EndEnum
3105
+
UnsignedEnum 7:4 PRI_BITS
3106
+
0b0011 4BITS
3107
+
0b0100 5BITS
3108
+
EndEnum
3109
+
UnsignedEnum 3:0 ID_BITS
3110
+
0b0000 16BITS
3111
+
0b0001 24BITS
3112
+
EndEnum
3113
+
EndSysreg
3114
+
3115
+
Sysreg ICC_ICSR_EL1 3 0 12 10 4
3116
+
Res0 63:48
3117
+
Field 47:32 IAFFID
3118
+
Res0 31:16
3119
+
Field 15:11 Priority
3120
+
Res0 10:6
3121
+
Field 5 HM
3122
+
Field 4 Active
3123
+
Field 3 IRM
3124
+
Field 2 Pending
3125
+
Field 1 Enabled
3126
+
Field 0 F
3127
+
EndSysreg
3128
+
3129
+
SysregFields ICC_PPI_ENABLERx_EL1
3130
+
Field 63 EN63
3131
+
Field 62 EN62
3132
+
Field 61 EN61
3133
+
Field 60 EN60
3134
+
Field 59 EN59
3135
+
Field 58 EN58
3136
+
Field 57 EN57
3137
+
Field 56 EN56
3138
+
Field 55 EN55
3139
+
Field 54 EN54
3140
+
Field 53 EN53
3141
+
Field 52 EN52
3142
+
Field 51 EN51
3143
+
Field 50 EN50
3144
+
Field 49 EN49
3145
+
Field 48 EN48
3146
+
Field 47 EN47
3147
+
Field 46 EN46
3148
+
Field 45 EN45
3149
+
Field 44 EN44
3150
+
Field 43 EN43
3151
+
Field 42 EN42
3152
+
Field 41 EN41
3153
+
Field 40 EN40
3154
+
Field 39 EN39
3155
+
Field 38 EN38
3156
+
Field 37 EN37
3157
+
Field 36 EN36
3158
+
Field 35 EN35
3159
+
Field 34 EN34
3160
+
Field 33 EN33
3161
+
Field 32 EN32
3162
+
Field 31 EN31
3163
+
Field 30 EN30
3164
+
Field 29 EN29
3165
+
Field 28 EN28
3166
+
Field 27 EN27
3167
+
Field 26 EN26
3168
+
Field 25 EN25
3169
+
Field 24 EN24
3170
+
Field 23 EN23
3171
+
Field 22 EN22
3172
+
Field 21 EN21
3173
+
Field 20 EN20
3174
+
Field 19 EN19
3175
+
Field 18 EN18
3176
+
Field 17 EN17
3177
+
Field 16 EN16
3178
+
Field 15 EN15
3179
+
Field 14 EN14
3180
+
Field 13 EN13
3181
+
Field 12 EN12
3182
+
Field 11 EN11
3183
+
Field 10 EN10
3184
+
Field 9 EN9
3185
+
Field 8 EN8
3186
+
Field 7 EN7
3187
+
Field 6 EN6
3188
+
Field 5 EN5
3189
+
Field 4 EN4
3190
+
Field 3 EN3
3191
+
Field 2 EN2
3192
+
Field 1 EN1
3193
+
Field 0 EN0
3194
+
EndSysregFields
3195
+
3196
+
Sysreg ICC_PPI_ENABLER0_EL1 3 0 12 10 6
3197
+
Fields ICC_PPI_ENABLERx_EL1
3198
+
EndSysreg
3199
+
3200
+
Sysreg ICC_PPI_ENABLER1_EL1 3 0 12 10 7
3201
+
Fields ICC_PPI_ENABLERx_EL1
3202
+
EndSysreg
3203
+
3204
+
SysregFields ICC_PPI_ACTIVERx_EL1
3205
+
Field 63 Active63
3206
+
Field 62 Active62
3207
+
Field 61 Active61
3208
+
Field 60 Active60
3209
+
Field 59 Active59
3210
+
Field 58 Active58
3211
+
Field 57 Active57
3212
+
Field 56 Active56
3213
+
Field 55 Active55
3214
+
Field 54 Active54
3215
+
Field 53 Active53
3216
+
Field 52 Active52
3217
+
Field 51 Active51
3218
+
Field 50 Active50
3219
+
Field 49 Active49
3220
+
Field 48 Active48
3221
+
Field 47 Active47
3222
+
Field 46 Active46
3223
+
Field 45 Active45
3224
+
Field 44 Active44
3225
+
Field 43 Active43
3226
+
Field 42 Active42
3227
+
Field 41 Active41
3228
+
Field 40 Active40
3229
+
Field 39 Active39
3230
+
Field 38 Active38
3231
+
Field 37 Active37
3232
+
Field 36 Active36
3233
+
Field 35 Active35
3234
+
Field 34 Active34
3235
+
Field 33 Active33
3236
+
Field 32 Active32
3237
+
Field 31 Active31
3238
+
Field 30 Active30
3239
+
Field 29 Active29
3240
+
Field 28 Active28
3241
+
Field 27 Active27
3242
+
Field 26 Active26
3243
+
Field 25 Active25
3244
+
Field 24 Active24
3245
+
Field 23 Active23
3246
+
Field 22 Active22
3247
+
Field 21 Active21
3248
+
Field 20 Active20
3249
+
Field 19 Active19
3250
+
Field 18 Active18
3251
+
Field 17 Active17
3252
+
Field 16 Active16
3253
+
Field 15 Active15
3254
+
Field 14 Active14
3255
+
Field 13 Active13
3256
+
Field 12 Active12
3257
+
Field 11 Active11
3258
+
Field 10 Active10
3259
+
Field 9 Active9
3260
+
Field 8 Active8
3261
+
Field 7 Active7
3262
+
Field 6 Active6
3263
+
Field 5 Active5
3264
+
Field 4 Active4
3265
+
Field 3 Active3
3266
+
Field 2 Active2
3267
+
Field 1 Active1
3268
+
Field 0 Active0
3269
+
EndSysregFields
3270
+
3271
+
Sysreg ICC_PPI_CACTIVER0_EL1 3 0 12 13 0
3272
+
Fields ICC_PPI_ACTIVERx_EL1
3273
+
EndSysreg
3274
+
3275
+
Sysreg ICC_PPI_CACTIVER1_EL1 3 0 12 13 1
3276
+
Fields ICC_PPI_ACTIVERx_EL1
3277
+
EndSysreg
3278
+
3279
+
Sysreg ICC_PPI_SACTIVER0_EL1 3 0 12 13 2
3280
+
Fields ICC_PPI_ACTIVERx_EL1
3281
+
EndSysreg
3282
+
3283
+
Sysreg ICC_PPI_SACTIVER1_EL1 3 0 12 13 3
3284
+
Fields ICC_PPI_ACTIVERx_EL1
3285
+
EndSysreg
3286
+
3287
+
SysregFields ICC_PPI_PENDRx_EL1
3288
+
Field 63 Pend63
3289
+
Field 62 Pend62
3290
+
Field 61 Pend61
3291
+
Field 60 Pend60
3292
+
Field 59 Pend59
3293
+
Field 58 Pend58
3294
+
Field 57 Pend57
3295
+
Field 56 Pend56
3296
+
Field 55 Pend55
3297
+
Field 54 Pend54
3298
+
Field 53 Pend53
3299
+
Field 52 Pend52
3300
+
Field 51 Pend51
3301
+
Field 50 Pend50
3302
+
Field 49 Pend49
3303
+
Field 48 Pend48
3304
+
Field 47 Pend47
3305
+
Field 46 Pend46
3306
+
Field 45 Pend45
3307
+
Field 44 Pend44
3308
+
Field 43 Pend43
3309
+
Field 42 Pend42
3310
+
Field 41 Pend41
3311
+
Field 40 Pend40
3312
+
Field 39 Pend39
3313
+
Field 38 Pend38
3314
+
Field 37 Pend37
3315
+
Field 36 Pend36
3316
+
Field 35 Pend35
3317
+
Field 34 Pend34
3318
+
Field 33 Pend33
3319
+
Field 32 Pend32
3320
+
Field 31 Pend31
3321
+
Field 30 Pend30
3322
+
Field 29 Pend29
3323
+
Field 28 Pend28
3324
+
Field 27 Pend27
3325
+
Field 26 Pend26
3326
+
Field 25 Pend25
3327
+
Field 24 Pend24
3328
+
Field 23 Pend23
3329
+
Field 22 Pend22
3330
+
Field 21 Pend21
3331
+
Field 20 Pend20
3332
+
Field 19 Pend19
3333
+
Field 18 Pend18
3334
+
Field 17 Pend17
3335
+
Field 16 Pend16
3336
+
Field 15 Pend15
3337
+
Field 14 Pend14
3338
+
Field 13 Pend13
3339
+
Field 12 Pend12
3340
+
Field 11 Pend11
3341
+
Field 10 Pend10
3342
+
Field 9 Pend9
3343
+
Field 8 Pend8
3344
+
Field 7 Pend7
3345
+
Field 6 Pend6
3346
+
Field 5 Pend5
3347
+
Field 4 Pend4
3348
+
Field 3 Pend3
3349
+
Field 2 Pend2
3350
+
Field 1 Pend1
3351
+
Field 0 Pend0
3352
+
EndSysregFields
3353
+
3354
+
Sysreg ICC_PPI_CPENDR0_EL1 3 0 12 13 4
3355
+
Fields ICC_PPI_PENDRx_EL1
3356
+
EndSysreg
3357
+
3358
+
Sysreg ICC_PPI_CPENDR1_EL1 3 0 12 13 5
3359
+
Fields ICC_PPI_PENDRx_EL1
3360
+
EndSysreg
3361
+
3362
+
Sysreg ICC_PPI_SPENDR0_EL1 3 0 12 13 6
3363
+
Fields ICC_PPI_PENDRx_EL1
3364
+
EndSysreg
3365
+
3366
+
Sysreg ICC_PPI_SPENDR1_EL1 3 0 12 13 7
3367
+
Fields ICC_PPI_PENDRx_EL1
3368
+
EndSysreg
3369
+
3370
+
SysregFields ICC_PPI_PRIORITYRx_EL1
3371
+
Res0 63:61
3372
+
Field 60:56 Priority7
3373
+
Res0 55:53
3374
+
Field 52:48 Priority6
3375
+
Res0 47:45
3376
+
Field 44:40 Priority5
3377
+
Res0 39:37
3378
+
Field 36:32 Priority4
3379
+
Res0 31:29
3380
+
Field 28:24 Priority3
3381
+
Res0 23:21
3382
+
Field 20:16 Priority2
3383
+
Res0 15:13
3384
+
Field 12:8 Priority1
3385
+
Res0 7:5
3386
+
Field 4:0 Priority0
3387
+
EndSysregFields
3388
+
3389
+
Sysreg ICC_PPI_PRIORITYR0_EL1 3 0 12 14 0
3390
+
Fields ICC_PPI_PRIORITYRx_EL1
3391
+
EndSysreg
3392
+
3393
+
Sysreg ICC_PPI_PRIORITYR1_EL1 3 0 12 14 1
3394
+
Fields ICC_PPI_PRIORITYRx_EL1
3395
+
EndSysreg
3396
+
3397
+
Sysreg ICC_PPI_PRIORITYR2_EL1 3 0 12 14 2
3398
+
Fields ICC_PPI_PRIORITYRx_EL1
3399
+
EndSysreg
3400
+
3401
+
Sysreg ICC_PPI_PRIORITYR3_EL1 3 0 12 14 3
3402
+
Fields ICC_PPI_PRIORITYRx_EL1
3403
+
EndSysreg
3404
+
3405
+
Sysreg ICC_PPI_PRIORITYR4_EL1 3 0 12 14 4
3406
+
Fields ICC_PPI_PRIORITYRx_EL1
3407
+
EndSysreg
3408
+
3409
+
Sysreg ICC_PPI_PRIORITYR5_EL1 3 0 12 14 5
3410
+
Fields ICC_PPI_PRIORITYRx_EL1
3411
+
EndSysreg
3412
+
3413
+
Sysreg ICC_PPI_PRIORITYR6_EL1 3 0 12 14 6
3414
+
Fields ICC_PPI_PRIORITYRx_EL1
3415
+
EndSysreg
3416
+
3417
+
Sysreg ICC_PPI_PRIORITYR7_EL1 3 0 12 14 7
3418
+
Fields ICC_PPI_PRIORITYRx_EL1
3419
+
EndSysreg
3420
+
3421
+
Sysreg ICC_PPI_PRIORITYR8_EL1 3 0 12 15 0
3422
+
Fields ICC_PPI_PRIORITYRx_EL1
3423
+
EndSysreg
3424
+
3425
+
Sysreg ICC_PPI_PRIORITYR9_EL1 3 0 12 15 1
3426
+
Fields ICC_PPI_PRIORITYRx_EL1
3427
+
EndSysreg
3428
+
3429
+
Sysreg ICC_PPI_PRIORITYR10_EL1 3 0 12 15 2
3430
+
Fields ICC_PPI_PRIORITYRx_EL1
3431
+
EndSysreg
3432
+
3433
+
Sysreg ICC_PPI_PRIORITYR11_EL1 3 0 12 15 3
3434
+
Fields ICC_PPI_PRIORITYRx_EL1
3435
+
EndSysreg
3436
+
3437
+
Sysreg ICC_PPI_PRIORITYR12_EL1 3 0 12 15 4
3438
+
Fields ICC_PPI_PRIORITYRx_EL1
3439
+
EndSysreg
3440
+
3441
+
Sysreg ICC_PPI_PRIORITYR13_EL1 3 0 12 15 5
3442
+
Fields ICC_PPI_PRIORITYRx_EL1
3443
+
EndSysreg
3444
+
3445
+
Sysreg ICC_PPI_PRIORITYR14_EL1 3 0 12 15 6
3446
+
Fields ICC_PPI_PRIORITYRx_EL1
3447
+
EndSysreg
3448
+
3449
+
Sysreg ICC_PPI_PRIORITYR15_EL1 3 0 12 15 7
3450
+
Fields ICC_PPI_PRIORITYRx_EL1
3451
+
EndSysreg
3452
+
3027
3453
Sysreg PMSELR_EL0 3 3 9 12 5
3028
3454
Res0 63:5
3029
3455
Field 4:0 SEL
···
3533
3101
Field 15 SMPS
3534
3102
Res0 14:12
3535
3103
Field 11:0 AFFINITY
3104
+
EndSysreg
3105
+
3106
+
Sysreg ICC_CR0_EL1 3 1 12 0 1
3107
+
Res0 63:39
3108
+
Field 38 PID
3109
+
Field 37:32 IPPT
3110
+
Res0 31:1
3111
+
Field 0 EN
3112
+
EndSysreg
3113
+
3114
+
Sysreg ICC_PCR_EL1 3 1 12 0 2
3115
+
Res0 63:5
3116
+
Field 4:0 PRIORITY
3536
3117
EndSysreg
3537
3118
3538
3119
Sysreg CSSELR_EL1 3 2 0 0 0
···
4434
3989
Field 15:0 PhyPARTID28
4435
3990
EndSysreg
4436
3991
3992
+
Sysreg ICH_HFGRTR_EL2 3 4 12 9 4
3993
+
Res0 63:21
3994
+
Field 20 ICC_PPI_ACTIVERn_EL1
3995
+
Field 19 ICC_PPI_PRIORITYRn_EL1
3996
+
Field 18 ICC_PPI_PENDRn_EL1
3997
+
Field 17 ICC_PPI_ENABLERn_EL1
3998
+
Field 16 ICC_PPI_HMRn_EL1
3999
+
Res0 15:8
4000
+
Field 7 ICC_IAFFIDR_EL1
4001
+
Field 6 ICC_ICSR_EL1
4002
+
Field 5 ICC_PCR_EL1
4003
+
Field 4 ICC_HPPIR_EL1
4004
+
Field 3 ICC_HAPR_EL1
4005
+
Field 2 ICC_CR0_EL1
4006
+
Field 1 ICC_IDRn_EL1
4007
+
Field 0 ICC_APR_EL1
4008
+
EndSysreg
4009
+
4010
+
Sysreg ICH_HFGWTR_EL2 3 4 12 9 6
4011
+
Res0 63:21
4012
+
Field 20 ICC_PPI_ACTIVERn_EL1
4013
+
Field 19 ICC_PPI_PRIORITYRn_EL1
4014
+
Field 18 ICC_PPI_PENDRn_EL1
4015
+
Field 17 ICC_PPI_ENABLERn_EL1
4016
+
Res0 16:7
4017
+
Field 6 ICC_ICSR_EL1
4018
+
Field 5 ICC_PCR_EL1
4019
+
Res0 4:3
4020
+
Field 2 ICC_CR0_EL1
4021
+
Res0 1
4022
+
Field 0 ICC_APR_EL1
4023
+
EndSysreg
4024
+
4025
+
Sysreg ICH_HFGITR_EL2 3 4 12 9 7
4026
+
Res0 63:11
4027
+
Field 10 GICRCDNMIA
4028
+
Field 9 GICRCDIA
4029
+
Field 8 GICCDDI
4030
+
Field 7 GICCDEOI
4031
+
Field 6 GICCDHM
4032
+
Field 5 GICCDRCFG
4033
+
Field 4 GICCDPEND
4034
+
Field 3 GICCDAFF
4035
+
Field 2 GICCDPRI
4036
+
Field 1 GICCDDIS
4037
+
Field 0 GICCDEN
4038
+
EndSysreg
4039
+
4437
4040
Sysreg ICH_HCR_EL2 3 4 12 11 0
4438
4041
Res0 63:32
4439
4042
Field 31:27 EOIcount
···
4528
4035
Field 2 LRENP
4529
4036
Field 1 U
4530
4037
Field 0 EOI
4038
+
EndSysreg
4039
+
4040
+
Sysreg ICH_VCTLR_EL2 3 4 12 11 4
4041
+
Res0 63:2
4042
+
Field 1 V3
4043
+
Field 0 En
4531
4044
EndSysreg
4532
4045
4533
4046
Sysreg CONTEXTIDR_EL2 3 4 13 0 1
···
4649
4150
EndSysreg
4650
4151
4651
4152
Sysreg TCR2_EL1 3 0 2 0 3
4652
-
Res0 63:16
4153
+
Res0 63:22
4154
+
Field 21 FNGNA1
4155
+
Field 20 FNGNA0
4156
+
Res0 19
4157
+
Field 18 FNG1
4158
+
Field 17 FNG0
4159
+
Field 16 A2
4653
4160
Field 15 DisCH1
4654
4161
Field 14 DisCH0
4655
4162
Res0 13:12
···
4679
4174
EndSysreg
4680
4175
4681
4176
Sysreg TCR2_EL2 3 4 2 0 3
4682
-
Res0 63:16
4177
+
Res0 63:19
4178
+
Field 18 FNG1
4179
+
Field 17 FNG0
4180
+
Field 16 A2
4683
4181
Field 15 DisCH1
4684
4182
Field 14 DisCH0
4685
4183
Field 13 AMEC1
+12
drivers/irqchip/Kconfig
+12
drivers/irqchip/Kconfig
···
41
41
select HAVE_ARM_SMCCC_DISCOVERY
42
42
select IRQ_MSI_IOMMU
43
43
44
+
config ARM_GIC_ITS_PARENT
45
+
bool
46
+
44
47
config ARM_GIC_V3_ITS
45
48
bool
46
49
select GENERIC_MSI_IRQ
47
50
select IRQ_MSI_LIB
51
+
select ARM_GIC_ITS_PARENT
48
52
default ARM_GIC_V3
49
53
select IRQ_MSI_IOMMU
50
54
···
57
53
depends on ARM_GIC_V3_ITS
58
54
depends on FSL_MC_BUS
59
55
default ARM_GIC_V3_ITS
56
+
57
+
config ARM_GIC_V5
58
+
bool
59
+
select IRQ_DOMAIN_HIERARCHY
60
+
select GENERIC_IRQ_EFFECTIVE_AFF_MASK
61
+
select GENERIC_MSI_IRQ
62
+
select IRQ_MSI_LIB
63
+
select ARM_GIC_ITS_PARENT
60
64
61
65
config ARM_NVIC
62
66
bool
+4
-1
drivers/irqchip/Makefile
+4
-1
drivers/irqchip/Makefile
···
33
33
obj-$(CONFIG_IRQ_MSI_LIB) += irq-msi-lib.o
34
34
obj-$(CONFIG_ARM_GIC_V2M) += irq-gic-v2m.o
35
35
obj-$(CONFIG_ARM_GIC_V3) += irq-gic-v3.o irq-gic-v3-mbi.o irq-gic-common.o
36
-
obj-$(CONFIG_ARM_GIC_V3_ITS) += irq-gic-v3-its.o irq-gic-v4.o irq-gic-v3-its-msi-parent.o
36
+
obj-$(CONFIG_ARM_GIC_ITS_PARENT) += irq-gic-its-msi-parent.o
37
+
obj-$(CONFIG_ARM_GIC_V3_ITS) += irq-gic-v3-its.o irq-gic-v4.o
37
38
obj-$(CONFIG_ARM_GIC_V3_ITS_FSL_MC) += irq-gic-v3-its-fsl-mc-msi.o
38
39
obj-$(CONFIG_PARTITION_PERCPU) += irq-partition-percpu.o
40
+
obj-$(CONFIG_ARM_GIC_V5) += irq-gic-v5.o irq-gic-v5-irs.o irq-gic-v5-its.o \
41
+
irq-gic-v5-iwb.o
39
42
obj-$(CONFIG_HISILICON_IRQ_MBIGEN) += irq-mbigen.o
40
43
obj-$(CONFIG_ARM_NVIC) += irq-nvic.o
41
44
obj-$(CONFIG_ARM_VIC) += irq-vic.o
-2
drivers/irqchip/irq-gic-common.h
-2
drivers/irqchip/irq-gic-common.h
···
29
29
void gic_enable_of_quirks(const struct device_node *np,
30
30
const struct gic_quirk *quirks, void *data);
31
31
32
-
extern const struct msi_parent_ops gic_v3_its_msi_parent_ops;
33
-
34
32
#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0)
35
33
#define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1)
36
34
#define RDIST_FLAGS_FORCE_NON_SHAREABLE (1 << 2)
+12
drivers/irqchip/irq-gic-its-msi-parent.h
+12
drivers/irqchip/irq-gic-its-msi-parent.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* Copyright (C) 2024 ARM Limited, All Rights Reserved.
4
+
*/
5
+
6
+
#ifndef _IRQ_GIC_ITS_MSI_PARENT_H
7
+
#define _IRQ_GIC_ITS_MSI_PARENT_H
8
+
9
+
extern const struct msi_parent_ops gic_v3_its_msi_parent_ops;
10
+
extern const struct msi_parent_ops gic_v5_its_msi_parent_ops;
11
+
12
+
#endif /* _IRQ_GIC_ITS_MSI_PARENT_H */
+167
-1
drivers/irqchip/irq-gic-v3-its-msi-parent.c
drivers/irqchip/irq-gic-its-msi-parent.c
+167
-1
drivers/irqchip/irq-gic-v3-its-msi-parent.c
drivers/irqchip/irq-gic-its-msi-parent.c
···
5
5
// Copyright (C) 2022 Intel
6
6
7
7
#include <linux/acpi_iort.h>
8
+
#include <linux/of_address.h>
8
9
#include <linux/pci.h>
9
10
10
-
#include "irq-gic-common.h"
11
+
#include "irq-gic-its-msi-parent.h"
11
12
#include <linux/irqchip/irq-msi-lib.h>
12
13
13
14
#define ITS_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \
···
18
17
#define ITS_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \
19
18
MSI_FLAG_PCI_MSIX | \
20
19
MSI_FLAG_MULTI_PCI_MSI)
20
+
21
+
static int its_translate_frame_address(struct device_node *msi_node, phys_addr_t *pa)
22
+
{
23
+
struct resource res;
24
+
int ret;
25
+
26
+
ret = of_property_match_string(msi_node, "reg-names", "ns-translate");
27
+
if (ret < 0)
28
+
return ret;
29
+
30
+
ret = of_address_to_resource(msi_node, ret, &res);
31
+
if (ret)
32
+
return ret;
33
+
34
+
*pa = res.start;
35
+
return 0;
36
+
}
21
37
22
38
#ifdef CONFIG_PCI_MSI
23
39
static int its_pci_msi_vec_count(struct pci_dev *pdev, void *data)
···
100
82
msi_info = msi_get_domain_info(domain->parent);
101
83
return msi_info->ops->msi_prepare(domain->parent, dev, nvec, info);
102
84
}
85
+
86
+
static int its_v5_pci_msi_prepare(struct irq_domain *domain, struct device *dev,
87
+
int nvec, msi_alloc_info_t *info)
88
+
{
89
+
struct device_node *msi_node = NULL;
90
+
struct msi_domain_info *msi_info;
91
+
struct pci_dev *pdev;
92
+
phys_addr_t pa;
93
+
u32 rid;
94
+
int ret;
95
+
96
+
if (!dev_is_pci(dev))
97
+
return -EINVAL;
98
+
99
+
pdev = to_pci_dev(dev);
100
+
101
+
rid = pci_msi_map_rid_ctlr_node(pdev, &msi_node);
102
+
if (!msi_node)
103
+
return -ENODEV;
104
+
105
+
ret = its_translate_frame_address(msi_node, &pa);
106
+
if (ret)
107
+
return -ENODEV;
108
+
109
+
of_node_put(msi_node);
110
+
111
+
/* ITS specific DeviceID */
112
+
info->scratchpad[0].ul = rid;
113
+
/* ITS translate frame physical address */
114
+
info->scratchpad[1].ul = pa;
115
+
116
+
/* Always allocate power of two vectors */
117
+
nvec = roundup_pow_of_two(nvec);
118
+
119
+
msi_info = msi_get_domain_info(domain->parent);
120
+
return msi_info->ops->msi_prepare(domain->parent, dev, nvec, info);
121
+
}
103
122
#else /* CONFIG_PCI_MSI */
104
123
#define its_pci_msi_prepare NULL
124
+
#define its_v5_pci_msi_prepare NULL
105
125
#endif /* !CONFIG_PCI_MSI */
106
126
107
127
static int of_pmsi_get_dev_id(struct irq_domain *domain, struct device *dev,
···
174
118
return ret;
175
119
}
176
120
121
+
static int of_v5_pmsi_get_msi_info(struct irq_domain *domain, struct device *dev,
122
+
u32 *dev_id, phys_addr_t *pa)
123
+
{
124
+
int ret, index = 0;
125
+
/*
126
+
* Retrieve the DeviceID and the ITS translate frame node pointer
127
+
* out of the msi-parent property.
128
+
*/
129
+
do {
130
+
struct of_phandle_args args;
131
+
132
+
ret = of_parse_phandle_with_args(dev->of_node,
133
+
"msi-parent", "#msi-cells",
134
+
index, &args);
135
+
if (ret)
136
+
break;
137
+
/*
138
+
* The IRQ domain fwnode is the msi controller parent
139
+
* in GICv5 (where the msi controller nodes are the
140
+
* ITS translate frames).
141
+
*/
142
+
if (args.np->parent == irq_domain_get_of_node(domain)) {
143
+
if (WARN_ON(args.args_count != 1))
144
+
return -EINVAL;
145
+
*dev_id = args.args[0];
146
+
147
+
ret = its_translate_frame_address(args.np, pa);
148
+
if (ret)
149
+
return -ENODEV;
150
+
break;
151
+
}
152
+
index++;
153
+
} while (!ret);
154
+
155
+
if (ret) {
156
+
struct device_node *np = NULL;
157
+
158
+
ret = of_map_id(dev->of_node, dev->id, "msi-map", "msi-map-mask", &np, dev_id);
159
+
if (np) {
160
+
ret = its_translate_frame_address(np, pa);
161
+
of_node_put(np);
162
+
}
163
+
}
164
+
165
+
return ret;
166
+
}
167
+
177
168
int __weak iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id)
178
169
{
179
170
return -1;
···
249
146
msi_info = msi_get_domain_info(domain->parent);
250
147
return msi_info->ops->msi_prepare(domain->parent,
251
148
dev, nvec, info);
149
+
}
150
+
151
+
static int its_v5_pmsi_prepare(struct irq_domain *domain, struct device *dev,
152
+
int nvec, msi_alloc_info_t *info)
153
+
{
154
+
struct msi_domain_info *msi_info;
155
+
phys_addr_t pa;
156
+
u32 dev_id;
157
+
int ret;
158
+
159
+
if (!dev->of_node)
160
+
return -ENODEV;
161
+
162
+
ret = of_v5_pmsi_get_msi_info(domain->parent, dev, &dev_id, &pa);
163
+
if (ret)
164
+
return ret;
165
+
166
+
/* ITS specific DeviceID */
167
+
info->scratchpad[0].ul = dev_id;
168
+
/* ITS translate frame physical address */
169
+
info->scratchpad[1].ul = pa;
170
+
171
+
/* Allocate always as a power of 2 */
172
+
nvec = roundup_pow_of_two(nvec);
173
+
174
+
msi_info = msi_get_domain_info(domain->parent);
175
+
return msi_info->ops->msi_prepare(domain->parent, dev, nvec, info);
252
176
}
253
177
254
178
static void its_msi_teardown(struct irq_domain *domain, msi_alloc_info_t *info)
···
329
199
return true;
330
200
}
331
201
202
+
static bool its_v5_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
203
+
struct irq_domain *real_parent, struct msi_domain_info *info)
204
+
{
205
+
if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
206
+
return false;
207
+
208
+
switch (info->bus_token) {
209
+
case DOMAIN_BUS_PCI_DEVICE_MSI:
210
+
case DOMAIN_BUS_PCI_DEVICE_MSIX:
211
+
info->ops->msi_prepare = its_v5_pci_msi_prepare;
212
+
info->ops->msi_teardown = its_msi_teardown;
213
+
break;
214
+
case DOMAIN_BUS_DEVICE_MSI:
215
+
case DOMAIN_BUS_WIRED_TO_MSI:
216
+
info->ops->msi_prepare = its_v5_pmsi_prepare;
217
+
info->ops->msi_teardown = its_msi_teardown;
218
+
break;
219
+
default:
220
+
/* Confused. How did the lib return true? */
221
+
WARN_ON_ONCE(1);
222
+
return false;
223
+
}
224
+
225
+
return true;
226
+
}
227
+
332
228
const struct msi_parent_ops gic_v3_its_msi_parent_ops = {
333
229
.supported_flags = ITS_MSI_FLAGS_SUPPORTED,
334
230
.required_flags = ITS_MSI_FLAGS_REQUIRED,
···
363
207
.bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI,
364
208
.prefix = "ITS-",
365
209
.init_dev_msi_info = its_init_dev_msi_info,
210
+
};
211
+
212
+
const struct msi_parent_ops gic_v5_its_msi_parent_ops = {
213
+
.supported_flags = ITS_MSI_FLAGS_SUPPORTED,
214
+
.required_flags = ITS_MSI_FLAGS_REQUIRED,
215
+
.chip_flags = MSI_CHIP_FLAG_SET_EOI,
216
+
.bus_select_token = DOMAIN_BUS_NEXUS,
217
+
.bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI,
218
+
.prefix = "ITS-v5-",
219
+
.init_dev_msi_info = its_v5_init_dev_msi_info,
366
220
};
+1
drivers/irqchip/irq-gic-v3-its.c
+1
drivers/irqchip/irq-gic-v3-its.c
+822
drivers/irqchip/irq-gic-v5-irs.c
+822
drivers/irqchip/irq-gic-v5-irs.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Copyright (C) 2024-2025 ARM Limited, All Rights Reserved.
4
+
*/
5
+
6
+
#define pr_fmt(fmt) "GICv5 IRS: " fmt
7
+
8
+
#include <linux/log2.h>
9
+
#include <linux/of.h>
10
+
#include <linux/of_address.h>
11
+
12
+
#include <linux/irqchip.h>
13
+
#include <linux/irqchip/arm-gic-v5.h>
14
+
15
+
/*
16
+
* Hardcoded ID_BITS limit for systems supporting only a 1-level IST
17
+
* table. Systems supporting only a 1-level IST table aren't expected
18
+
* to require more than 2^12 LPIs. Tweak as required.
19
+
*/
20
+
#define LPI_ID_BITS_LINEAR 12
21
+
22
+
#define IRS_FLAGS_NON_COHERENT BIT(0)
23
+
24
+
static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data);
25
+
static LIST_HEAD(irs_nodes);
26
+
27
+
static u32 irs_readl_relaxed(struct gicv5_irs_chip_data *irs_data,
28
+
const u32 reg_offset)
29
+
{
30
+
return readl_relaxed(irs_data->irs_base + reg_offset);
31
+
}
32
+
33
+
static void irs_writel_relaxed(struct gicv5_irs_chip_data *irs_data,
34
+
const u32 val, const u32 reg_offset)
35
+
{
36
+
writel_relaxed(val, irs_data->irs_base + reg_offset);
37
+
}
38
+
39
+
static u64 irs_readq_relaxed(struct gicv5_irs_chip_data *irs_data,
40
+
const u32 reg_offset)
41
+
{
42
+
return readq_relaxed(irs_data->irs_base + reg_offset);
43
+
}
44
+
45
+
static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data,
46
+
const u64 val, const u32 reg_offset)
47
+
{
48
+
writeq_relaxed(val, irs_data->irs_base + reg_offset);
49
+
}
50
+
51
+
/*
52
+
* The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register
53
+
* provides the memory barriers (through MMIO accessors)
54
+
* required to synchronize CPU and GIC access to IST memory.
55
+
*/
56
+
static int gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data *irs_data)
57
+
{
58
+
return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_IST_STATUSR,
59
+
GICV5_IRS_IST_STATUSR_IDLE, NULL);
60
+
}
61
+
62
+
static int __init gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data *irs_data,
63
+
unsigned int lpi_id_bits,
64
+
unsigned int istsz)
65
+
{
66
+
size_t l2istsz;
67
+
u32 n, cfgr;
68
+
void *ist;
69
+
u64 baser;
70
+
int ret;
71
+
72
+
/* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
73
+
n = max(5, lpi_id_bits + 1 + istsz);
74
+
75
+
l2istsz = BIT(n + 1);
76
+
/*
77
+
* Check memory requirements. For a linear IST we cap the
78
+
* number of ID bits to a value that should never exceed
79
+
* kmalloc interface memory allocation limits, so this
80
+
* check is really belt and braces.
81
+
*/
82
+
if (l2istsz > KMALLOC_MAX_SIZE) {
83
+
u8 lpi_id_cap = ilog2(KMALLOC_MAX_SIZE) - 2 + istsz;
84
+
85
+
pr_warn("Limiting LPI ID bits from %u to %u\n",
86
+
lpi_id_bits, lpi_id_cap);
87
+
lpi_id_bits = lpi_id_cap;
88
+
l2istsz = KMALLOC_MAX_SIZE;
89
+
}
90
+
91
+
ist = kzalloc(l2istsz, GFP_KERNEL);
92
+
if (!ist)
93
+
return -ENOMEM;
94
+
95
+
if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
96
+
dcache_clean_inval_poc((unsigned long)ist,
97
+
(unsigned long)ist + l2istsz);
98
+
else
99
+
dsb(ishst);
100
+
101
+
cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
102
+
GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR) |
103
+
FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) |
104
+
FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ,
105
+
GICV5_IRS_IST_CFGR_L2SZ_4K) |
106
+
FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
107
+
irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
108
+
109
+
gicv5_global_data.ist.l2 = false;
110
+
111
+
baser = (virt_to_phys(ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
112
+
FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
113
+
irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
114
+
115
+
ret = gicv5_irs_ist_synchronise(irs_data);
116
+
if (ret) {
117
+
kfree(ist);
118
+
return ret;
119
+
}
120
+
121
+
return 0;
122
+
}
123
+
124
+
static int __init gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data *irs_data,
125
+
unsigned int lpi_id_bits,
126
+
unsigned int istsz,
127
+
unsigned int l2sz)
128
+
{
129
+
__le64 *l1ist;
130
+
u32 cfgr, n;
131
+
size_t l1sz;
132
+
u64 baser;
133
+
int ret;
134
+
135
+
/* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
136
+
n = max(5, lpi_id_bits - ((10 - istsz) + (2 * l2sz)) + 2);
137
+
138
+
l1sz = BIT(n + 1);
139
+
140
+
l1ist = kzalloc(l1sz, GFP_KERNEL);
141
+
if (!l1ist)
142
+
return -ENOMEM;
143
+
144
+
if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
145
+
dcache_clean_inval_poc((unsigned long)l1ist,
146
+
(unsigned long)l1ist + l1sz);
147
+
else
148
+
dsb(ishst);
149
+
150
+
cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
151
+
GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL) |
152
+
FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) |
153
+
FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, l2sz) |
154
+
FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
155
+
irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
156
+
157
+
/*
158
+
* The L2SZ determine bits required at L2 level. Number of bytes
159
+
* required by metadata is reported through istsz - the number of bits
160
+
* covered by L2 entries scales accordingly.
161
+
*/
162
+
gicv5_global_data.ist.l2_size = BIT(11 + (2 * l2sz) + 1);
163
+
gicv5_global_data.ist.l2_bits = (10 - istsz) + (2 * l2sz);
164
+
gicv5_global_data.ist.l1ist_addr = l1ist;
165
+
gicv5_global_data.ist.l2 = true;
166
+
167
+
baser = (virt_to_phys(l1ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
168
+
FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
169
+
irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
170
+
171
+
ret = gicv5_irs_ist_synchronise(irs_data);
172
+
if (ret) {
173
+
kfree(l1ist);
174
+
return ret;
175
+
}
176
+
177
+
return 0;
178
+
}
179
+
180
+
/*
181
+
* Alloc L2 IST entries on demand.
182
+
*
183
+
* Locking/serialization is guaranteed by irqdomain core code by
184
+
* taking the hierarchical domain struct irq_domain.root->mutex.
185
+
*/
186
+
int gicv5_irs_iste_alloc(const u32 lpi)
187
+
{
188
+
struct gicv5_irs_chip_data *irs_data;
189
+
unsigned int index;
190
+
u32 l2istr, l2bits;
191
+
__le64 *l1ist;
192
+
size_t l2size;
193
+
void *l2ist;
194
+
int ret;
195
+
196
+
if (!gicv5_global_data.ist.l2)
197
+
return 0;
198
+
199
+
irs_data = per_cpu(per_cpu_irs_data, smp_processor_id());
200
+
if (!irs_data)
201
+
return -ENOENT;
202
+
203
+
l2size = gicv5_global_data.ist.l2_size;
204
+
l2bits = gicv5_global_data.ist.l2_bits;
205
+
l1ist = gicv5_global_data.ist.l1ist_addr;
206
+
index = lpi >> l2bits;
207
+
208
+
if (FIELD_GET(GICV5_ISTL1E_VALID, le64_to_cpu(l1ist[index])))
209
+
return 0;
210
+
211
+
l2ist = kzalloc(l2size, GFP_KERNEL);
212
+
if (!l2ist)
213
+
return -ENOMEM;
214
+
215
+
l1ist[index] = cpu_to_le64(virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK);
216
+
217
+
if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
218
+
dcache_clean_inval_poc((unsigned long)l2ist,
219
+
(unsigned long)l2ist + l2size);
220
+
dcache_clean_poc((unsigned long)(l1ist + index),
221
+
(unsigned long)(l1ist + index) + sizeof(*l1ist));
222
+
} else {
223
+
dsb(ishst);
224
+
}
225
+
226
+
l2istr = FIELD_PREP(GICV5_IRS_MAP_L2_ISTR_ID, lpi);
227
+
irs_writel_relaxed(irs_data, l2istr, GICV5_IRS_MAP_L2_ISTR);
228
+
229
+
ret = gicv5_irs_ist_synchronise(irs_data);
230
+
if (ret) {
231
+
l1ist[index] = 0;
232
+
kfree(l2ist);
233
+
return ret;
234
+
}
235
+
236
+
/*
237
+
* Make sure we invalidate the cache line pulled before the IRS
238
+
* had a chance to update the L1 entry and mark it valid.
239
+
*/
240
+
if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
241
+
/*
242
+
* gicv5_irs_ist_synchronise() includes memory
243
+
* barriers (MMIO accessors) required to guarantee that the
244
+
* following dcache invalidation is not executed before the
245
+
* IST mapping operation has completed.
246
+
*/
247
+
dcache_inval_poc((unsigned long)(l1ist + index),
248
+
(unsigned long)(l1ist + index) + sizeof(*l1ist));
249
+
}
250
+
251
+
return 0;
252
+
}
253
+
254
+
/*
255
+
* Try to match the L2 IST size to the pagesize, and if this is not possible
256
+
* pick the smallest supported L2 size in order to minimise the requirement for
257
+
* physically contiguous blocks of memory as page-sized allocations are
258
+
* guaranteed to be physically contiguous, and are by definition the easiest to
259
+
* find.
260
+
*
261
+
* Fall back to the smallest supported size (in the event that the pagesize
262
+
* itself is not supported) again serves to make it easier to find physically
263
+
* contiguous blocks of memory.
264
+
*/
265
+
static unsigned int gicv5_irs_l2_sz(u32 idr2)
266
+
{
267
+
switch (PAGE_SIZE) {
268
+
case SZ_64K:
269
+
if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2))
270
+
return GICV5_IRS_IST_CFGR_L2SZ_64K;
271
+
fallthrough;
272
+
case SZ_4K:
273
+
if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
274
+
return GICV5_IRS_IST_CFGR_L2SZ_4K;
275
+
fallthrough;
276
+
case SZ_16K:
277
+
if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2))
278
+
return GICV5_IRS_IST_CFGR_L2SZ_16K;
279
+
break;
280
+
}
281
+
282
+
if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
283
+
return GICV5_IRS_IST_CFGR_L2SZ_4K;
284
+
285
+
return GICV5_IRS_IST_CFGR_L2SZ_64K;
286
+
}
287
+
288
+
static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data)
289
+
{
290
+
u32 lpi_id_bits, idr2_id_bits, idr2_min_lpi_id_bits, l2_iste_sz, l2sz;
291
+
u32 l2_iste_sz_split, idr2;
292
+
bool two_levels, istmd;
293
+
u64 baser;
294
+
int ret;
295
+
296
+
baser = irs_readq_relaxed(irs_data, GICV5_IRS_IST_BASER);
297
+
if (FIELD_GET(GICV5_IRS_IST_BASER_VALID, baser)) {
298
+
pr_err("IST is marked as valid already; cannot allocate\n");
299
+
return -EPERM;
300
+
}
301
+
302
+
idr2 = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
303
+
two_levels = !!FIELD_GET(GICV5_IRS_IDR2_IST_LEVELS, idr2);
304
+
305
+
idr2_id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2);
306
+
idr2_min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2);
307
+
308
+
/*
309
+
* For two level tables we are always supporting the maximum allowed
310
+
* number of IDs.
311
+
*
312
+
* For 1-level tables, we should support a number of bits that
313
+
* is >= min_lpi_id_bits but cap it to LPI_ID_BITS_LINEAR lest
314
+
* the level 1-table gets too large and its memory allocation
315
+
* may fail.
316
+
*/
317
+
if (two_levels) {
318
+
lpi_id_bits = idr2_id_bits;
319
+
} else {
320
+
lpi_id_bits = max(LPI_ID_BITS_LINEAR, idr2_min_lpi_id_bits);
321
+
lpi_id_bits = min(lpi_id_bits, idr2_id_bits);
322
+
}
323
+
324
+
/*
325
+
* Cap the ID bits according to the CPUIF supported ID bits
326
+
*/
327
+
lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits);
328
+
329
+
if (two_levels)
330
+
l2sz = gicv5_irs_l2_sz(idr2);
331
+
332
+
istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2);
333
+
334
+
l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_4;
335
+
336
+
if (istmd) {
337
+
l2_iste_sz_split = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2);
338
+
339
+
if (lpi_id_bits < l2_iste_sz_split)
340
+
l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_8;
341
+
else
342
+
l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_16;
343
+
}
344
+
345
+
/*
346
+
* Follow GICv5 specification recommendation to opt in for two
347
+
* level tables (ref: 10.2.1.14 IRS_IST_CFGR).
348
+
*/
349
+
if (two_levels && (lpi_id_bits > ((10 - l2_iste_sz) + (2 * l2sz)))) {
350
+
ret = gicv5_irs_init_ist_two_level(irs_data, lpi_id_bits,
351
+
l2_iste_sz, l2sz);
352
+
} else {
353
+
ret = gicv5_irs_init_ist_linear(irs_data, lpi_id_bits,
354
+
l2_iste_sz);
355
+
}
356
+
if (ret)
357
+
return ret;
358
+
359
+
gicv5_init_lpis(BIT(lpi_id_bits));
360
+
361
+
return 0;
362
+
}
363
+
364
+
struct iaffid_entry {
365
+
u16 iaffid;
366
+
bool valid;
367
+
};
368
+
369
+
static DEFINE_PER_CPU(struct iaffid_entry, cpu_iaffid);
370
+
371
+
int gicv5_irs_cpu_to_iaffid(int cpuid, u16 *iaffid)
372
+
{
373
+
if (!per_cpu(cpu_iaffid, cpuid).valid) {
374
+
pr_err("IAFFID for CPU %d has not been initialised\n", cpuid);
375
+
return -ENODEV;
376
+
}
377
+
378
+
*iaffid = per_cpu(cpu_iaffid, cpuid).iaffid;
379
+
380
+
return 0;
381
+
}
382
+
383
+
struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id)
384
+
{
385
+
struct gicv5_irs_chip_data *irs_data;
386
+
u32 min, max;
387
+
388
+
list_for_each_entry(irs_data, &irs_nodes, entry) {
389
+
if (!irs_data->spi_range)
390
+
continue;
391
+
392
+
min = irs_data->spi_min;
393
+
max = irs_data->spi_min + irs_data->spi_range - 1;
394
+
if (spi_id >= min && spi_id <= max)
395
+
return irs_data;
396
+
}
397
+
398
+
return NULL;
399
+
}
400
+
401
+
static int gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data *irs_data)
402
+
{
403
+
u32 statusr;
404
+
int ret;
405
+
406
+
ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_SPI_STATUSR,
407
+
GICV5_IRS_SPI_STATUSR_IDLE, &statusr);
408
+
if (ret)
409
+
return ret;
410
+
411
+
return !!FIELD_GET(GICV5_IRS_SPI_STATUSR_V, statusr) ? 0 : -EIO;
412
+
}
413
+
414
+
static int gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data *irs_data,
415
+
bool selr)
416
+
{
417
+
bool valid = true;
418
+
u32 statusr;
419
+
int ret;
420
+
421
+
ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_PE_STATUSR,
422
+
GICV5_IRS_PE_STATUSR_IDLE, &statusr);
423
+
if (ret)
424
+
return ret;
425
+
426
+
if (selr)
427
+
valid = !!FIELD_GET(GICV5_IRS_PE_STATUSR_V, statusr);
428
+
429
+
return valid ? 0 : -EIO;
430
+
}
431
+
432
+
static int gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data *irs_data)
433
+
{
434
+
return gicv5_irs_wait_for_irs_pe(irs_data, true);
435
+
}
436
+
437
+
static int gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data *irs_data)
438
+
{
439
+
return gicv5_irs_wait_for_irs_pe(irs_data, false);
440
+
}
441
+
442
+
int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type)
443
+
{
444
+
struct gicv5_irs_chip_data *irs_data = d->chip_data;
445
+
u32 selr, cfgr;
446
+
bool level;
447
+
int ret;
448
+
449
+
/*
450
+
* There is no distinction between HIGH/LOW for level IRQs
451
+
* and RISING/FALLING for edge IRQs in the architecture,
452
+
* hence consider them equivalent.
453
+
*/
454
+
switch (type) {
455
+
case IRQ_TYPE_EDGE_RISING:
456
+
case IRQ_TYPE_EDGE_FALLING:
457
+
level = false;
458
+
break;
459
+
case IRQ_TYPE_LEVEL_HIGH:
460
+
case IRQ_TYPE_LEVEL_LOW:
461
+
level = true;
462
+
break;
463
+
default:
464
+
return -EINVAL;
465
+
}
466
+
467
+
guard(raw_spinlock)(&irs_data->spi_config_lock);
468
+
469
+
selr = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, d->hwirq);
470
+
irs_writel_relaxed(irs_data, selr, GICV5_IRS_SPI_SELR);
471
+
ret = gicv5_irs_wait_for_spi_op(irs_data);
472
+
if (ret)
473
+
return ret;
474
+
475
+
cfgr = FIELD_PREP(GICV5_IRS_SPI_CFGR_TM, level);
476
+
irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_SPI_CFGR);
477
+
478
+
return gicv5_irs_wait_for_spi_op(irs_data);
479
+
}
480
+
481
+
static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data)
482
+
{
483
+
return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_CR0,
484
+
GICV5_IRS_CR0_IDLE, NULL);
485
+
}
486
+
487
+
void gicv5_irs_syncr(void)
488
+
{
489
+
struct gicv5_irs_chip_data *irs_data;
490
+
u32 syncr;
491
+
492
+
irs_data = list_first_entry_or_null(&irs_nodes, struct gicv5_irs_chip_data, entry);
493
+
if (WARN_ON_ONCE(!irs_data))
494
+
return;
495
+
496
+
syncr = FIELD_PREP(GICV5_IRS_SYNCR_SYNC, 1);
497
+
irs_writel_relaxed(irs_data, syncr, GICV5_IRS_SYNCR);
498
+
499
+
gicv5_wait_for_op(irs_data->irs_base, GICV5_IRS_SYNC_STATUSR,
500
+
GICV5_IRS_SYNC_STATUSR_IDLE);
501
+
}
502
+
503
+
int gicv5_irs_register_cpu(int cpuid)
504
+
{
505
+
struct gicv5_irs_chip_data *irs_data;
506
+
u32 selr, cr0;
507
+
u16 iaffid;
508
+
int ret;
509
+
510
+
ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid);
511
+
if (ret) {
512
+
pr_err("IAFFID for CPU %d has not been initialised\n", cpuid);
513
+
return ret;
514
+
}
515
+
516
+
irs_data = per_cpu(per_cpu_irs_data, cpuid);
517
+
if (!irs_data) {
518
+
pr_err("No IRS associated with CPU %u\n", cpuid);
519
+
return -ENXIO;
520
+
}
521
+
522
+
selr = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, iaffid);
523
+
irs_writel_relaxed(irs_data, selr, GICV5_IRS_PE_SELR);
524
+
525
+
ret = gicv5_irs_wait_for_pe_selr(irs_data);
526
+
if (ret) {
527
+
pr_err("IAFFID 0x%x used in IRS_PE_SELR is invalid\n", iaffid);
528
+
return -ENXIO;
529
+
}
530
+
531
+
cr0 = FIELD_PREP(GICV5_IRS_PE_CR0_DPS, 0x1);
532
+
irs_writel_relaxed(irs_data, cr0, GICV5_IRS_PE_CR0);
533
+
534
+
ret = gicv5_irs_wait_for_pe_cr0(irs_data);
535
+
if (ret)
536
+
return ret;
537
+
538
+
pr_debug("CPU %d enabled PE IAFFID 0x%x\n", cpuid, iaffid);
539
+
540
+
return 0;
541
+
}
542
+
543
+
static void __init gicv5_irs_init_bases(struct gicv5_irs_chip_data *irs_data,
544
+
void __iomem *irs_base,
545
+
struct fwnode_handle *handle)
546
+
{
547
+
struct device_node *np = to_of_node(handle);
548
+
u32 cr0, cr1;
549
+
550
+
irs_data->fwnode = handle;
551
+
irs_data->irs_base = irs_base;
552
+
553
+
if (of_property_read_bool(np, "dma-noncoherent")) {
554
+
/*
555
+
* A non-coherent IRS implies that some cache levels cannot be
556
+
* used coherently by the cores and GIC. Our only option is to mark
557
+
* memory attributes for the GIC as non-cacheable; by default,
558
+
* non-cacheable memory attributes imply outer-shareable
559
+
* shareability, the value written into IRS_CR1_SH is ignored.
560
+
*/
561
+
cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_NO_WRITE_ALLOC) |
562
+
FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_NO_READ_ALLOC) |
563
+
FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_NO_WRITE_ALLOC) |
564
+
FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_NO_READ_ALLOC) |
565
+
FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_NO_READ_ALLOC) |
566
+
FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_NO_READ_ALLOC) |
567
+
FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_NO_WRITE_ALLOC) |
568
+
FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_NO_READ_ALLOC) |
569
+
FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_NON_CACHE) |
570
+
FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_NON_CACHE);
571
+
irs_data->flags |= IRS_FLAGS_NON_COHERENT;
572
+
} else {
573
+
cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_WRITE_ALLOC) |
574
+
FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_READ_ALLOC) |
575
+
FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_WRITE_ALLOC) |
576
+
FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_READ_ALLOC) |
577
+
FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_READ_ALLOC) |
578
+
FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_READ_ALLOC) |
579
+
FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_WRITE_ALLOC) |
580
+
FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_READ_ALLOC) |
581
+
FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_WB_CACHE) |
582
+
FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_WB_CACHE) |
583
+
FIELD_PREP(GICV5_IRS_CR1_SH, GICV5_INNER_SHARE);
584
+
}
585
+
586
+
irs_writel_relaxed(irs_data, cr1, GICV5_IRS_CR1);
587
+
588
+
cr0 = FIELD_PREP(GICV5_IRS_CR0_IRSEN, 0x1);
589
+
irs_writel_relaxed(irs_data, cr0, GICV5_IRS_CR0);
590
+
gicv5_irs_wait_for_idle(irs_data);
591
+
}
592
+
593
+
static int __init gicv5_irs_of_init_affinity(struct device_node *node,
594
+
struct gicv5_irs_chip_data *irs_data,
595
+
u8 iaffid_bits)
596
+
{
597
+
/*
598
+
* Detect IAFFID<->CPU mappings from the device tree and
599
+
* record IRS<->CPU topology information.
600
+
*/
601
+
u16 iaffid_mask = GENMASK(iaffid_bits - 1, 0);
602
+
int ret, i, ncpus, niaffids;
603
+
604
+
ncpus = of_count_phandle_with_args(node, "cpus", NULL);
605
+
if (ncpus < 0)
606
+
return -EINVAL;
607
+
608
+
niaffids = of_property_count_elems_of_size(node, "arm,iaffids",
609
+
sizeof(u16));
610
+
if (niaffids != ncpus)
611
+
return -EINVAL;
612
+
613
+
u16 *iaffids __free(kfree) = kcalloc(niaffids, sizeof(*iaffids), GFP_KERNEL);
614
+
if (!iaffids)
615
+
return -ENOMEM;
616
+
617
+
ret = of_property_read_u16_array(node, "arm,iaffids", iaffids, niaffids);
618
+
if (ret)
619
+
return ret;
620
+
621
+
for (i = 0; i < ncpus; i++) {
622
+
struct device_node *cpu_node;
623
+
int cpu;
624
+
625
+
cpu_node = of_parse_phandle(node, "cpus", i);
626
+
if (WARN_ON(!cpu_node))
627
+
continue;
628
+
629
+
cpu = of_cpu_node_to_id(cpu_node);
630
+
of_node_put(cpu_node);
631
+
if (WARN_ON(cpu < 0))
632
+
continue;
633
+
634
+
if (iaffids[i] & ~iaffid_mask) {
635
+
pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n",
636
+
cpu, iaffids[i]);
637
+
continue;
638
+
}
639
+
640
+
per_cpu(cpu_iaffid, cpu).iaffid = iaffids[i];
641
+
per_cpu(cpu_iaffid, cpu).valid = true;
642
+
643
+
/* We also know that the CPU is connected to this IRS */
644
+
per_cpu(per_cpu_irs_data, cpu) = irs_data;
645
+
}
646
+
647
+
return ret;
648
+
}
649
+
650
+
static void irs_setup_pri_bits(u32 idr1)
651
+
{
652
+
switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) {
653
+
case GICV5_IRS_IDR1_PRIORITY_BITS_1BITS:
654
+
gicv5_global_data.irs_pri_bits = 1;
655
+
break;
656
+
case GICV5_IRS_IDR1_PRIORITY_BITS_2BITS:
657
+
gicv5_global_data.irs_pri_bits = 2;
658
+
break;
659
+
case GICV5_IRS_IDR1_PRIORITY_BITS_3BITS:
660
+
gicv5_global_data.irs_pri_bits = 3;
661
+
break;
662
+
case GICV5_IRS_IDR1_PRIORITY_BITS_4BITS:
663
+
gicv5_global_data.irs_pri_bits = 4;
664
+
break;
665
+
case GICV5_IRS_IDR1_PRIORITY_BITS_5BITS:
666
+
gicv5_global_data.irs_pri_bits = 5;
667
+
break;
668
+
default:
669
+
pr_warn("Detected wrong IDR priority bits value 0x%lx\n",
670
+
FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1));
671
+
gicv5_global_data.irs_pri_bits = 1;
672
+
break;
673
+
}
674
+
}
675
+
676
+
static int __init gicv5_irs_init(struct device_node *node)
677
+
{
678
+
struct gicv5_irs_chip_data *irs_data;
679
+
void __iomem *irs_base;
680
+
u32 idr, spi_count;
681
+
u8 iaffid_bits;
682
+
int ret;
683
+
684
+
irs_data = kzalloc(sizeof(*irs_data), GFP_KERNEL);
685
+
if (!irs_data)
686
+
return -ENOMEM;
687
+
688
+
raw_spin_lock_init(&irs_data->spi_config_lock);
689
+
690
+
ret = of_property_match_string(node, "reg-names", "ns-config");
691
+
if (ret < 0) {
692
+
pr_err("%pOF: ns-config reg-name not present\n", node);
693
+
goto out_err;
694
+
}
695
+
696
+
irs_base = of_io_request_and_map(node, ret, of_node_full_name(node));
697
+
if (IS_ERR(irs_base)) {
698
+
pr_err("%pOF: unable to map GICv5 IRS registers\n", node);
699
+
ret = PTR_ERR(irs_base);
700
+
goto out_err;
701
+
}
702
+
703
+
gicv5_irs_init_bases(irs_data, irs_base, &node->fwnode);
704
+
705
+
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1);
706
+
iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1;
707
+
708
+
ret = gicv5_irs_of_init_affinity(node, irs_data, iaffid_bits);
709
+
if (ret) {
710
+
pr_err("Failed to parse CPU IAFFIDs from the device tree!\n");
711
+
goto out_iomem;
712
+
}
713
+
714
+
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
715
+
if (WARN(!FIELD_GET(GICV5_IRS_IDR2_LPI, idr),
716
+
"LPI support not available - no IPIs, can't proceed\n")) {
717
+
ret = -ENODEV;
718
+
goto out_iomem;
719
+
}
720
+
721
+
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR7);
722
+
irs_data->spi_min = FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, idr);
723
+
724
+
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR6);
725
+
irs_data->spi_range = FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, idr);
726
+
727
+
if (irs_data->spi_range) {
728
+
pr_info("%s detected SPI range [%u-%u]\n",
729
+
of_node_full_name(node),
730
+
irs_data->spi_min,
731
+
irs_data->spi_min +
732
+
irs_data->spi_range - 1);
733
+
}
734
+
735
+
/*
736
+
* Do the global setting only on the first IRS.
737
+
* Global properties (iaffid_bits, global spi count) are guaranteed to
738
+
* be consistent across IRSes by the architecture.
739
+
*/
740
+
if (list_empty(&irs_nodes)) {
741
+
742
+
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1);
743
+
irs_setup_pri_bits(idr);
744
+
745
+
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR5);
746
+
747
+
spi_count = FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, idr);
748
+
gicv5_global_data.global_spi_count = spi_count;
749
+
750
+
gicv5_init_lpi_domain();
751
+
752
+
pr_debug("Detected %u SPIs globally\n", spi_count);
753
+
}
754
+
755
+
list_add_tail(&irs_data->entry, &irs_nodes);
756
+
757
+
return 0;
758
+
759
+
out_iomem:
760
+
iounmap(irs_base);
761
+
out_err:
762
+
kfree(irs_data);
763
+
return ret;
764
+
}
765
+
766
+
void __init gicv5_irs_remove(void)
767
+
{
768
+
struct gicv5_irs_chip_data *irs_data, *tmp_data;
769
+
770
+
gicv5_free_lpi_domain();
771
+
gicv5_deinit_lpis();
772
+
773
+
list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) {
774
+
iounmap(irs_data->irs_base);
775
+
list_del(&irs_data->entry);
776
+
kfree(irs_data);
777
+
}
778
+
}
779
+
780
+
int __init gicv5_irs_enable(void)
781
+
{
782
+
struct gicv5_irs_chip_data *irs_data;
783
+
int ret;
784
+
785
+
irs_data = list_first_entry_or_null(&irs_nodes,
786
+
struct gicv5_irs_chip_data, entry);
787
+
if (!irs_data)
788
+
return -ENODEV;
789
+
790
+
ret = gicv5_irs_init_ist(irs_data);
791
+
if (ret) {
792
+
pr_err("Failed to init IST\n");
793
+
return ret;
794
+
}
795
+
796
+
return 0;
797
+
}
798
+
799
+
void __init gicv5_irs_its_probe(void)
800
+
{
801
+
struct gicv5_irs_chip_data *irs_data;
802
+
803
+
list_for_each_entry(irs_data, &irs_nodes, entry)
804
+
gicv5_its_of_probe(to_of_node(irs_data->fwnode));
805
+
}
806
+
807
+
int __init gicv5_irs_of_probe(struct device_node *parent)
808
+
{
809
+
struct device_node *np;
810
+
int ret;
811
+
812
+
for_each_available_child_of_node(parent, np) {
813
+
if (!of_device_is_compatible(np, "arm,gic-v5-irs"))
814
+
continue;
815
+
816
+
ret = gicv5_irs_init(np);
817
+
if (ret)
818
+
pr_err("Failed to init IRS %s\n", np->full_name);
819
+
}
820
+
821
+
return list_empty(&irs_nodes) ? -ENODEV : 0;
822
+
}
+1228
drivers/irqchip/irq-gic-v5-its.c
+1228
drivers/irqchip/irq-gic-v5-its.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Copyright (C) 2024-2025 ARM Limited, All Rights Reserved.
4
+
*/
5
+
6
+
#define pr_fmt(fmt) "GICv5 ITS: " fmt
7
+
8
+
#include <linux/bitmap.h>
9
+
#include <linux/iommu.h>
10
+
#include <linux/init.h>
11
+
#include <linux/kernel.h>
12
+
#include <linux/msi.h>
13
+
#include <linux/of.h>
14
+
#include <linux/of_address.h>
15
+
#include <linux/of_irq.h>
16
+
#include <linux/slab.h>
17
+
18
+
#include <linux/irqchip.h>
19
+
#include <linux/irqchip/arm-gic-v5.h>
20
+
#include <linux/irqchip/irq-msi-lib.h>
21
+
22
+
#include "irq-gic-its-msi-parent.h"
23
+
24
+
#define ITS_FLAGS_NON_COHERENT BIT(0)
25
+
26
+
struct gicv5_its_chip_data {
27
+
struct xarray its_devices;
28
+
struct mutex dev_alloc_lock;
29
+
struct fwnode_handle *fwnode;
30
+
struct gicv5_its_devtab_cfg devtab_cfgr;
31
+
void __iomem *its_base;
32
+
u32 flags;
33
+
unsigned int msi_domain_flags;
34
+
};
35
+
36
+
struct gicv5_its_dev {
37
+
struct gicv5_its_chip_data *its_node;
38
+
struct gicv5_its_itt_cfg itt_cfg;
39
+
unsigned long *event_map;
40
+
u32 device_id;
41
+
u32 num_events;
42
+
phys_addr_t its_trans_phys_base;
43
+
};
44
+
45
+
static u32 its_readl_relaxed(struct gicv5_its_chip_data *its_node, const u32 reg_offset)
46
+
{
47
+
return readl_relaxed(its_node->its_base + reg_offset);
48
+
}
49
+
50
+
static void its_writel_relaxed(struct gicv5_its_chip_data *its_node, const u32 val,
51
+
const u32 reg_offset)
52
+
{
53
+
writel_relaxed(val, its_node->its_base + reg_offset);
54
+
}
55
+
56
+
static void its_writeq_relaxed(struct gicv5_its_chip_data *its_node, const u64 val,
57
+
const u32 reg_offset)
58
+
{
59
+
writeq_relaxed(val, its_node->its_base + reg_offset);
60
+
}
61
+
62
+
static void gicv5_its_dcache_clean(struct gicv5_its_chip_data *its, void *start,
63
+
size_t sz)
64
+
{
65
+
void *end = start + sz;
66
+
67
+
if (its->flags & ITS_FLAGS_NON_COHERENT)
68
+
dcache_clean_inval_poc((unsigned long)start, (unsigned long)end);
69
+
else
70
+
dsb(ishst);
71
+
}
72
+
73
+
static void its_write_table_entry(struct gicv5_its_chip_data *its, __le64 *entry,
74
+
u64 val)
75
+
{
76
+
WRITE_ONCE(*entry, cpu_to_le64(val));
77
+
gicv5_its_dcache_clean(its, entry, sizeof(*entry));
78
+
}
79
+
80
+
#define devtab_cfgr_field(its, f) \
81
+
FIELD_GET(GICV5_ITS_DT_CFGR_##f, (its)->devtab_cfgr.cfgr)
82
+
83
+
static int gicv5_its_cache_sync(struct gicv5_its_chip_data *its)
84
+
{
85
+
return gicv5_wait_for_op_atomic(its->its_base, GICV5_ITS_STATUSR,
86
+
GICV5_ITS_STATUSR_IDLE, NULL);
87
+
}
88
+
89
+
static void gicv5_its_syncr(struct gicv5_its_chip_data *its,
90
+
struct gicv5_its_dev *its_dev)
91
+
{
92
+
u64 syncr;
93
+
94
+
syncr = FIELD_PREP(GICV5_ITS_SYNCR_SYNC, 1) |
95
+
FIELD_PREP(GICV5_ITS_SYNCR_DEVICEID, its_dev->device_id);
96
+
97
+
its_writeq_relaxed(its, syncr, GICV5_ITS_SYNCR);
98
+
99
+
gicv5_wait_for_op(its->its_base, GICV5_ITS_SYNC_STATUSR, GICV5_ITS_SYNC_STATUSR_IDLE);
100
+
}
101
+
102
+
/* Number of bits required for each L2 {device/interrupt translation} table size */
103
+
#define ITS_L2SZ_64K_L2_BITS 13
104
+
#define ITS_L2SZ_16K_L2_BITS 11
105
+
#define ITS_L2SZ_4K_L2_BITS 9
106
+
107
+
static unsigned int gicv5_its_l2sz_to_l2_bits(unsigned int sz)
108
+
{
109
+
switch (sz) {
110
+
case GICV5_ITS_DT_ITT_CFGR_L2SZ_64k:
111
+
return ITS_L2SZ_64K_L2_BITS;
112
+
case GICV5_ITS_DT_ITT_CFGR_L2SZ_16k:
113
+
return ITS_L2SZ_16K_L2_BITS;
114
+
case GICV5_ITS_DT_ITT_CFGR_L2SZ_4k:
115
+
default:
116
+
return ITS_L2SZ_4K_L2_BITS;
117
+
}
118
+
}
119
+
120
+
static int gicv5_its_itt_cache_inv(struct gicv5_its_chip_data *its, u32 device_id,
121
+
u16 event_id)
122
+
{
123
+
u32 eventr, eidr;
124
+
u64 didr;
125
+
126
+
didr = FIELD_PREP(GICV5_ITS_DIDR_DEVICEID, device_id);
127
+
eidr = FIELD_PREP(GICV5_ITS_EIDR_EVENTID, event_id);
128
+
eventr = FIELD_PREP(GICV5_ITS_INV_EVENTR_I, 0x1);
129
+
130
+
its_writeq_relaxed(its, didr, GICV5_ITS_DIDR);
131
+
its_writel_relaxed(its, eidr, GICV5_ITS_EIDR);
132
+
its_writel_relaxed(its, eventr, GICV5_ITS_INV_EVENTR);
133
+
134
+
return gicv5_its_cache_sync(its);
135
+
}
136
+
137
+
static void gicv5_its_free_itt_linear(struct gicv5_its_dev *its_dev)
138
+
{
139
+
kfree(its_dev->itt_cfg.linear.itt);
140
+
}
141
+
142
+
static void gicv5_its_free_itt_two_level(struct gicv5_its_dev *its_dev)
143
+
{
144
+
unsigned int i, num_ents = its_dev->itt_cfg.l2.num_l1_ents;
145
+
146
+
for (i = 0; i < num_ents; i++)
147
+
kfree(its_dev->itt_cfg.l2.l2ptrs[i]);
148
+
149
+
kfree(its_dev->itt_cfg.l2.l2ptrs);
150
+
kfree(its_dev->itt_cfg.l2.l1itt);
151
+
}
152
+
153
+
static void gicv5_its_free_itt(struct gicv5_its_dev *its_dev)
154
+
{
155
+
if (!its_dev->itt_cfg.l2itt)
156
+
gicv5_its_free_itt_linear(its_dev);
157
+
else
158
+
gicv5_its_free_itt_two_level(its_dev);
159
+
}
160
+
161
+
static int gicv5_its_create_itt_linear(struct gicv5_its_chip_data *its,
162
+
struct gicv5_its_dev *its_dev,
163
+
unsigned int event_id_bits)
164
+
{
165
+
unsigned int num_ents = BIT(event_id_bits);
166
+
__le64 *itt;
167
+
168
+
itt = kcalloc(num_ents, sizeof(*itt), GFP_KERNEL);
169
+
if (!itt)
170
+
return -ENOMEM;
171
+
172
+
its_dev->itt_cfg.linear.itt = itt;
173
+
its_dev->itt_cfg.linear.num_ents = num_ents;
174
+
its_dev->itt_cfg.l2itt = false;
175
+
its_dev->itt_cfg.event_id_bits = event_id_bits;
176
+
177
+
gicv5_its_dcache_clean(its, itt, num_ents * sizeof(*itt));
178
+
179
+
return 0;
180
+
}
181
+
182
+
/*
183
+
* Allocate a two-level ITT. All ITT entries are allocated in one go, unlike
184
+
* with the device table. Span may be used to limit the second level table
185
+
* size, where possible.
186
+
*/
187
+
static int gicv5_its_create_itt_two_level(struct gicv5_its_chip_data *its,
188
+
struct gicv5_its_dev *its_dev,
189
+
unsigned int event_id_bits,
190
+
unsigned int itt_l2sz,
191
+
unsigned int num_events)
192
+
{
193
+
unsigned int l1_bits, l2_bits, span, events_per_l2_table;
194
+
unsigned int i, complete_tables, final_span, num_ents;
195
+
__le64 *itt_l1, *itt_l2, **l2ptrs;
196
+
int ret;
197
+
u64 val;
198
+
199
+
ret = gicv5_its_l2sz_to_l2_bits(itt_l2sz);
200
+
if (ret >= event_id_bits) {
201
+
pr_debug("Incorrect l2sz (0x%x) for %u EventID bits. Cannot allocate ITT\n",
202
+
itt_l2sz, event_id_bits);
203
+
return -EINVAL;
204
+
}
205
+
206
+
l2_bits = ret;
207
+
208
+
l1_bits = event_id_bits - l2_bits;
209
+
210
+
num_ents = BIT(l1_bits);
211
+
212
+
itt_l1 = kcalloc(num_ents, sizeof(*itt_l1), GFP_KERNEL);
213
+
if (!itt_l1)
214
+
return -ENOMEM;
215
+
216
+
l2ptrs = kcalloc(num_ents, sizeof(*l2ptrs), GFP_KERNEL);
217
+
if (!l2ptrs) {
218
+
kfree(itt_l1);
219
+
return -ENOMEM;
220
+
}
221
+
222
+
its_dev->itt_cfg.l2.l2ptrs = l2ptrs;
223
+
224
+
its_dev->itt_cfg.l2.l2sz = itt_l2sz;
225
+
its_dev->itt_cfg.l2.l1itt = itt_l1;
226
+
its_dev->itt_cfg.l2.num_l1_ents = num_ents;
227
+
its_dev->itt_cfg.l2itt = true;
228
+
its_dev->itt_cfg.event_id_bits = event_id_bits;
229
+
230
+
/*
231
+
* Need to determine how many entries there are per L2 - this is based
232
+
* on the number of bits in the table.
233
+
*/
234
+
events_per_l2_table = BIT(l2_bits);
235
+
complete_tables = num_events / events_per_l2_table;
236
+
final_span = order_base_2(num_events % events_per_l2_table);
237
+
238
+
for (i = 0; i < num_ents; i++) {
239
+
size_t l2sz;
240
+
241
+
span = i == complete_tables ? final_span : l2_bits;
242
+
243
+
itt_l2 = kcalloc(BIT(span), sizeof(*itt_l2), GFP_KERNEL);
244
+
if (!itt_l2) {
245
+
ret = -ENOMEM;
246
+
goto out_free;
247
+
}
248
+
249
+
its_dev->itt_cfg.l2.l2ptrs[i] = itt_l2;
250
+
251
+
l2sz = BIT(span) * sizeof(*itt_l2);
252
+
253
+
gicv5_its_dcache_clean(its, itt_l2, l2sz);
254
+
255
+
val = (virt_to_phys(itt_l2) & GICV5_ITTL1E_L2_ADDR_MASK) |
256
+
FIELD_PREP(GICV5_ITTL1E_SPAN, span) |
257
+
FIELD_PREP(GICV5_ITTL1E_VALID, 0x1);
258
+
259
+
WRITE_ONCE(itt_l1[i], cpu_to_le64(val));
260
+
}
261
+
262
+
gicv5_its_dcache_clean(its, itt_l1, num_ents * sizeof(*itt_l1));
263
+
264
+
return 0;
265
+
266
+
out_free:
267
+
for (i = i - 1; i >= 0; i--)
268
+
kfree(its_dev->itt_cfg.l2.l2ptrs[i]);
269
+
270
+
kfree(its_dev->itt_cfg.l2.l2ptrs);
271
+
kfree(itt_l1);
272
+
return ret;
273
+
}
274
+
275
+
/*
276
+
* Function to check whether the device table or ITT table support
277
+
* a two-level table and if so depending on the number of id_bits
278
+
* requested, determine whether a two-level table is required.
279
+
*
280
+
* Return the 2-level size value if a two level table is deemed
281
+
* necessary.
282
+
*/
283
+
static bool gicv5_its_l2sz_two_level(bool devtab, u32 its_idr1, u8 id_bits, u8 *sz)
284
+
{
285
+
unsigned int l2_bits, l2_sz;
286
+
287
+
if (devtab && !FIELD_GET(GICV5_ITS_IDR1_DT_LEVELS, its_idr1))
288
+
return false;
289
+
290
+
if (!devtab && !FIELD_GET(GICV5_ITS_IDR1_ITT_LEVELS, its_idr1))
291
+
return false;
292
+
293
+
/*
294
+
* Pick an L2 size that matches the pagesize; if a match
295
+
* is not found, go for the smallest supported l2 size granule.
296
+
*
297
+
* This ensures that we will always be able to allocate
298
+
* contiguous memory at L2.
299
+
*/
300
+
switch (PAGE_SIZE) {
301
+
case SZ_64K:
302
+
if (GICV5_ITS_IDR1_L2SZ_SUPPORT_64KB(its_idr1)) {
303
+
l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_64k;
304
+
break;
305
+
}
306
+
fallthrough;
307
+
case SZ_4K:
308
+
if (GICV5_ITS_IDR1_L2SZ_SUPPORT_4KB(its_idr1)) {
309
+
l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_4k;
310
+
break;
311
+
}
312
+
fallthrough;
313
+
case SZ_16K:
314
+
if (GICV5_ITS_IDR1_L2SZ_SUPPORT_16KB(its_idr1)) {
315
+
l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_16k;
316
+
break;
317
+
}
318
+
if (GICV5_ITS_IDR1_L2SZ_SUPPORT_4KB(its_idr1)) {
319
+
l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_4k;
320
+
break;
321
+
}
322
+
if (GICV5_ITS_IDR1_L2SZ_SUPPORT_64KB(its_idr1)) {
323
+
l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_64k;
324
+
break;
325
+
}
326
+
327
+
l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_4k;
328
+
break;
329
+
}
330
+
331
+
l2_bits = gicv5_its_l2sz_to_l2_bits(l2_sz);
332
+
333
+
if (l2_bits > id_bits)
334
+
return false;
335
+
336
+
*sz = l2_sz;
337
+
338
+
return true;
339
+
}
340
+
341
+
static __le64 *gicv5_its_device_get_itte_ref(struct gicv5_its_dev *its_dev,
342
+
u16 event_id)
343
+
{
344
+
unsigned int l1_idx, l2_idx, l2_bits;
345
+
__le64 *l2_itt;
346
+
347
+
if (!its_dev->itt_cfg.l2itt) {
348
+
__le64 *itt = its_dev->itt_cfg.linear.itt;
349
+
350
+
return &itt[event_id];
351
+
}
352
+
353
+
l2_bits = gicv5_its_l2sz_to_l2_bits(its_dev->itt_cfg.l2.l2sz);
354
+
l1_idx = event_id >> l2_bits;
355
+
l2_idx = event_id & GENMASK(l2_bits - 1, 0);
356
+
l2_itt = its_dev->itt_cfg.l2.l2ptrs[l1_idx];
357
+
358
+
return &l2_itt[l2_idx];
359
+
}
360
+
361
+
static int gicv5_its_device_cache_inv(struct gicv5_its_chip_data *its,
362
+
struct gicv5_its_dev *its_dev)
363
+
{
364
+
u32 devicer;
365
+
u64 didr;
366
+
367
+
didr = FIELD_PREP(GICV5_ITS_DIDR_DEVICEID, its_dev->device_id);
368
+
devicer = FIELD_PREP(GICV5_ITS_INV_DEVICER_I, 0x1) |
369
+
FIELD_PREP(GICV5_ITS_INV_DEVICER_EVENTID_BITS,
370
+
its_dev->itt_cfg.event_id_bits) |
371
+
FIELD_PREP(GICV5_ITS_INV_DEVICER_L1, 0x0);
372
+
its_writeq_relaxed(its, didr, GICV5_ITS_DIDR);
373
+
its_writel_relaxed(its, devicer, GICV5_ITS_INV_DEVICER);
374
+
375
+
return gicv5_its_cache_sync(its);
376
+
}
377
+
378
+
/*
379
+
* Allocate a level 2 device table entry, update L1 parent to reference it.
380
+
* Only used for 2-level device tables, and it is called on demand.
381
+
*/
382
+
static int gicv5_its_alloc_l2_devtab(struct gicv5_its_chip_data *its,
383
+
unsigned int l1_index)
384
+
{
385
+
__le64 *l2devtab, *l1devtab = its->devtab_cfgr.l2.l1devtab;
386
+
u8 span, l2sz, l2_bits;
387
+
u64 l1dte;
388
+
389
+
if (FIELD_GET(GICV5_DTL1E_VALID, le64_to_cpu(l1devtab[l1_index])))
390
+
return 0;
391
+
392
+
span = FIELD_GET(GICV5_DTL1E_SPAN, le64_to_cpu(l1devtab[l1_index]));
393
+
l2sz = devtab_cfgr_field(its, L2SZ);
394
+
395
+
l2_bits = gicv5_its_l2sz_to_l2_bits(l2sz);
396
+
397
+
/*
398
+
* Span allows us to create a smaller L2 device table.
399
+
* If it is too large, use the number of allowed L2 bits.
400
+
*/
401
+
if (span > l2_bits)
402
+
span = l2_bits;
403
+
404
+
l2devtab = kcalloc(BIT(span), sizeof(*l2devtab), GFP_KERNEL);
405
+
if (!l2devtab)
406
+
return -ENOMEM;
407
+
408
+
its->devtab_cfgr.l2.l2ptrs[l1_index] = l2devtab;
409
+
410
+
l1dte = FIELD_PREP(GICV5_DTL1E_SPAN, span) |
411
+
(virt_to_phys(l2devtab) & GICV5_DTL1E_L2_ADDR_MASK) |
412
+
FIELD_PREP(GICV5_DTL1E_VALID, 0x1);
413
+
its_write_table_entry(its, &l1devtab[l1_index], l1dte);
414
+
415
+
return 0;
416
+
}
417
+
418
+
static __le64 *gicv5_its_devtab_get_dte_ref(struct gicv5_its_chip_data *its,
419
+
u32 device_id, bool alloc)
420
+
{
421
+
u8 str = devtab_cfgr_field(its, STRUCTURE);
422
+
unsigned int l2sz, l2_bits, l1_idx, l2_idx;
423
+
__le64 *l2devtab;
424
+
int ret;
425
+
426
+
if (str == GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR) {
427
+
l2devtab = its->devtab_cfgr.linear.devtab;
428
+
return &l2devtab[device_id];
429
+
}
430
+
431
+
l2sz = devtab_cfgr_field(its, L2SZ);
432
+
l2_bits = gicv5_its_l2sz_to_l2_bits(l2sz);
433
+
l1_idx = device_id >> l2_bits;
434
+
l2_idx = device_id & GENMASK(l2_bits - 1, 0);
435
+
436
+
if (alloc) {
437
+
/*
438
+
* Allocate a new L2 device table here before
439
+
* continuing. We make the assumption that the span in
440
+
* the L1 table has been set correctly, and blindly use
441
+
* that value.
442
+
*/
443
+
ret = gicv5_its_alloc_l2_devtab(its, l1_idx);
444
+
if (ret)
445
+
return NULL;
446
+
}
447
+
448
+
l2devtab = its->devtab_cfgr.l2.l2ptrs[l1_idx];
449
+
return &l2devtab[l2_idx];
450
+
}
451
+
452
+
/*
453
+
* Register a new device in the device table. Allocate an ITT and
454
+
* program the L2DTE entry according to the ITT structure that
455
+
* was chosen.
456
+
*/
457
+
static int gicv5_its_device_register(struct gicv5_its_chip_data *its,
458
+
struct gicv5_its_dev *its_dev)
459
+
{
460
+
u8 event_id_bits, device_id_bits, itt_struct, itt_l2sz;
461
+
phys_addr_t itt_phys_base;
462
+
bool two_level_itt;
463
+
u32 idr1, idr2;
464
+
__le64 *dte;
465
+
u64 val;
466
+
int ret;
467
+
468
+
device_id_bits = devtab_cfgr_field(its, DEVICEID_BITS);
469
+
470
+
if (its_dev->device_id >= BIT(device_id_bits)) {
471
+
pr_err("Supplied DeviceID (%u) outside of Device Table range (%u)!",
472
+
its_dev->device_id, (u32)GENMASK(device_id_bits - 1, 0));
473
+
return -EINVAL;
474
+
}
475
+
476
+
dte = gicv5_its_devtab_get_dte_ref(its, its_dev->device_id, true);
477
+
if (!dte)
478
+
return -ENOMEM;
479
+
480
+
if (FIELD_GET(GICV5_DTL2E_VALID, le64_to_cpu(*dte)))
481
+
return -EBUSY;
482
+
483
+
/*
484
+
* Determine how many bits we need, validate those against the max.
485
+
* Based on these, determine if we should go for a 1- or 2-level ITT.
486
+
*/
487
+
event_id_bits = order_base_2(its_dev->num_events);
488
+
489
+
idr2 = its_readl_relaxed(its, GICV5_ITS_IDR2);
490
+
491
+
if (event_id_bits > FIELD_GET(GICV5_ITS_IDR2_EVENTID_BITS, idr2)) {
492
+
pr_err("Required EventID bits (%u) larger than supported bits (%u)!",
493
+
event_id_bits,
494
+
(u8)FIELD_GET(GICV5_ITS_IDR2_EVENTID_BITS, idr2));
495
+
return -EINVAL;
496
+
}
497
+
498
+
idr1 = its_readl_relaxed(its, GICV5_ITS_IDR1);
499
+
500
+
/*
501
+
* L2 ITT size is programmed into the L2DTE regardless of
502
+
* whether a two-level or linear ITT is built, init it.
503
+
*/
504
+
itt_l2sz = 0;
505
+
506
+
two_level_itt = gicv5_its_l2sz_two_level(false, idr1, event_id_bits,
507
+
&itt_l2sz);
508
+
if (two_level_itt)
509
+
ret = gicv5_its_create_itt_two_level(its, its_dev, event_id_bits,
510
+
itt_l2sz,
511
+
its_dev->num_events);
512
+
else
513
+
ret = gicv5_its_create_itt_linear(its, its_dev, event_id_bits);
514
+
if (ret)
515
+
return ret;
516
+
517
+
itt_phys_base = two_level_itt ? virt_to_phys(its_dev->itt_cfg.l2.l1itt) :
518
+
virt_to_phys(its_dev->itt_cfg.linear.itt);
519
+
520
+
itt_struct = two_level_itt ? GICV5_ITS_DT_ITT_CFGR_STRUCTURE_TWO_LEVEL :
521
+
GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR;
522
+
523
+
val = FIELD_PREP(GICV5_DTL2E_EVENT_ID_BITS, event_id_bits) |
524
+
FIELD_PREP(GICV5_DTL2E_ITT_STRUCTURE, itt_struct) |
525
+
(itt_phys_base & GICV5_DTL2E_ITT_ADDR_MASK) |
526
+
FIELD_PREP(GICV5_DTL2E_ITT_L2SZ, itt_l2sz) |
527
+
FIELD_PREP(GICV5_DTL2E_VALID, 0x1);
528
+
529
+
its_write_table_entry(its, dte, val);
530
+
531
+
ret = gicv5_its_device_cache_inv(its, its_dev);
532
+
if (ret) {
533
+
its_write_table_entry(its, dte, 0);
534
+
gicv5_its_free_itt(its_dev);
535
+
return ret;
536
+
}
537
+
538
+
return 0;
539
+
}
540
+
541
+
/*
542
+
* Unregister a device in the device table. Lookup the device by ID, free the
543
+
* corresponding ITT, mark the device as invalid in the device table.
544
+
*/
545
+
static int gicv5_its_device_unregister(struct gicv5_its_chip_data *its,
546
+
struct gicv5_its_dev *its_dev)
547
+
{
548
+
__le64 *dte;
549
+
550
+
dte = gicv5_its_devtab_get_dte_ref(its, its_dev->device_id, false);
551
+
552
+
if (!FIELD_GET(GICV5_DTL2E_VALID, le64_to_cpu(*dte))) {
553
+
pr_debug("Device table entry for DeviceID 0x%x is not valid. Nothing to clean up!",
554
+
its_dev->device_id);
555
+
return -EINVAL;
556
+
}
557
+
558
+
/* Zero everything - make it clear that this is an invalid entry */
559
+
its_write_table_entry(its, dte, 0);
560
+
561
+
gicv5_its_free_itt(its_dev);
562
+
563
+
return gicv5_its_device_cache_inv(its, its_dev);
564
+
}
565
+
566
+
/*
567
+
* Allocate a 1-level device table. All entries are allocated, but marked
568
+
* invalid.
569
+
*/
570
+
static int gicv5_its_alloc_devtab_linear(struct gicv5_its_chip_data *its,
571
+
u8 device_id_bits)
572
+
{
573
+
__le64 *devtab;
574
+
size_t sz;
575
+
u64 baser;
576
+
u32 cfgr;
577
+
578
+
/*
579
+
* We expect a GICv5 implementation requiring a large number of
580
+
* deviceID bits to support a 2-level device table. If that's not
581
+
* the case, cap the number of deviceIDs supported according to the
582
+
* kmalloc limits so that the system can chug along with a linear
583
+
* device table.
584
+
*/
585
+
sz = BIT_ULL(device_id_bits) * sizeof(*devtab);
586
+
if (sz > KMALLOC_MAX_SIZE) {
587
+
u8 device_id_cap = ilog2(KMALLOC_MAX_SIZE/sizeof(*devtab));
588
+
589
+
pr_warn("Limiting device ID bits from %u to %u\n",
590
+
device_id_bits, device_id_cap);
591
+
device_id_bits = device_id_cap;
592
+
}
593
+
594
+
devtab = kcalloc(BIT(device_id_bits), sizeof(*devtab), GFP_KERNEL);
595
+
if (!devtab)
596
+
return -ENOMEM;
597
+
598
+
gicv5_its_dcache_clean(its, devtab, sz);
599
+
600
+
cfgr = FIELD_PREP(GICV5_ITS_DT_CFGR_STRUCTURE,
601
+
GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR) |
602
+
FIELD_PREP(GICV5_ITS_DT_CFGR_L2SZ, 0) |
603
+
FIELD_PREP(GICV5_ITS_DT_CFGR_DEVICEID_BITS, device_id_bits);
604
+
its_writel_relaxed(its, cfgr, GICV5_ITS_DT_CFGR);
605
+
606
+
baser = virt_to_phys(devtab) & GICV5_ITS_DT_BASER_ADDR_MASK;
607
+
its_writeq_relaxed(its, baser, GICV5_ITS_DT_BASER);
608
+
609
+
its->devtab_cfgr.cfgr = cfgr;
610
+
its->devtab_cfgr.linear.devtab = devtab;
611
+
612
+
return 0;
613
+
}
614
+
615
+
/*
616
+
* Allocate a 2-level device table. L2 entries are not allocated,
617
+
* they are allocated on-demand.
618
+
*/
619
+
static int gicv5_its_alloc_devtab_two_level(struct gicv5_its_chip_data *its,
620
+
u8 device_id_bits,
621
+
u8 devtab_l2sz)
622
+
{
623
+
unsigned int l1_bits, l2_bits, i;
624
+
__le64 *l1devtab, **l2ptrs;
625
+
size_t l1_sz;
626
+
u64 baser;
627
+
u32 cfgr;
628
+
629
+
l2_bits = gicv5_its_l2sz_to_l2_bits(devtab_l2sz);
630
+
631
+
l1_bits = device_id_bits - l2_bits;
632
+
l1_sz = BIT(l1_bits) * sizeof(*l1devtab);
633
+
/*
634
+
* With 2-level device table support it is highly unlikely
635
+
* that we are not able to allocate the required amount of
636
+
* device table memory to cover deviceID space; cap the
637
+
* deviceID space if we encounter such set-up.
638
+
* If this ever becomes a problem we could revisit the policy
639
+
* behind level 2 size selection to reduce level-1 deviceID bits.
640
+
*/
641
+
if (l1_sz > KMALLOC_MAX_SIZE) {
642
+
l1_bits = ilog2(KMALLOC_MAX_SIZE/sizeof(*l1devtab));
643
+
644
+
pr_warn("Limiting device ID bits from %u to %u\n",
645
+
device_id_bits, l1_bits + l2_bits);
646
+
device_id_bits = l1_bits + l2_bits;
647
+
l1_sz = KMALLOC_MAX_SIZE;
648
+
}
649
+
650
+
l1devtab = kcalloc(BIT(l1_bits), sizeof(*l1devtab), GFP_KERNEL);
651
+
if (!l1devtab)
652
+
return -ENOMEM;
653
+
654
+
l2ptrs = kcalloc(BIT(l1_bits), sizeof(*l2ptrs), GFP_KERNEL);
655
+
if (!l2ptrs) {
656
+
kfree(l1devtab);
657
+
return -ENOMEM;
658
+
}
659
+
660
+
for (i = 0; i < BIT(l1_bits); i++)
661
+
l1devtab[i] = cpu_to_le64(FIELD_PREP(GICV5_DTL1E_SPAN, l2_bits));
662
+
663
+
gicv5_its_dcache_clean(its, l1devtab, l1_sz);
664
+
665
+
cfgr = FIELD_PREP(GICV5_ITS_DT_CFGR_STRUCTURE,
666
+
GICV5_ITS_DT_ITT_CFGR_STRUCTURE_TWO_LEVEL) |
667
+
FIELD_PREP(GICV5_ITS_DT_CFGR_L2SZ, devtab_l2sz) |
668
+
FIELD_PREP(GICV5_ITS_DT_CFGR_DEVICEID_BITS, device_id_bits);
669
+
its_writel_relaxed(its, cfgr, GICV5_ITS_DT_CFGR);
670
+
671
+
baser = virt_to_phys(l1devtab) & GICV5_ITS_DT_BASER_ADDR_MASK;
672
+
its_writeq_relaxed(its, baser, GICV5_ITS_DT_BASER);
673
+
674
+
its->devtab_cfgr.cfgr = cfgr;
675
+
its->devtab_cfgr.l2.l1devtab = l1devtab;
676
+
its->devtab_cfgr.l2.l2ptrs = l2ptrs;
677
+
678
+
return 0;
679
+
}
680
+
681
+
/*
682
+
* Initialise the device table as either 1- or 2-level depending on what is
683
+
* supported by the hardware.
684
+
*/
685
+
static int gicv5_its_init_devtab(struct gicv5_its_chip_data *its)
686
+
{
687
+
u8 device_id_bits, devtab_l2sz;
688
+
bool two_level_devtab;
689
+
u32 idr1;
690
+
691
+
idr1 = its_readl_relaxed(its, GICV5_ITS_IDR1);
692
+
693
+
device_id_bits = FIELD_GET(GICV5_ITS_IDR1_DEVICEID_BITS, idr1);
694
+
two_level_devtab = gicv5_its_l2sz_two_level(true, idr1, device_id_bits,
695
+
&devtab_l2sz);
696
+
if (two_level_devtab)
697
+
return gicv5_its_alloc_devtab_two_level(its, device_id_bits,
698
+
devtab_l2sz);
699
+
else
700
+
return gicv5_its_alloc_devtab_linear(its, device_id_bits);
701
+
}
702
+
703
+
static void gicv5_its_deinit_devtab(struct gicv5_its_chip_data *its)
704
+
{
705
+
u8 str = devtab_cfgr_field(its, STRUCTURE);
706
+
707
+
if (str == GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR) {
708
+
kfree(its->devtab_cfgr.linear.devtab);
709
+
} else {
710
+
kfree(its->devtab_cfgr.l2.l1devtab);
711
+
kfree(its->devtab_cfgr.l2.l2ptrs);
712
+
}
713
+
}
714
+
715
+
static void gicv5_its_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
716
+
{
717
+
struct gicv5_its_dev *its_dev = irq_data_get_irq_chip_data(d);
718
+
u64 addr = its_dev->its_trans_phys_base;
719
+
720
+
msg->data = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq);
721
+
msi_msg_set_addr(irq_data_get_msi_desc(d), msg, addr);
722
+
}
723
+
724
+
static const struct irq_chip gicv5_its_irq_chip = {
725
+
.name = "GICv5-ITS-MSI",
726
+
.irq_mask = irq_chip_mask_parent,
727
+
.irq_unmask = irq_chip_unmask_parent,
728
+
.irq_eoi = irq_chip_eoi_parent,
729
+
.irq_set_affinity = irq_chip_set_affinity_parent,
730
+
.irq_get_irqchip_state = irq_chip_get_parent_state,
731
+
.irq_set_irqchip_state = irq_chip_set_parent_state,
732
+
.irq_compose_msi_msg = gicv5_its_compose_msi_msg,
733
+
};
734
+
735
+
static struct gicv5_its_dev *gicv5_its_find_device(struct gicv5_its_chip_data *its,
736
+
u32 device_id)
737
+
{
738
+
struct gicv5_its_dev *dev = xa_load(&its->its_devices, device_id);
739
+
740
+
return dev ? dev : ERR_PTR(-ENODEV);
741
+
}
742
+
743
+
static struct gicv5_its_dev *gicv5_its_alloc_device(struct gicv5_its_chip_data *its, int nvec,
744
+
u32 dev_id)
745
+
{
746
+
struct gicv5_its_dev *its_dev;
747
+
void *entry;
748
+
int ret;
749
+
750
+
its_dev = gicv5_its_find_device(its, dev_id);
751
+
if (!IS_ERR(its_dev)) {
752
+
pr_err("A device with this DeviceID (0x%x) has already been registered.\n",
753
+
dev_id);
754
+
755
+
return ERR_PTR(-EBUSY);
756
+
}
757
+
758
+
its_dev = kzalloc(sizeof(*its_dev), GFP_KERNEL);
759
+
if (!its_dev)
760
+
return ERR_PTR(-ENOMEM);
761
+
762
+
its_dev->device_id = dev_id;
763
+
its_dev->num_events = nvec;
764
+
765
+
ret = gicv5_its_device_register(its, its_dev);
766
+
if (ret) {
767
+
pr_err("Failed to register the device\n");
768
+
goto out_dev_free;
769
+
}
770
+
771
+
gicv5_its_device_cache_inv(its, its_dev);
772
+
773
+
its_dev->its_node = its;
774
+
775
+
its_dev->event_map = (unsigned long *)bitmap_zalloc(its_dev->num_events, GFP_KERNEL);
776
+
if (!its_dev->event_map) {
777
+
ret = -ENOMEM;
778
+
goto out_unregister;
779
+
}
780
+
781
+
entry = xa_store(&its->its_devices, dev_id, its_dev, GFP_KERNEL);
782
+
if (xa_is_err(entry)) {
783
+
ret = xa_err(entry);
784
+
goto out_bitmap_free;
785
+
}
786
+
787
+
return its_dev;
788
+
789
+
out_bitmap_free:
790
+
bitmap_free(its_dev->event_map);
791
+
out_unregister:
792
+
gicv5_its_device_unregister(its, its_dev);
793
+
out_dev_free:
794
+
kfree(its_dev);
795
+
return ERR_PTR(ret);
796
+
}
797
+
798
+
static int gicv5_its_msi_prepare(struct irq_domain *domain, struct device *dev,
799
+
int nvec, msi_alloc_info_t *info)
800
+
{
801
+
u32 dev_id = info->scratchpad[0].ul;
802
+
struct msi_domain_info *msi_info;
803
+
struct gicv5_its_chip_data *its;
804
+
struct gicv5_its_dev *its_dev;
805
+
806
+
msi_info = msi_get_domain_info(domain);
807
+
its = msi_info->data;
808
+
809
+
guard(mutex)(&its->dev_alloc_lock);
810
+
811
+
its_dev = gicv5_its_alloc_device(its, nvec, dev_id);
812
+
if (IS_ERR(its_dev))
813
+
return PTR_ERR(its_dev);
814
+
815
+
its_dev->its_trans_phys_base = info->scratchpad[1].ul;
816
+
info->scratchpad[0].ptr = its_dev;
817
+
818
+
return 0;
819
+
}
820
+
821
+
static void gicv5_its_msi_teardown(struct irq_domain *domain, msi_alloc_info_t *info)
822
+
{
823
+
struct gicv5_its_dev *its_dev = info->scratchpad[0].ptr;
824
+
struct msi_domain_info *msi_info;
825
+
struct gicv5_its_chip_data *its;
826
+
827
+
msi_info = msi_get_domain_info(domain);
828
+
its = msi_info->data;
829
+
830
+
guard(mutex)(&its->dev_alloc_lock);
831
+
832
+
if (WARN_ON_ONCE(!bitmap_empty(its_dev->event_map, its_dev->num_events)))
833
+
return;
834
+
835
+
xa_erase(&its->its_devices, its_dev->device_id);
836
+
bitmap_free(its_dev->event_map);
837
+
gicv5_its_device_unregister(its, its_dev);
838
+
kfree(its_dev);
839
+
}
840
+
841
+
static struct msi_domain_ops gicv5_its_msi_domain_ops = {
842
+
.msi_prepare = gicv5_its_msi_prepare,
843
+
.msi_teardown = gicv5_its_msi_teardown,
844
+
};
845
+
846
+
static int gicv5_its_map_event(struct gicv5_its_dev *its_dev, u16 event_id, u32 lpi)
847
+
{
848
+
struct gicv5_its_chip_data *its = its_dev->its_node;
849
+
u64 itt_entry;
850
+
__le64 *itte;
851
+
852
+
itte = gicv5_its_device_get_itte_ref(its_dev, event_id);
853
+
854
+
if (FIELD_GET(GICV5_ITTL2E_VALID, *itte))
855
+
return -EEXIST;
856
+
857
+
itt_entry = FIELD_PREP(GICV5_ITTL2E_LPI_ID, lpi) |
858
+
FIELD_PREP(GICV5_ITTL2E_VALID, 0x1);
859
+
860
+
its_write_table_entry(its, itte, itt_entry);
861
+
862
+
gicv5_its_itt_cache_inv(its, its_dev->device_id, event_id);
863
+
864
+
return 0;
865
+
}
866
+
867
+
static void gicv5_its_unmap_event(struct gicv5_its_dev *its_dev, u16 event_id)
868
+
{
869
+
struct gicv5_its_chip_data *its = its_dev->its_node;
870
+
u64 itte_val;
871
+
__le64 *itte;
872
+
873
+
itte = gicv5_its_device_get_itte_ref(its_dev, event_id);
874
+
875
+
itte_val = le64_to_cpu(*itte);
876
+
itte_val &= ~GICV5_ITTL2E_VALID;
877
+
878
+
its_write_table_entry(its, itte, itte_val);
879
+
880
+
gicv5_its_itt_cache_inv(its, its_dev->device_id, event_id);
881
+
}
882
+
883
+
static int gicv5_its_alloc_eventid(struct gicv5_its_dev *its_dev, msi_alloc_info_t *info,
884
+
unsigned int nr_irqs, u32 *eventid)
885
+
{
886
+
int event_id_base;
887
+
888
+
if (!(info->flags & MSI_ALLOC_FLAGS_FIXED_MSG_DATA)) {
889
+
event_id_base = bitmap_find_free_region(its_dev->event_map,
890
+
its_dev->num_events,
891
+
get_count_order(nr_irqs));
892
+
if (event_id_base < 0)
893
+
return event_id_base;
894
+
} else {
895
+
/*
896
+
* We want to have a fixed EventID mapped for hardcoded
897
+
* message data allocations.
898
+
*/
899
+
if (WARN_ON_ONCE(nr_irqs != 1))
900
+
return -EINVAL;
901
+
902
+
event_id_base = info->hwirq;
903
+
904
+
if (event_id_base >= its_dev->num_events) {
905
+
pr_err("EventID ouside of ITT range; cannot allocate an ITT entry!\n");
906
+
907
+
return -EINVAL;
908
+
}
909
+
910
+
if (test_and_set_bit(event_id_base, its_dev->event_map)) {
911
+
pr_warn("Can't reserve event_id bitmap\n");
912
+
return -EINVAL;
913
+
914
+
}
915
+
}
916
+
917
+
*eventid = event_id_base;
918
+
919
+
return 0;
920
+
}
921
+
922
+
static void gicv5_its_free_eventid(struct gicv5_its_dev *its_dev, u32 event_id_base,
923
+
unsigned int nr_irqs)
924
+
{
925
+
bitmap_release_region(its_dev->event_map, event_id_base,
926
+
get_count_order(nr_irqs));
927
+
}
928
+
929
+
static int gicv5_its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
930
+
unsigned int nr_irqs, void *arg)
931
+
{
932
+
u32 device_id, event_id_base, lpi;
933
+
struct gicv5_its_dev *its_dev;
934
+
msi_alloc_info_t *info = arg;
935
+
irq_hw_number_t hwirq;
936
+
struct irq_data *irqd;
937
+
int ret, i;
938
+
939
+
its_dev = info->scratchpad[0].ptr;
940
+
941
+
ret = gicv5_its_alloc_eventid(its_dev, info, nr_irqs, &event_id_base);
942
+
if (ret)
943
+
return ret;
944
+
945
+
ret = iommu_dma_prepare_msi(info->desc, its_dev->its_trans_phys_base);
946
+
if (ret)
947
+
goto out_eventid;
948
+
949
+
device_id = its_dev->device_id;
950
+
951
+
for (i = 0; i < nr_irqs; i++) {
952
+
lpi = gicv5_alloc_lpi();
953
+
if (ret < 0) {
954
+
pr_debug("Failed to find free LPI!\n");
955
+
goto out_eventid;
956
+
}
957
+
958
+
ret = irq_domain_alloc_irqs_parent(domain, virq + i, 1, &lpi);
959
+
if (ret)
960
+
goto out_free_lpi;
961
+
962
+
/*
963
+
* Store eventid and deviceid into the hwirq for later use.
964
+
*
965
+
* hwirq = event_id << 32 | device_id
966
+
*/
967
+
hwirq = FIELD_PREP(GICV5_ITS_HWIRQ_DEVICE_ID, device_id) |
968
+
FIELD_PREP(GICV5_ITS_HWIRQ_EVENT_ID, (u64)event_id_base + i);
969
+
irq_domain_set_info(domain, virq + i, hwirq,
970
+
&gicv5_its_irq_chip, its_dev,
971
+
handle_fasteoi_irq, NULL, NULL);
972
+
973
+
irqd = irq_get_irq_data(virq + i);
974
+
irqd_set_single_target(irqd);
975
+
irqd_set_affinity_on_activate(irqd);
976
+
irqd_set_resend_when_in_progress(irqd);
977
+
}
978
+
979
+
return 0;
980
+
981
+
out_free_lpi:
982
+
gicv5_free_lpi(lpi);
983
+
out_eventid:
984
+
gicv5_its_free_eventid(its_dev, event_id_base, nr_irqs);
985
+
return ret;
986
+
}
987
+
988
+
static void gicv5_its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
989
+
unsigned int nr_irqs)
990
+
{
991
+
struct irq_data *d = irq_domain_get_irq_data(domain, virq);
992
+
struct gicv5_its_chip_data *its;
993
+
struct gicv5_its_dev *its_dev;
994
+
u16 event_id_base;
995
+
unsigned int i;
996
+
997
+
its_dev = irq_data_get_irq_chip_data(d);
998
+
its = its_dev->its_node;
999
+
1000
+
event_id_base = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq);
1001
+
1002
+
bitmap_release_region(its_dev->event_map, event_id_base,
1003
+
get_count_order(nr_irqs));
1004
+
1005
+
/* Hierarchically free irq data */
1006
+
for (i = 0; i < nr_irqs; i++) {
1007
+
d = irq_domain_get_irq_data(domain, virq + i);
1008
+
1009
+
gicv5_free_lpi(d->parent_data->hwirq);
1010
+
irq_domain_reset_irq_data(d);
1011
+
irq_domain_free_irqs_parent(domain, virq + i, 1);
1012
+
}
1013
+
1014
+
gicv5_its_syncr(its, its_dev);
1015
+
gicv5_irs_syncr();
1016
+
}
1017
+
1018
+
static int gicv5_its_irq_domain_activate(struct irq_domain *domain, struct irq_data *d,
1019
+
bool reserve)
1020
+
{
1021
+
struct gicv5_its_dev *its_dev = irq_data_get_irq_chip_data(d);
1022
+
u16 event_id;
1023
+
u32 lpi;
1024
+
1025
+
event_id = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq);
1026
+
lpi = d->parent_data->hwirq;
1027
+
1028
+
return gicv5_its_map_event(its_dev, event_id, lpi);
1029
+
}
1030
+
1031
+
static void gicv5_its_irq_domain_deactivate(struct irq_domain *domain,
1032
+
struct irq_data *d)
1033
+
{
1034
+
struct gicv5_its_dev *its_dev = irq_data_get_irq_chip_data(d);
1035
+
u16 event_id;
1036
+
1037
+
event_id = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq);
1038
+
1039
+
gicv5_its_unmap_event(its_dev, event_id);
1040
+
}
1041
+
1042
+
static const struct irq_domain_ops gicv5_its_irq_domain_ops = {
1043
+
.alloc = gicv5_its_irq_domain_alloc,
1044
+
.free = gicv5_its_irq_domain_free,
1045
+
.activate = gicv5_its_irq_domain_activate,
1046
+
.deactivate = gicv5_its_irq_domain_deactivate,
1047
+
.select = msi_lib_irq_domain_select,
1048
+
};
1049
+
1050
+
static int gicv5_its_write_cr0(struct gicv5_its_chip_data *its, bool enable)
1051
+
{
1052
+
u32 cr0 = FIELD_PREP(GICV5_ITS_CR0_ITSEN, enable);
1053
+
1054
+
its_writel_relaxed(its, cr0, GICV5_ITS_CR0);
1055
+
return gicv5_wait_for_op_atomic(its->its_base, GICV5_ITS_CR0,
1056
+
GICV5_ITS_CR0_IDLE, NULL);
1057
+
}
1058
+
1059
+
static int gicv5_its_enable(struct gicv5_its_chip_data *its)
1060
+
{
1061
+
return gicv5_its_write_cr0(its, true);
1062
+
}
1063
+
1064
+
static int gicv5_its_disable(struct gicv5_its_chip_data *its)
1065
+
{
1066
+
return gicv5_its_write_cr0(its, false);
1067
+
}
1068
+
1069
+
static void gicv5_its_print_info(struct gicv5_its_chip_data *its_node)
1070
+
{
1071
+
bool devtab_linear;
1072
+
u8 device_id_bits;
1073
+
u8 str;
1074
+
1075
+
device_id_bits = devtab_cfgr_field(its_node, DEVICEID_BITS);
1076
+
1077
+
str = devtab_cfgr_field(its_node, STRUCTURE);
1078
+
devtab_linear = (str == GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR);
1079
+
1080
+
pr_info("ITS %s enabled using %s device table device_id_bits %u\n",
1081
+
fwnode_get_name(its_node->fwnode),
1082
+
devtab_linear ? "linear" : "2-level",
1083
+
device_id_bits);
1084
+
}
1085
+
1086
+
static int gicv5_its_init_domain(struct gicv5_its_chip_data *its, struct irq_domain *parent)
1087
+
{
1088
+
struct irq_domain_info dom_info = {
1089
+
.fwnode = its->fwnode,
1090
+
.ops = &gicv5_its_irq_domain_ops,
1091
+
.domain_flags = its->msi_domain_flags,
1092
+
.parent = parent,
1093
+
};
1094
+
struct msi_domain_info *info;
1095
+
1096
+
info = kzalloc(sizeof(*info), GFP_KERNEL);
1097
+
if (!info)
1098
+
return -ENOMEM;
1099
+
1100
+
info->ops = &gicv5_its_msi_domain_ops;
1101
+
info->data = its;
1102
+
dom_info.host_data = info;
1103
+
1104
+
if (!msi_create_parent_irq_domain(&dom_info, &gic_v5_its_msi_parent_ops)) {
1105
+
kfree(info);
1106
+
return -ENOMEM;
1107
+
}
1108
+
1109
+
return 0;
1110
+
}
1111
+
1112
+
static int __init gicv5_its_init_bases(void __iomem *its_base, struct fwnode_handle *handle,
1113
+
struct irq_domain *parent_domain)
1114
+
{
1115
+
struct device_node *np = to_of_node(handle);
1116
+
struct gicv5_its_chip_data *its_node;
1117
+
u32 cr0, cr1;
1118
+
bool enabled;
1119
+
int ret;
1120
+
1121
+
its_node = kzalloc(sizeof(*its_node), GFP_KERNEL);
1122
+
if (!its_node)
1123
+
return -ENOMEM;
1124
+
1125
+
mutex_init(&its_node->dev_alloc_lock);
1126
+
xa_init(&its_node->its_devices);
1127
+
its_node->fwnode = handle;
1128
+
its_node->its_base = its_base;
1129
+
its_node->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI |
1130
+
IRQ_DOMAIN_FLAG_FWNODE_PARENT;
1131
+
1132
+
cr0 = its_readl_relaxed(its_node, GICV5_ITS_CR0);
1133
+
enabled = FIELD_GET(GICV5_ITS_CR0_ITSEN, cr0);
1134
+
if (WARN(enabled, "ITS %s enabled, disabling it before proceeding\n", np->full_name)) {
1135
+
ret = gicv5_its_disable(its_node);
1136
+
if (ret)
1137
+
goto out_free_node;
1138
+
}
1139
+
1140
+
if (of_property_read_bool(np, "dma-noncoherent")) {
1141
+
/*
1142
+
* A non-coherent ITS implies that some cache levels cannot be
1143
+
* used coherently by the cores and GIC. Our only option is to mark
1144
+
* memory attributes for the GIC as non-cacheable; by default,
1145
+
* non-cacheable memory attributes imply outer-shareable
1146
+
* shareability, the value written into ITS_CR1_SH is ignored.
1147
+
*/
1148
+
cr1 = FIELD_PREP(GICV5_ITS_CR1_ITT_RA, GICV5_NO_READ_ALLOC) |
1149
+
FIELD_PREP(GICV5_ITS_CR1_DT_RA, GICV5_NO_READ_ALLOC) |
1150
+
FIELD_PREP(GICV5_ITS_CR1_IC, GICV5_NON_CACHE) |
1151
+
FIELD_PREP(GICV5_ITS_CR1_OC, GICV5_NON_CACHE);
1152
+
its_node->flags |= ITS_FLAGS_NON_COHERENT;
1153
+
} else {
1154
+
cr1 = FIELD_PREP(GICV5_ITS_CR1_ITT_RA, GICV5_READ_ALLOC) |
1155
+
FIELD_PREP(GICV5_ITS_CR1_DT_RA, GICV5_READ_ALLOC) |
1156
+
FIELD_PREP(GICV5_ITS_CR1_IC, GICV5_WB_CACHE) |
1157
+
FIELD_PREP(GICV5_ITS_CR1_OC, GICV5_WB_CACHE) |
1158
+
FIELD_PREP(GICV5_ITS_CR1_SH, GICV5_INNER_SHARE);
1159
+
}
1160
+
1161
+
its_writel_relaxed(its_node, cr1, GICV5_ITS_CR1);
1162
+
1163
+
ret = gicv5_its_init_devtab(its_node);
1164
+
if (ret)
1165
+
goto out_free_node;
1166
+
1167
+
ret = gicv5_its_enable(its_node);
1168
+
if (ret)
1169
+
goto out_free_devtab;
1170
+
1171
+
ret = gicv5_its_init_domain(its_node, parent_domain);
1172
+
if (ret)
1173
+
goto out_disable_its;
1174
+
1175
+
gicv5_its_print_info(its_node);
1176
+
1177
+
return 0;
1178
+
1179
+
out_disable_its:
1180
+
gicv5_its_disable(its_node);
1181
+
out_free_devtab:
1182
+
gicv5_its_deinit_devtab(its_node);
1183
+
out_free_node:
1184
+
kfree(its_node);
1185
+
return ret;
1186
+
}
1187
+
1188
+
static int __init gicv5_its_init(struct device_node *node)
1189
+
{
1190
+
void __iomem *its_base;
1191
+
int ret, idx;
1192
+
1193
+
idx = of_property_match_string(node, "reg-names", "ns-config");
1194
+
if (idx < 0) {
1195
+
pr_err("%pOF: ns-config reg-name not present\n", node);
1196
+
return -ENODEV;
1197
+
}
1198
+
1199
+
its_base = of_io_request_and_map(node, idx, of_node_full_name(node));
1200
+
if (IS_ERR(its_base)) {
1201
+
pr_err("%pOF: unable to map GICv5 ITS_CONFIG_FRAME\n", node);
1202
+
return PTR_ERR(its_base);
1203
+
}
1204
+
1205
+
ret = gicv5_its_init_bases(its_base, of_fwnode_handle(node),
1206
+
gicv5_global_data.lpi_domain);
1207
+
if (ret)
1208
+
goto out_unmap;
1209
+
1210
+
return 0;
1211
+
1212
+
out_unmap:
1213
+
iounmap(its_base);
1214
+
return ret;
1215
+
}
1216
+
1217
+
void __init gicv5_its_of_probe(struct device_node *parent)
1218
+
{
1219
+
struct device_node *np;
1220
+
1221
+
for_each_available_child_of_node(parent, np) {
1222
+
if (!of_device_is_compatible(np, "arm,gic-v5-its"))
1223
+
continue;
1224
+
1225
+
if (gicv5_its_init(np))
1226
+
pr_err("Failed to init ITS %s\n", np->full_name);
1227
+
}
1228
+
}
+284
drivers/irqchip/irq-gic-v5-iwb.c
+284
drivers/irqchip/irq-gic-v5-iwb.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Copyright (C) 2024-2025 ARM Limited, All Rights Reserved.
4
+
*/
5
+
#define pr_fmt(fmt) "GICv5 IWB: " fmt
6
+
7
+
#include <linux/init.h>
8
+
#include <linux/kernel.h>
9
+
#include <linux/msi.h>
10
+
#include <linux/of.h>
11
+
#include <linux/of_address.h>
12
+
#include <linux/of_platform.h>
13
+
14
+
#include <linux/irqchip.h>
15
+
#include <linux/irqchip/arm-gic-v5.h>
16
+
17
+
struct gicv5_iwb_chip_data {
18
+
void __iomem *iwb_base;
19
+
u16 nr_regs;
20
+
};
21
+
22
+
static u32 iwb_readl_relaxed(struct gicv5_iwb_chip_data *iwb_node, const u32 reg_offset)
23
+
{
24
+
return readl_relaxed(iwb_node->iwb_base + reg_offset);
25
+
}
26
+
27
+
static void iwb_writel_relaxed(struct gicv5_iwb_chip_data *iwb_node, const u32 val,
28
+
const u32 reg_offset)
29
+
{
30
+
writel_relaxed(val, iwb_node->iwb_base + reg_offset);
31
+
}
32
+
33
+
static int gicv5_iwb_wait_for_wenabler(struct gicv5_iwb_chip_data *iwb_node)
34
+
{
35
+
return gicv5_wait_for_op_atomic(iwb_node->iwb_base, GICV5_IWB_WENABLE_STATUSR,
36
+
GICV5_IWB_WENABLE_STATUSR_IDLE, NULL);
37
+
}
38
+
39
+
static int __gicv5_iwb_set_wire_enable(struct gicv5_iwb_chip_data *iwb_node,
40
+
u32 iwb_wire, bool enable)
41
+
{
42
+
u32 n = iwb_wire / 32;
43
+
u8 i = iwb_wire % 32;
44
+
u32 val;
45
+
46
+
if (n >= iwb_node->nr_regs) {
47
+
pr_err("IWB_WENABLER<n> is invalid for n=%u\n", n);
48
+
return -EINVAL;
49
+
}
50
+
51
+
/*
52
+
* Enable IWB wire/pin at this point
53
+
* Note: This is not the same as enabling the interrupt
54
+
*/
55
+
val = iwb_readl_relaxed(iwb_node, GICV5_IWB_WENABLER + (4 * n));
56
+
if (enable)
57
+
val |= BIT(i);
58
+
else
59
+
val &= ~BIT(i);
60
+
iwb_writel_relaxed(iwb_node, val, GICV5_IWB_WENABLER + (4 * n));
61
+
62
+
return gicv5_iwb_wait_for_wenabler(iwb_node);
63
+
}
64
+
65
+
static int gicv5_iwb_enable_wire(struct gicv5_iwb_chip_data *iwb_node,
66
+
u32 iwb_wire)
67
+
{
68
+
return __gicv5_iwb_set_wire_enable(iwb_node, iwb_wire, true);
69
+
}
70
+
71
+
static int gicv5_iwb_disable_wire(struct gicv5_iwb_chip_data *iwb_node,
72
+
u32 iwb_wire)
73
+
{
74
+
return __gicv5_iwb_set_wire_enable(iwb_node, iwb_wire, false);
75
+
}
76
+
77
+
static void gicv5_iwb_irq_disable(struct irq_data *d)
78
+
{
79
+
struct gicv5_iwb_chip_data *iwb_node = irq_data_get_irq_chip_data(d);
80
+
81
+
gicv5_iwb_disable_wire(iwb_node, d->hwirq);
82
+
irq_chip_disable_parent(d);
83
+
}
84
+
85
+
static void gicv5_iwb_irq_enable(struct irq_data *d)
86
+
{
87
+
struct gicv5_iwb_chip_data *iwb_node = irq_data_get_irq_chip_data(d);
88
+
89
+
gicv5_iwb_enable_wire(iwb_node, d->hwirq);
90
+
irq_chip_enable_parent(d);
91
+
}
92
+
93
+
static int gicv5_iwb_set_type(struct irq_data *d, unsigned int type)
94
+
{
95
+
struct gicv5_iwb_chip_data *iwb_node = irq_data_get_irq_chip_data(d);
96
+
u32 iwb_wire, n, wtmr;
97
+
u8 i;
98
+
99
+
iwb_wire = d->hwirq;
100
+
i = iwb_wire % 32;
101
+
n = iwb_wire / 32;
102
+
103
+
if (n >= iwb_node->nr_regs) {
104
+
pr_err_once("reg %u out of range\n", n);
105
+
return -EINVAL;
106
+
}
107
+
108
+
wtmr = iwb_readl_relaxed(iwb_node, GICV5_IWB_WTMR + (4 * n));
109
+
110
+
switch (type) {
111
+
case IRQ_TYPE_LEVEL_HIGH:
112
+
case IRQ_TYPE_LEVEL_LOW:
113
+
wtmr |= BIT(i);
114
+
break;
115
+
case IRQ_TYPE_EDGE_RISING:
116
+
case IRQ_TYPE_EDGE_FALLING:
117
+
wtmr &= ~BIT(i);
118
+
break;
119
+
default:
120
+
pr_debug("unexpected wire trigger mode");
121
+
return -EINVAL;
122
+
}
123
+
124
+
iwb_writel_relaxed(iwb_node, wtmr, GICV5_IWB_WTMR + (4 * n));
125
+
126
+
return 0;
127
+
}
128
+
129
+
static void gicv5_iwb_domain_set_desc(msi_alloc_info_t *alloc_info, struct msi_desc *desc)
130
+
{
131
+
alloc_info->desc = desc;
132
+
alloc_info->hwirq = (u32)desc->data.icookie.value;
133
+
}
134
+
135
+
static int gicv5_iwb_irq_domain_translate(struct irq_domain *d, struct irq_fwspec *fwspec,
136
+
irq_hw_number_t *hwirq,
137
+
unsigned int *type)
138
+
{
139
+
if (!is_of_node(fwspec->fwnode))
140
+
return -EINVAL;
141
+
142
+
if (fwspec->param_count < 2)
143
+
return -EINVAL;
144
+
145
+
/*
146
+
* param[0] is be the wire
147
+
* param[1] is the interrupt type
148
+
*/
149
+
*hwirq = fwspec->param[0];
150
+
*type = fwspec->param[1] & IRQ_TYPE_SENSE_MASK;
151
+
152
+
return 0;
153
+
}
154
+
155
+
static void gicv5_iwb_write_msi_msg(struct irq_data *d, struct msi_msg *msg) {}
156
+
157
+
static const struct msi_domain_template iwb_msi_template = {
158
+
.chip = {
159
+
.name = "GICv5-IWB",
160
+
.irq_mask = irq_chip_mask_parent,
161
+
.irq_unmask = irq_chip_unmask_parent,
162
+
.irq_enable = gicv5_iwb_irq_enable,
163
+
.irq_disable = gicv5_iwb_irq_disable,
164
+
.irq_eoi = irq_chip_eoi_parent,
165
+
.irq_set_type = gicv5_iwb_set_type,
166
+
.irq_write_msi_msg = gicv5_iwb_write_msi_msg,
167
+
.irq_set_affinity = irq_chip_set_affinity_parent,
168
+
.irq_get_irqchip_state = irq_chip_get_parent_state,
169
+
.irq_set_irqchip_state = irq_chip_set_parent_state,
170
+
.flags = IRQCHIP_SET_TYPE_MASKED |
171
+
IRQCHIP_SKIP_SET_WAKE |
172
+
IRQCHIP_MASK_ON_SUSPEND,
173
+
},
174
+
175
+
.ops = {
176
+
.set_desc = gicv5_iwb_domain_set_desc,
177
+
.msi_translate = gicv5_iwb_irq_domain_translate,
178
+
},
179
+
180
+
.info = {
181
+
.bus_token = DOMAIN_BUS_WIRED_TO_MSI,
182
+
.flags = MSI_FLAG_USE_DEV_FWNODE,
183
+
},
184
+
185
+
.alloc_info = {
186
+
.flags = MSI_ALLOC_FLAGS_FIXED_MSG_DATA,
187
+
},
188
+
};
189
+
190
+
static bool gicv5_iwb_create_device_domain(struct device *dev, unsigned int size,
191
+
struct gicv5_iwb_chip_data *iwb_node)
192
+
{
193
+
if (WARN_ON_ONCE(!dev->msi.domain))
194
+
return false;
195
+
196
+
return msi_create_device_irq_domain(dev, MSI_DEFAULT_DOMAIN,
197
+
&iwb_msi_template, size,
198
+
NULL, iwb_node);
199
+
}
200
+
201
+
static struct gicv5_iwb_chip_data *
202
+
gicv5_iwb_init_bases(void __iomem *iwb_base, struct platform_device *pdev)
203
+
{
204
+
u32 nr_wires, idr0, cr0;
205
+
unsigned int n;
206
+
int ret;
207
+
208
+
struct gicv5_iwb_chip_data *iwb_node __free(kfree) = kzalloc(sizeof(*iwb_node),
209
+
GFP_KERNEL);
210
+
if (!iwb_node)
211
+
return ERR_PTR(-ENOMEM);
212
+
213
+
iwb_node->iwb_base = iwb_base;
214
+
215
+
idr0 = iwb_readl_relaxed(iwb_node, GICV5_IWB_IDR0);
216
+
nr_wires = (FIELD_GET(GICV5_IWB_IDR0_IW_RANGE, idr0) + 1) * 32;
217
+
218
+
cr0 = iwb_readl_relaxed(iwb_node, GICV5_IWB_CR0);
219
+
if (!FIELD_GET(GICV5_IWB_CR0_IWBEN, cr0)) {
220
+
dev_err(&pdev->dev, "IWB must be enabled in firmware\n");
221
+
return ERR_PTR(-EINVAL);
222
+
}
223
+
224
+
iwb_node->nr_regs = FIELD_GET(GICV5_IWB_IDR0_IW_RANGE, idr0) + 1;
225
+
226
+
for (n = 0; n < iwb_node->nr_regs; n++)
227
+
iwb_writel_relaxed(iwb_node, 0, GICV5_IWB_WENABLER + (sizeof(u32) * n));
228
+
229
+
ret = gicv5_iwb_wait_for_wenabler(iwb_node);
230
+
if (ret)
231
+
return ERR_PTR(ret);
232
+
233
+
if (!gicv5_iwb_create_device_domain(&pdev->dev, nr_wires, iwb_node))
234
+
return ERR_PTR(-ENOMEM);
235
+
236
+
return_ptr(iwb_node);
237
+
}
238
+
239
+
static int gicv5_iwb_device_probe(struct platform_device *pdev)
240
+
{
241
+
struct gicv5_iwb_chip_data *iwb_node;
242
+
void __iomem *iwb_base;
243
+
struct resource *res;
244
+
int ret;
245
+
246
+
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
247
+
if (!res)
248
+
return -EINVAL;
249
+
250
+
iwb_base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
251
+
if (!iwb_base) {
252
+
dev_err(&pdev->dev, "failed to ioremap %pR\n", res);
253
+
return -ENOMEM;
254
+
}
255
+
256
+
iwb_node = gicv5_iwb_init_bases(iwb_base, pdev);
257
+
if (IS_ERR(iwb_node)) {
258
+
ret = PTR_ERR(iwb_node);
259
+
goto out_unmap;
260
+
}
261
+
262
+
return 0;
263
+
264
+
out_unmap:
265
+
iounmap(iwb_base);
266
+
return ret;
267
+
}
268
+
269
+
static const struct of_device_id gicv5_iwb_of_match[] = {
270
+
{ .compatible = "arm,gic-v5-iwb" },
271
+
{ /* END */ }
272
+
};
273
+
MODULE_DEVICE_TABLE(of, gicv5_iwb_of_match);
274
+
275
+
static struct platform_driver gicv5_iwb_platform_driver = {
276
+
.driver = {
277
+
.name = "GICv5 IWB",
278
+
.of_match_table = gicv5_iwb_of_match,
279
+
.suppress_bind_attrs = true,
280
+
},
281
+
.probe = gicv5_iwb_device_probe,
282
+
};
283
+
284
+
module_platform_driver(gicv5_iwb_platform_driver);
+1137
drivers/irqchip/irq-gic-v5.c
+1137
drivers/irqchip/irq-gic-v5.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Copyright (C) 2024-2025 ARM Limited, All Rights Reserved.
4
+
*/
5
+
6
+
#define pr_fmt(fmt) "GICv5: " fmt
7
+
8
+
#include <linux/cpuhotplug.h>
9
+
#include <linux/idr.h>
10
+
#include <linux/irqdomain.h>
11
+
#include <linux/slab.h>
12
+
#include <linux/wordpart.h>
13
+
14
+
#include <linux/irqchip.h>
15
+
#include <linux/irqchip/arm-gic-v5.h>
16
+
#include <linux/irqchip/arm-vgic-info.h>
17
+
18
+
#include <asm/cpufeature.h>
19
+
#include <asm/exception.h>
20
+
21
+
static u8 pri_bits __ro_after_init = 5;
22
+
23
+
#define GICV5_IRQ_PRI_MASK 0x1f
24
+
#define GICV5_IRQ_PRI_MI (GICV5_IRQ_PRI_MASK & GENMASK(4, 5 - pri_bits))
25
+
26
+
#define PPI_NR 128
27
+
28
+
static bool gicv5_cpuif_has_gcie(void)
29
+
{
30
+
return this_cpu_has_cap(ARM64_HAS_GICV5_CPUIF);
31
+
}
32
+
33
+
struct gicv5_chip_data gicv5_global_data __read_mostly;
34
+
35
+
static DEFINE_IDA(lpi_ida);
36
+
static u32 num_lpis __ro_after_init;
37
+
38
+
void __init gicv5_init_lpis(u32 lpis)
39
+
{
40
+
num_lpis = lpis;
41
+
}
42
+
43
+
void __init gicv5_deinit_lpis(void)
44
+
{
45
+
num_lpis = 0;
46
+
}
47
+
48
+
static int alloc_lpi(void)
49
+
{
50
+
if (!num_lpis)
51
+
return -ENOSPC;
52
+
53
+
return ida_alloc_max(&lpi_ida, num_lpis - 1, GFP_KERNEL);
54
+
}
55
+
56
+
static void release_lpi(u32 lpi)
57
+
{
58
+
ida_free(&lpi_ida, lpi);
59
+
}
60
+
61
+
int gicv5_alloc_lpi(void)
62
+
{
63
+
return alloc_lpi();
64
+
}
65
+
66
+
void gicv5_free_lpi(u32 lpi)
67
+
{
68
+
release_lpi(lpi);
69
+
}
70
+
71
+
static void gicv5_ppi_priority_init(void)
72
+
{
73
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR0_EL1);
74
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR1_EL1);
75
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR2_EL1);
76
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR3_EL1);
77
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR4_EL1);
78
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR5_EL1);
79
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR6_EL1);
80
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR7_EL1);
81
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR8_EL1);
82
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR9_EL1);
83
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR10_EL1);
84
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR11_EL1);
85
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR12_EL1);
86
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR13_EL1);
87
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR14_EL1);
88
+
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR15_EL1);
89
+
90
+
/*
91
+
* Context syncronization required to make sure system register writes
92
+
* effects are synchronised.
93
+
*/
94
+
isb();
95
+
}
96
+
97
+
static void gicv5_hwirq_init(irq_hw_number_t hwirq, u8 priority, u8 hwirq_type)
98
+
{
99
+
u64 cdpri, cdaff;
100
+
u16 iaffid;
101
+
int ret;
102
+
103
+
if (hwirq_type == GICV5_HWIRQ_TYPE_LPI || hwirq_type == GICV5_HWIRQ_TYPE_SPI) {
104
+
cdpri = FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, priority) |
105
+
FIELD_PREP(GICV5_GIC_CDPRI_TYPE_MASK, hwirq_type) |
106
+
FIELD_PREP(GICV5_GIC_CDPRI_ID_MASK, hwirq);
107
+
gic_insn(cdpri, CDPRI);
108
+
109
+
ret = gicv5_irs_cpu_to_iaffid(smp_processor_id(), &iaffid);
110
+
111
+
if (WARN_ON_ONCE(ret))
112
+
return;
113
+
114
+
cdaff = FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, iaffid) |
115
+
FIELD_PREP(GICV5_GIC_CDAFF_TYPE_MASK, hwirq_type) |
116
+
FIELD_PREP(GICV5_GIC_CDAFF_ID_MASK, hwirq);
117
+
gic_insn(cdaff, CDAFF);
118
+
}
119
+
}
120
+
121
+
static void gicv5_ppi_irq_mask(struct irq_data *d)
122
+
{
123
+
u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64);
124
+
125
+
if (d->hwirq < 64)
126
+
sysreg_clear_set_s(SYS_ICC_PPI_ENABLER0_EL1, hwirq_id_bit, 0);
127
+
else
128
+
sysreg_clear_set_s(SYS_ICC_PPI_ENABLER1_EL1, hwirq_id_bit, 0);
129
+
130
+
/*
131
+
* We must ensure that the disable takes effect immediately to
132
+
* guarantee that the lazy-disabled IRQ mechanism works.
133
+
* A context synchronization event is required to guarantee it.
134
+
* Reference: I_ZLTKB/R_YRGMH GICv5 specification - section 2.9.1.
135
+
*/
136
+
isb();
137
+
}
138
+
139
+
static void gicv5_iri_irq_mask(struct irq_data *d, u8 hwirq_type)
140
+
{
141
+
u64 cddis;
142
+
143
+
cddis = FIELD_PREP(GICV5_GIC_CDDIS_ID_MASK, d->hwirq) |
144
+
FIELD_PREP(GICV5_GIC_CDDIS_TYPE_MASK, hwirq_type);
145
+
146
+
gic_insn(cddis, CDDIS);
147
+
/*
148
+
* We must make sure that GIC CDDIS write effects are propagated
149
+
* immediately to make sure the disable takes effect to guarantee
150
+
* that the lazy-disabled IRQ mechanism works.
151
+
* Rule R_XCLJC states that the effects of a GIC system instruction
152
+
* complete in finite time.
153
+
* The GSB ensures completion of the GIC instruction and prevents
154
+
* loads, stores and GIC instructions from executing part of their
155
+
* functionality before the GSB SYS.
156
+
*/
157
+
gsb_sys();
158
+
}
159
+
160
+
static void gicv5_spi_irq_mask(struct irq_data *d)
161
+
{
162
+
gicv5_iri_irq_mask(d, GICV5_HWIRQ_TYPE_SPI);
163
+
}
164
+
165
+
static void gicv5_lpi_irq_mask(struct irq_data *d)
166
+
{
167
+
gicv5_iri_irq_mask(d, GICV5_HWIRQ_TYPE_LPI);
168
+
}
169
+
170
+
static void gicv5_ppi_irq_unmask(struct irq_data *d)
171
+
{
172
+
u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64);
173
+
174
+
if (d->hwirq < 64)
175
+
sysreg_clear_set_s(SYS_ICC_PPI_ENABLER0_EL1, 0, hwirq_id_bit);
176
+
else
177
+
sysreg_clear_set_s(SYS_ICC_PPI_ENABLER1_EL1, 0, hwirq_id_bit);
178
+
/*
179
+
* We must ensure that the enable takes effect in finite time - a
180
+
* context synchronization event is required to guarantee it, we
181
+
* can not take for granted that would happen (eg a core going straight
182
+
* into idle after enabling a PPI).
183
+
* Reference: I_ZLTKB/R_YRGMH GICv5 specification - section 2.9.1.
184
+
*/
185
+
isb();
186
+
}
187
+
188
+
static void gicv5_iri_irq_unmask(struct irq_data *d, u8 hwirq_type)
189
+
{
190
+
u64 cden;
191
+
192
+
cden = FIELD_PREP(GICV5_GIC_CDEN_ID_MASK, d->hwirq) |
193
+
FIELD_PREP(GICV5_GIC_CDEN_TYPE_MASK, hwirq_type);
194
+
/*
195
+
* Rule R_XCLJC states that the effects of a GIC system instruction
196
+
* complete in finite time and that's the only requirement when
197
+
* unmasking an SPI/LPI IRQ.
198
+
*/
199
+
gic_insn(cden, CDEN);
200
+
}
201
+
202
+
static void gicv5_spi_irq_unmask(struct irq_data *d)
203
+
{
204
+
gicv5_iri_irq_unmask(d, GICV5_HWIRQ_TYPE_SPI);
205
+
}
206
+
207
+
static void gicv5_lpi_irq_unmask(struct irq_data *d)
208
+
{
209
+
gicv5_iri_irq_unmask(d, GICV5_HWIRQ_TYPE_LPI);
210
+
}
211
+
212
+
static void gicv5_hwirq_eoi(u32 hwirq_id, u8 hwirq_type)
213
+
{
214
+
u64 cddi;
215
+
216
+
cddi = FIELD_PREP(GICV5_GIC_CDDI_ID_MASK, hwirq_id) |
217
+
FIELD_PREP(GICV5_GIC_CDDI_TYPE_MASK, hwirq_type);
218
+
219
+
gic_insn(cddi, CDDI);
220
+
221
+
gic_insn(0, CDEOI);
222
+
}
223
+
224
+
static void gicv5_ppi_irq_eoi(struct irq_data *d)
225
+
{
226
+
/* Skip deactivate for forwarded PPI interrupts */
227
+
if (irqd_is_forwarded_to_vcpu(d)) {
228
+
gic_insn(0, CDEOI);
229
+
return;
230
+
}
231
+
232
+
gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_PPI);
233
+
}
234
+
235
+
static void gicv5_spi_irq_eoi(struct irq_data *d)
236
+
{
237
+
gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_SPI);
238
+
}
239
+
240
+
static void gicv5_lpi_irq_eoi(struct irq_data *d)
241
+
{
242
+
gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_LPI);
243
+
}
244
+
245
+
static int gicv5_iri_irq_set_affinity(struct irq_data *d,
246
+
const struct cpumask *mask_val,
247
+
bool force, u8 hwirq_type)
248
+
{
249
+
int ret, cpuid;
250
+
u16 iaffid;
251
+
u64 cdaff;
252
+
253
+
if (force)
254
+
cpuid = cpumask_first(mask_val);
255
+
else
256
+
cpuid = cpumask_any_and(mask_val, cpu_online_mask);
257
+
258
+
ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid);
259
+
if (ret)
260
+
return ret;
261
+
262
+
cdaff = FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, iaffid) |
263
+
FIELD_PREP(GICV5_GIC_CDAFF_TYPE_MASK, hwirq_type) |
264
+
FIELD_PREP(GICV5_GIC_CDAFF_ID_MASK, d->hwirq);
265
+
gic_insn(cdaff, CDAFF);
266
+
267
+
irq_data_update_effective_affinity(d, cpumask_of(cpuid));
268
+
269
+
return IRQ_SET_MASK_OK_DONE;
270
+
}
271
+
272
+
static int gicv5_spi_irq_set_affinity(struct irq_data *d,
273
+
const struct cpumask *mask_val,
274
+
bool force)
275
+
{
276
+
return gicv5_iri_irq_set_affinity(d, mask_val, force,
277
+
GICV5_HWIRQ_TYPE_SPI);
278
+
}
279
+
280
+
static int gicv5_lpi_irq_set_affinity(struct irq_data *d,
281
+
const struct cpumask *mask_val,
282
+
bool force)
283
+
{
284
+
return gicv5_iri_irq_set_affinity(d, mask_val, force,
285
+
GICV5_HWIRQ_TYPE_LPI);
286
+
}
287
+
288
+
enum ppi_reg {
289
+
PPI_PENDING,
290
+
PPI_ACTIVE,
291
+
PPI_HM
292
+
};
293
+
294
+
static __always_inline u64 read_ppi_sysreg_s(unsigned int irq,
295
+
const enum ppi_reg which)
296
+
{
297
+
switch (which) {
298
+
case PPI_PENDING:
299
+
return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_SPENDR0_EL1) :
300
+
read_sysreg_s(SYS_ICC_PPI_SPENDR1_EL1);
301
+
case PPI_ACTIVE:
302
+
return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_SACTIVER0_EL1) :
303
+
read_sysreg_s(SYS_ICC_PPI_SACTIVER1_EL1);
304
+
case PPI_HM:
305
+
return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_HMR0_EL1) :
306
+
read_sysreg_s(SYS_ICC_PPI_HMR1_EL1);
307
+
default:
308
+
BUILD_BUG_ON(1);
309
+
}
310
+
}
311
+
312
+
static __always_inline void write_ppi_sysreg_s(unsigned int irq, bool set,
313
+
const enum ppi_reg which)
314
+
{
315
+
u64 bit = BIT_ULL(irq % 64);
316
+
317
+
switch (which) {
318
+
case PPI_PENDING:
319
+
if (set) {
320
+
if (irq < 64)
321
+
write_sysreg_s(bit, SYS_ICC_PPI_SPENDR0_EL1);
322
+
else
323
+
write_sysreg_s(bit, SYS_ICC_PPI_SPENDR1_EL1);
324
+
} else {
325
+
if (irq < 64)
326
+
write_sysreg_s(bit, SYS_ICC_PPI_CPENDR0_EL1);
327
+
else
328
+
write_sysreg_s(bit, SYS_ICC_PPI_CPENDR1_EL1);
329
+
}
330
+
return;
331
+
case PPI_ACTIVE:
332
+
if (set) {
333
+
if (irq < 64)
334
+
write_sysreg_s(bit, SYS_ICC_PPI_SACTIVER0_EL1);
335
+
else
336
+
write_sysreg_s(bit, SYS_ICC_PPI_SACTIVER1_EL1);
337
+
} else {
338
+
if (irq < 64)
339
+
write_sysreg_s(bit, SYS_ICC_PPI_CACTIVER0_EL1);
340
+
else
341
+
write_sysreg_s(bit, SYS_ICC_PPI_CACTIVER1_EL1);
342
+
}
343
+
return;
344
+
default:
345
+
BUILD_BUG_ON(1);
346
+
}
347
+
}
348
+
349
+
static int gicv5_ppi_irq_get_irqchip_state(struct irq_data *d,
350
+
enum irqchip_irq_state which,
351
+
bool *state)
352
+
{
353
+
u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64);
354
+
355
+
switch (which) {
356
+
case IRQCHIP_STATE_PENDING:
357
+
*state = !!(read_ppi_sysreg_s(d->hwirq, PPI_PENDING) & hwirq_id_bit);
358
+
return 0;
359
+
case IRQCHIP_STATE_ACTIVE:
360
+
*state = !!(read_ppi_sysreg_s(d->hwirq, PPI_ACTIVE) & hwirq_id_bit);
361
+
return 0;
362
+
default:
363
+
pr_debug("Unexpected PPI irqchip state\n");
364
+
return -EINVAL;
365
+
}
366
+
}
367
+
368
+
static int gicv5_iri_irq_get_irqchip_state(struct irq_data *d,
369
+
enum irqchip_irq_state which,
370
+
bool *state, u8 hwirq_type)
371
+
{
372
+
u64 icsr, cdrcfg;
373
+
374
+
cdrcfg = d->hwirq | FIELD_PREP(GICV5_GIC_CDRCFG_TYPE_MASK, hwirq_type);
375
+
376
+
gic_insn(cdrcfg, CDRCFG);
377
+
isb();
378
+
icsr = read_sysreg_s(SYS_ICC_ICSR_EL1);
379
+
380
+
if (FIELD_GET(ICC_ICSR_EL1_F, icsr)) {
381
+
pr_err("ICSR_EL1 is invalid\n");
382
+
return -EINVAL;
383
+
}
384
+
385
+
switch (which) {
386
+
case IRQCHIP_STATE_PENDING:
387
+
*state = !!(FIELD_GET(ICC_ICSR_EL1_Pending, icsr));
388
+
return 0;
389
+
390
+
case IRQCHIP_STATE_ACTIVE:
391
+
*state = !!(FIELD_GET(ICC_ICSR_EL1_Active, icsr));
392
+
return 0;
393
+
394
+
default:
395
+
pr_debug("Unexpected irqchip_irq_state\n");
396
+
return -EINVAL;
397
+
}
398
+
}
399
+
400
+
static int gicv5_spi_irq_get_irqchip_state(struct irq_data *d,
401
+
enum irqchip_irq_state which,
402
+
bool *state)
403
+
{
404
+
return gicv5_iri_irq_get_irqchip_state(d, which, state,
405
+
GICV5_HWIRQ_TYPE_SPI);
406
+
}
407
+
408
+
static int gicv5_lpi_irq_get_irqchip_state(struct irq_data *d,
409
+
enum irqchip_irq_state which,
410
+
bool *state)
411
+
{
412
+
return gicv5_iri_irq_get_irqchip_state(d, which, state,
413
+
GICV5_HWIRQ_TYPE_LPI);
414
+
}
415
+
416
+
static int gicv5_ppi_irq_set_irqchip_state(struct irq_data *d,
417
+
enum irqchip_irq_state which,
418
+
bool state)
419
+
{
420
+
switch (which) {
421
+
case IRQCHIP_STATE_PENDING:
422
+
write_ppi_sysreg_s(d->hwirq, state, PPI_PENDING);
423
+
return 0;
424
+
case IRQCHIP_STATE_ACTIVE:
425
+
write_ppi_sysreg_s(d->hwirq, state, PPI_ACTIVE);
426
+
return 0;
427
+
default:
428
+
pr_debug("Unexpected PPI irqchip state\n");
429
+
return -EINVAL;
430
+
}
431
+
}
432
+
433
+
static void gicv5_iri_irq_write_pending_state(struct irq_data *d, bool state,
434
+
u8 hwirq_type)
435
+
{
436
+
u64 cdpend;
437
+
438
+
cdpend = FIELD_PREP(GICV5_GIC_CDPEND_TYPE_MASK, hwirq_type) |
439
+
FIELD_PREP(GICV5_GIC_CDPEND_ID_MASK, d->hwirq) |
440
+
FIELD_PREP(GICV5_GIC_CDPEND_PENDING_MASK, state);
441
+
442
+
gic_insn(cdpend, CDPEND);
443
+
}
444
+
445
+
static void gicv5_spi_irq_write_pending_state(struct irq_data *d, bool state)
446
+
{
447
+
gicv5_iri_irq_write_pending_state(d, state, GICV5_HWIRQ_TYPE_SPI);
448
+
}
449
+
450
+
static void gicv5_lpi_irq_write_pending_state(struct irq_data *d, bool state)
451
+
{
452
+
gicv5_iri_irq_write_pending_state(d, state, GICV5_HWIRQ_TYPE_LPI);
453
+
}
454
+
455
+
static int gicv5_spi_irq_set_irqchip_state(struct irq_data *d,
456
+
enum irqchip_irq_state which,
457
+
bool state)
458
+
{
459
+
switch (which) {
460
+
case IRQCHIP_STATE_PENDING:
461
+
gicv5_spi_irq_write_pending_state(d, state);
462
+
break;
463
+
default:
464
+
pr_debug("Unexpected irqchip_irq_state\n");
465
+
return -EINVAL;
466
+
}
467
+
468
+
return 0;
469
+
}
470
+
471
+
static int gicv5_lpi_irq_set_irqchip_state(struct irq_data *d,
472
+
enum irqchip_irq_state which,
473
+
bool state)
474
+
{
475
+
switch (which) {
476
+
case IRQCHIP_STATE_PENDING:
477
+
gicv5_lpi_irq_write_pending_state(d, state);
478
+
break;
479
+
480
+
default:
481
+
pr_debug("Unexpected irqchip_irq_state\n");
482
+
return -EINVAL;
483
+
}
484
+
485
+
return 0;
486
+
}
487
+
488
+
static int gicv5_spi_irq_retrigger(struct irq_data *data)
489
+
{
490
+
return !gicv5_spi_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING,
491
+
true);
492
+
}
493
+
494
+
static int gicv5_lpi_irq_retrigger(struct irq_data *data)
495
+
{
496
+
return !gicv5_lpi_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING,
497
+
true);
498
+
}
499
+
500
+
static void gicv5_ipi_send_single(struct irq_data *d, unsigned int cpu)
501
+
{
502
+
/* Mark the LPI pending */
503
+
irq_chip_retrigger_hierarchy(d);
504
+
}
505
+
506
+
static bool gicv5_ppi_irq_is_level(irq_hw_number_t hwirq)
507
+
{
508
+
u64 bit = BIT_ULL(hwirq % 64);
509
+
510
+
return !!(read_ppi_sysreg_s(hwirq, PPI_HM) & bit);
511
+
}
512
+
513
+
static int gicv5_ppi_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
514
+
{
515
+
if (vcpu)
516
+
irqd_set_forwarded_to_vcpu(d);
517
+
else
518
+
irqd_clr_forwarded_to_vcpu(d);
519
+
520
+
return 0;
521
+
}
522
+
523
+
static const struct irq_chip gicv5_ppi_irq_chip = {
524
+
.name = "GICv5-PPI",
525
+
.irq_mask = gicv5_ppi_irq_mask,
526
+
.irq_unmask = gicv5_ppi_irq_unmask,
527
+
.irq_eoi = gicv5_ppi_irq_eoi,
528
+
.irq_get_irqchip_state = gicv5_ppi_irq_get_irqchip_state,
529
+
.irq_set_irqchip_state = gicv5_ppi_irq_set_irqchip_state,
530
+
.irq_set_vcpu_affinity = gicv5_ppi_irq_set_vcpu_affinity,
531
+
.flags = IRQCHIP_SKIP_SET_WAKE |
532
+
IRQCHIP_MASK_ON_SUSPEND,
533
+
};
534
+
535
+
static const struct irq_chip gicv5_spi_irq_chip = {
536
+
.name = "GICv5-SPI",
537
+
.irq_mask = gicv5_spi_irq_mask,
538
+
.irq_unmask = gicv5_spi_irq_unmask,
539
+
.irq_eoi = gicv5_spi_irq_eoi,
540
+
.irq_set_type = gicv5_spi_irq_set_type,
541
+
.irq_set_affinity = gicv5_spi_irq_set_affinity,
542
+
.irq_retrigger = gicv5_spi_irq_retrigger,
543
+
.irq_get_irqchip_state = gicv5_spi_irq_get_irqchip_state,
544
+
.irq_set_irqchip_state = gicv5_spi_irq_set_irqchip_state,
545
+
.flags = IRQCHIP_SET_TYPE_MASKED |
546
+
IRQCHIP_SKIP_SET_WAKE |
547
+
IRQCHIP_MASK_ON_SUSPEND,
548
+
};
549
+
550
+
static const struct irq_chip gicv5_lpi_irq_chip = {
551
+
.name = "GICv5-LPI",
552
+
.irq_mask = gicv5_lpi_irq_mask,
553
+
.irq_unmask = gicv5_lpi_irq_unmask,
554
+
.irq_eoi = gicv5_lpi_irq_eoi,
555
+
.irq_set_affinity = gicv5_lpi_irq_set_affinity,
556
+
.irq_retrigger = gicv5_lpi_irq_retrigger,
557
+
.irq_get_irqchip_state = gicv5_lpi_irq_get_irqchip_state,
558
+
.irq_set_irqchip_state = gicv5_lpi_irq_set_irqchip_state,
559
+
.flags = IRQCHIP_SKIP_SET_WAKE |
560
+
IRQCHIP_MASK_ON_SUSPEND,
561
+
};
562
+
563
+
static const struct irq_chip gicv5_ipi_irq_chip = {
564
+
.name = "GICv5-IPI",
565
+
.irq_mask = irq_chip_mask_parent,
566
+
.irq_unmask = irq_chip_unmask_parent,
567
+
.irq_eoi = irq_chip_eoi_parent,
568
+
.irq_set_affinity = irq_chip_set_affinity_parent,
569
+
.irq_get_irqchip_state = irq_chip_get_parent_state,
570
+
.irq_set_irqchip_state = irq_chip_set_parent_state,
571
+
.ipi_send_single = gicv5_ipi_send_single,
572
+
.flags = IRQCHIP_SKIP_SET_WAKE |
573
+
IRQCHIP_MASK_ON_SUSPEND,
574
+
};
575
+
576
+
static __always_inline int gicv5_irq_domain_translate(struct irq_domain *d,
577
+
struct irq_fwspec *fwspec,
578
+
irq_hw_number_t *hwirq,
579
+
unsigned int *type,
580
+
const u8 hwirq_type)
581
+
{
582
+
if (!is_of_node(fwspec->fwnode))
583
+
return -EINVAL;
584
+
585
+
if (fwspec->param_count < 3)
586
+
return -EINVAL;
587
+
588
+
if (fwspec->param[0] != hwirq_type)
589
+
return -EINVAL;
590
+
591
+
*hwirq = fwspec->param[1];
592
+
593
+
switch (hwirq_type) {
594
+
case GICV5_HWIRQ_TYPE_PPI:
595
+
/*
596
+
* Handling mode is hardcoded for PPIs, set the type using
597
+
* HW reported value.
598
+
*/
599
+
*type = gicv5_ppi_irq_is_level(*hwirq) ? IRQ_TYPE_LEVEL_LOW :
600
+
IRQ_TYPE_EDGE_RISING;
601
+
break;
602
+
case GICV5_HWIRQ_TYPE_SPI:
603
+
*type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
604
+
break;
605
+
default:
606
+
BUILD_BUG_ON(1);
607
+
}
608
+
609
+
return 0;
610
+
}
611
+
612
+
static int gicv5_irq_ppi_domain_translate(struct irq_domain *d,
613
+
struct irq_fwspec *fwspec,
614
+
irq_hw_number_t *hwirq,
615
+
unsigned int *type)
616
+
{
617
+
return gicv5_irq_domain_translate(d, fwspec, hwirq, type,
618
+
GICV5_HWIRQ_TYPE_PPI);
619
+
}
620
+
621
+
static int gicv5_irq_ppi_domain_alloc(struct irq_domain *domain, unsigned int virq,
622
+
unsigned int nr_irqs, void *arg)
623
+
{
624
+
unsigned int type = IRQ_TYPE_NONE;
625
+
struct irq_fwspec *fwspec = arg;
626
+
irq_hw_number_t hwirq;
627
+
int ret;
628
+
629
+
if (WARN_ON_ONCE(nr_irqs != 1))
630
+
return -EINVAL;
631
+
632
+
ret = gicv5_irq_ppi_domain_translate(domain, fwspec, &hwirq, &type);
633
+
if (ret)
634
+
return ret;
635
+
636
+
if (type & IRQ_TYPE_LEVEL_MASK)
637
+
irq_set_status_flags(virq, IRQ_LEVEL);
638
+
639
+
irq_set_percpu_devid(virq);
640
+
irq_domain_set_info(domain, virq, hwirq, &gicv5_ppi_irq_chip, NULL,
641
+
handle_percpu_devid_irq, NULL, NULL);
642
+
643
+
return 0;
644
+
}
645
+
646
+
static void gicv5_irq_domain_free(struct irq_domain *domain, unsigned int virq,
647
+
unsigned int nr_irqs)
648
+
{
649
+
struct irq_data *d;
650
+
651
+
if (WARN_ON_ONCE(nr_irqs != 1))
652
+
return;
653
+
654
+
d = irq_domain_get_irq_data(domain, virq);
655
+
656
+
irq_set_handler(virq, NULL);
657
+
irq_domain_reset_irq_data(d);
658
+
}
659
+
660
+
static int gicv5_irq_ppi_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec,
661
+
enum irq_domain_bus_token bus_token)
662
+
{
663
+
if (fwspec->fwnode != d->fwnode)
664
+
return 0;
665
+
666
+
if (fwspec->param[0] != GICV5_HWIRQ_TYPE_PPI)
667
+
return 0;
668
+
669
+
return (d == gicv5_global_data.ppi_domain);
670
+
}
671
+
672
+
static const struct irq_domain_ops gicv5_irq_ppi_domain_ops = {
673
+
.translate = gicv5_irq_ppi_domain_translate,
674
+
.alloc = gicv5_irq_ppi_domain_alloc,
675
+
.free = gicv5_irq_domain_free,
676
+
.select = gicv5_irq_ppi_domain_select
677
+
};
678
+
679
+
static int gicv5_irq_spi_domain_translate(struct irq_domain *d,
680
+
struct irq_fwspec *fwspec,
681
+
irq_hw_number_t *hwirq,
682
+
unsigned int *type)
683
+
{
684
+
return gicv5_irq_domain_translate(d, fwspec, hwirq, type,
685
+
GICV5_HWIRQ_TYPE_SPI);
686
+
}
687
+
688
+
static int gicv5_irq_spi_domain_alloc(struct irq_domain *domain, unsigned int virq,
689
+
unsigned int nr_irqs, void *arg)
690
+
{
691
+
struct gicv5_irs_chip_data *chip_data;
692
+
unsigned int type = IRQ_TYPE_NONE;
693
+
struct irq_fwspec *fwspec = arg;
694
+
struct irq_data *irqd;
695
+
irq_hw_number_t hwirq;
696
+
int ret;
697
+
698
+
if (WARN_ON_ONCE(nr_irqs != 1))
699
+
return -EINVAL;
700
+
701
+
ret = gicv5_irq_spi_domain_translate(domain, fwspec, &hwirq, &type);
702
+
if (ret)
703
+
return ret;
704
+
705
+
irqd = irq_desc_get_irq_data(irq_to_desc(virq));
706
+
chip_data = gicv5_irs_lookup_by_spi_id(hwirq);
707
+
708
+
irq_domain_set_info(domain, virq, hwirq, &gicv5_spi_irq_chip, chip_data,
709
+
handle_fasteoi_irq, NULL, NULL);
710
+
irq_set_probe(virq);
711
+
irqd_set_single_target(irqd);
712
+
713
+
gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_SPI);
714
+
715
+
return 0;
716
+
}
717
+
718
+
static int gicv5_irq_spi_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec,
719
+
enum irq_domain_bus_token bus_token)
720
+
{
721
+
if (fwspec->fwnode != d->fwnode)
722
+
return 0;
723
+
724
+
if (fwspec->param[0] != GICV5_HWIRQ_TYPE_SPI)
725
+
return 0;
726
+
727
+
return (d == gicv5_global_data.spi_domain);
728
+
}
729
+
730
+
static const struct irq_domain_ops gicv5_irq_spi_domain_ops = {
731
+
.translate = gicv5_irq_spi_domain_translate,
732
+
.alloc = gicv5_irq_spi_domain_alloc,
733
+
.free = gicv5_irq_domain_free,
734
+
.select = gicv5_irq_spi_domain_select
735
+
};
736
+
737
+
static void gicv5_lpi_config_reset(struct irq_data *d)
738
+
{
739
+
u64 cdhm;
740
+
741
+
/*
742
+
* Reset LPIs handling mode to edge by default and clear pending
743
+
* state to make sure we start the LPI with a clean state from
744
+
* previous incarnations.
745
+
*/
746
+
cdhm = FIELD_PREP(GICV5_GIC_CDHM_HM_MASK, 0) |
747
+
FIELD_PREP(GICV5_GIC_CDHM_TYPE_MASK, GICV5_HWIRQ_TYPE_LPI) |
748
+
FIELD_PREP(GICV5_GIC_CDHM_ID_MASK, d->hwirq);
749
+
gic_insn(cdhm, CDHM);
750
+
751
+
gicv5_lpi_irq_write_pending_state(d, false);
752
+
}
753
+
754
+
static int gicv5_irq_lpi_domain_alloc(struct irq_domain *domain, unsigned int virq,
755
+
unsigned int nr_irqs, void *arg)
756
+
{
757
+
irq_hw_number_t hwirq;
758
+
struct irq_data *irqd;
759
+
u32 *lpi = arg;
760
+
int ret;
761
+
762
+
if (WARN_ON_ONCE(nr_irqs != 1))
763
+
return -EINVAL;
764
+
765
+
hwirq = *lpi;
766
+
767
+
irqd = irq_domain_get_irq_data(domain, virq);
768
+
769
+
irq_domain_set_info(domain, virq, hwirq, &gicv5_lpi_irq_chip, NULL,
770
+
handle_fasteoi_irq, NULL, NULL);
771
+
irqd_set_single_target(irqd);
772
+
773
+
ret = gicv5_irs_iste_alloc(hwirq);
774
+
if (ret < 0)
775
+
return ret;
776
+
777
+
gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_LPI);
778
+
gicv5_lpi_config_reset(irqd);
779
+
780
+
return 0;
781
+
}
782
+
783
+
static const struct irq_domain_ops gicv5_irq_lpi_domain_ops = {
784
+
.alloc = gicv5_irq_lpi_domain_alloc,
785
+
.free = gicv5_irq_domain_free,
786
+
};
787
+
788
+
void __init gicv5_init_lpi_domain(void)
789
+
{
790
+
struct irq_domain *d;
791
+
792
+
d = irq_domain_create_tree(NULL, &gicv5_irq_lpi_domain_ops, NULL);
793
+
gicv5_global_data.lpi_domain = d;
794
+
}
795
+
796
+
void __init gicv5_free_lpi_domain(void)
797
+
{
798
+
irq_domain_remove(gicv5_global_data.lpi_domain);
799
+
gicv5_global_data.lpi_domain = NULL;
800
+
}
801
+
802
+
static int gicv5_irq_ipi_domain_alloc(struct irq_domain *domain, unsigned int virq,
803
+
unsigned int nr_irqs, void *arg)
804
+
{
805
+
struct irq_data *irqd;
806
+
int ret, i;
807
+
u32 lpi;
808
+
809
+
for (i = 0; i < nr_irqs; i++) {
810
+
ret = gicv5_alloc_lpi();
811
+
if (ret < 0)
812
+
return ret;
813
+
814
+
lpi = ret;
815
+
816
+
ret = irq_domain_alloc_irqs_parent(domain, virq + i, 1, &lpi);
817
+
if (ret) {
818
+
gicv5_free_lpi(lpi);
819
+
return ret;
820
+
}
821
+
822
+
irqd = irq_domain_get_irq_data(domain, virq + i);
823
+
824
+
irq_domain_set_hwirq_and_chip(domain, virq + i, i,
825
+
&gicv5_ipi_irq_chip, NULL);
826
+
827
+
irqd_set_single_target(irqd);
828
+
829
+
irq_set_handler(virq + i, handle_percpu_irq);
830
+
}
831
+
832
+
return 0;
833
+
}
834
+
835
+
static void gicv5_irq_ipi_domain_free(struct irq_domain *domain, unsigned int virq,
836
+
unsigned int nr_irqs)
837
+
{
838
+
struct irq_data *d;
839
+
unsigned int i;
840
+
841
+
for (i = 0; i < nr_irqs; i++) {
842
+
d = irq_domain_get_irq_data(domain, virq + i);
843
+
844
+
if (!d)
845
+
return;
846
+
847
+
gicv5_free_lpi(d->parent_data->hwirq);
848
+
849
+
irq_set_handler(virq + i, NULL);
850
+
irq_domain_reset_irq_data(d);
851
+
irq_domain_free_irqs_parent(domain, virq + i, 1);
852
+
}
853
+
}
854
+
855
+
static const struct irq_domain_ops gicv5_irq_ipi_domain_ops = {
856
+
.alloc = gicv5_irq_ipi_domain_alloc,
857
+
.free = gicv5_irq_ipi_domain_free,
858
+
};
859
+
860
+
static void handle_irq_per_domain(u32 hwirq)
861
+
{
862
+
u8 hwirq_type = FIELD_GET(GICV5_HWIRQ_TYPE, hwirq);
863
+
u32 hwirq_id = FIELD_GET(GICV5_HWIRQ_ID, hwirq);
864
+
struct irq_domain *domain;
865
+
866
+
switch (hwirq_type) {
867
+
case GICV5_HWIRQ_TYPE_PPI:
868
+
domain = gicv5_global_data.ppi_domain;
869
+
break;
870
+
case GICV5_HWIRQ_TYPE_SPI:
871
+
domain = gicv5_global_data.spi_domain;
872
+
break;
873
+
case GICV5_HWIRQ_TYPE_LPI:
874
+
domain = gicv5_global_data.lpi_domain;
875
+
break;
876
+
default:
877
+
pr_err_once("Unknown IRQ type, bail out\n");
878
+
return;
879
+
}
880
+
881
+
if (generic_handle_domain_irq(domain, hwirq_id)) {
882
+
pr_err_once("Could not handle, hwirq = 0x%x", hwirq_id);
883
+
gicv5_hwirq_eoi(hwirq_id, hwirq_type);
884
+
}
885
+
}
886
+
887
+
static void __exception_irq_entry gicv5_handle_irq(struct pt_regs *regs)
888
+
{
889
+
bool valid;
890
+
u32 hwirq;
891
+
u64 ia;
892
+
893
+
ia = gicr_insn(CDIA);
894
+
valid = GICV5_GICR_CDIA_VALID(ia);
895
+
896
+
if (!valid)
897
+
return;
898
+
899
+
/*
900
+
* Ensure that the CDIA instruction effects (ie IRQ activation) are
901
+
* completed before handling the interrupt.
902
+
*/
903
+
gsb_ack();
904
+
905
+
/*
906
+
* Ensure instruction ordering between an acknowledgment and subsequent
907
+
* instructions in the IRQ handler using an ISB.
908
+
*/
909
+
isb();
910
+
911
+
hwirq = FIELD_GET(GICV5_HWIRQ_INTID, ia);
912
+
913
+
handle_irq_per_domain(hwirq);
914
+
}
915
+
916
+
static void gicv5_cpu_disable_interrupts(void)
917
+
{
918
+
u64 cr0;
919
+
920
+
cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 0);
921
+
write_sysreg_s(cr0, SYS_ICC_CR0_EL1);
922
+
}
923
+
924
+
static void gicv5_cpu_enable_interrupts(void)
925
+
{
926
+
u64 cr0, pcr;
927
+
928
+
write_sysreg_s(0, SYS_ICC_PPI_ENABLER0_EL1);
929
+
write_sysreg_s(0, SYS_ICC_PPI_ENABLER1_EL1);
930
+
931
+
gicv5_ppi_priority_init();
932
+
933
+
pcr = FIELD_PREP(ICC_PCR_EL1_PRIORITY, GICV5_IRQ_PRI_MI);
934
+
write_sysreg_s(pcr, SYS_ICC_PCR_EL1);
935
+
936
+
cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 1);
937
+
write_sysreg_s(cr0, SYS_ICC_CR0_EL1);
938
+
}
939
+
940
+
static int base_ipi_virq;
941
+
942
+
static int gicv5_starting_cpu(unsigned int cpu)
943
+
{
944
+
if (WARN(!gicv5_cpuif_has_gcie(),
945
+
"GICv5 system components present but CPU does not have FEAT_GCIE"))
946
+
return -ENODEV;
947
+
948
+
gicv5_cpu_enable_interrupts();
949
+
950
+
return gicv5_irs_register_cpu(cpu);
951
+
}
952
+
953
+
static void __init gicv5_smp_init(void)
954
+
{
955
+
unsigned int num_ipis = GICV5_IPIS_PER_CPU * nr_cpu_ids;
956
+
957
+
cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
958
+
"irqchip/arm/gicv5:starting",
959
+
gicv5_starting_cpu, NULL);
960
+
961
+
base_ipi_virq = irq_domain_alloc_irqs(gicv5_global_data.ipi_domain,
962
+
num_ipis, NUMA_NO_NODE, NULL);
963
+
if (WARN(base_ipi_virq <= 0, "IPI IRQ allocation was not successful"))
964
+
return;
965
+
966
+
set_smp_ipi_range_percpu(base_ipi_virq, GICV5_IPIS_PER_CPU, nr_cpu_ids);
967
+
}
968
+
969
+
static void __init gicv5_free_domains(void)
970
+
{
971
+
if (gicv5_global_data.ppi_domain)
972
+
irq_domain_remove(gicv5_global_data.ppi_domain);
973
+
if (gicv5_global_data.spi_domain)
974
+
irq_domain_remove(gicv5_global_data.spi_domain);
975
+
if (gicv5_global_data.ipi_domain)
976
+
irq_domain_remove(gicv5_global_data.ipi_domain);
977
+
978
+
gicv5_global_data.ppi_domain = NULL;
979
+
gicv5_global_data.spi_domain = NULL;
980
+
gicv5_global_data.ipi_domain = NULL;
981
+
}
982
+
983
+
static int __init gicv5_init_domains(struct fwnode_handle *handle)
984
+
{
985
+
u32 spi_count = gicv5_global_data.global_spi_count;
986
+
struct irq_domain *d;
987
+
988
+
d = irq_domain_create_linear(handle, PPI_NR, &gicv5_irq_ppi_domain_ops, NULL);
989
+
if (!d)
990
+
return -ENOMEM;
991
+
992
+
irq_domain_update_bus_token(d, DOMAIN_BUS_WIRED);
993
+
gicv5_global_data.ppi_domain = d;
994
+
995
+
if (spi_count) {
996
+
d = irq_domain_create_linear(handle, spi_count,
997
+
&gicv5_irq_spi_domain_ops, NULL);
998
+
999
+
if (!d) {
1000
+
gicv5_free_domains();
1001
+
return -ENOMEM;
1002
+
}
1003
+
1004
+
gicv5_global_data.spi_domain = d;
1005
+
irq_domain_update_bus_token(d, DOMAIN_BUS_WIRED);
1006
+
}
1007
+
1008
+
if (!WARN(!gicv5_global_data.lpi_domain,
1009
+
"LPI domain uninitialized, can't set up IPIs")) {
1010
+
d = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain,
1011
+
0, GICV5_IPIS_PER_CPU * nr_cpu_ids,
1012
+
NULL, &gicv5_irq_ipi_domain_ops,
1013
+
NULL);
1014
+
1015
+
if (!d) {
1016
+
gicv5_free_domains();
1017
+
return -ENOMEM;
1018
+
}
1019
+
gicv5_global_data.ipi_domain = d;
1020
+
}
1021
+
gicv5_global_data.fwnode = handle;
1022
+
1023
+
return 0;
1024
+
}
1025
+
1026
+
static void gicv5_set_cpuif_pribits(void)
1027
+
{
1028
+
u64 icc_idr0 = read_sysreg_s(SYS_ICC_IDR0_EL1);
1029
+
1030
+
switch (FIELD_GET(ICC_IDR0_EL1_PRI_BITS, icc_idr0)) {
1031
+
case ICC_IDR0_EL1_PRI_BITS_4BITS:
1032
+
gicv5_global_data.cpuif_pri_bits = 4;
1033
+
break;
1034
+
case ICC_IDR0_EL1_PRI_BITS_5BITS:
1035
+
gicv5_global_data.cpuif_pri_bits = 5;
1036
+
break;
1037
+
default:
1038
+
pr_err("Unexpected ICC_IDR0_EL1_PRI_BITS value, default to 4");
1039
+
gicv5_global_data.cpuif_pri_bits = 4;
1040
+
break;
1041
+
}
1042
+
}
1043
+
1044
+
static void gicv5_set_cpuif_idbits(void)
1045
+
{
1046
+
u32 icc_idr0 = read_sysreg_s(SYS_ICC_IDR0_EL1);
1047
+
1048
+
switch (FIELD_GET(ICC_IDR0_EL1_ID_BITS, icc_idr0)) {
1049
+
case ICC_IDR0_EL1_ID_BITS_16BITS:
1050
+
gicv5_global_data.cpuif_id_bits = 16;
1051
+
break;
1052
+
case ICC_IDR0_EL1_ID_BITS_24BITS:
1053
+
gicv5_global_data.cpuif_id_bits = 24;
1054
+
break;
1055
+
default:
1056
+
pr_err("Unexpected ICC_IDR0_EL1_ID_BITS value, default to 16");
1057
+
gicv5_global_data.cpuif_id_bits = 16;
1058
+
break;
1059
+
}
1060
+
}
1061
+
1062
+
#ifdef CONFIG_KVM
1063
+
static struct gic_kvm_info gic_v5_kvm_info __initdata;
1064
+
1065
+
static bool __init gicv5_cpuif_has_gcie_legacy(void)
1066
+
{
1067
+
u64 idr0 = read_sysreg_s(SYS_ICC_IDR0_EL1);
1068
+
return !!FIELD_GET(ICC_IDR0_EL1_GCIE_LEGACY, idr0);
1069
+
}
1070
+
1071
+
static void __init gic_of_setup_kvm_info(struct device_node *node)
1072
+
{
1073
+
gic_v5_kvm_info.type = GIC_V5;
1074
+
gic_v5_kvm_info.has_gcie_v3_compat = gicv5_cpuif_has_gcie_legacy();
1075
+
1076
+
/* GIC Virtual CPU interface maintenance interrupt */
1077
+
gic_v5_kvm_info.no_maint_irq_mask = false;
1078
+
gic_v5_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
1079
+
if (!gic_v5_kvm_info.maint_irq) {
1080
+
pr_warn("cannot find GICv5 virtual CPU interface maintenance interrupt\n");
1081
+
return;
1082
+
}
1083
+
1084
+
vgic_set_kvm_info(&gic_v5_kvm_info);
1085
+
}
1086
+
#else
1087
+
static inline void __init gic_of_setup_kvm_info(struct device_node *node)
1088
+
{
1089
+
}
1090
+
#endif // CONFIG_KVM
1091
+
1092
+
static int __init gicv5_of_init(struct device_node *node, struct device_node *parent)
1093
+
{
1094
+
int ret = gicv5_irs_of_probe(node);
1095
+
if (ret)
1096
+
return ret;
1097
+
1098
+
ret = gicv5_init_domains(of_fwnode_handle(node));
1099
+
if (ret)
1100
+
goto out_irs;
1101
+
1102
+
gicv5_set_cpuif_pribits();
1103
+
gicv5_set_cpuif_idbits();
1104
+
1105
+
pri_bits = min_not_zero(gicv5_global_data.cpuif_pri_bits,
1106
+
gicv5_global_data.irs_pri_bits);
1107
+
1108
+
ret = gicv5_starting_cpu(smp_processor_id());
1109
+
if (ret)
1110
+
goto out_dom;
1111
+
1112
+
ret = set_handle_irq(gicv5_handle_irq);
1113
+
if (ret)
1114
+
goto out_int;
1115
+
1116
+
ret = gicv5_irs_enable();
1117
+
if (ret)
1118
+
goto out_int;
1119
+
1120
+
gicv5_smp_init();
1121
+
1122
+
gicv5_irs_its_probe();
1123
+
1124
+
gic_of_setup_kvm_info(node);
1125
+
1126
+
return 0;
1127
+
1128
+
out_int:
1129
+
gicv5_cpu_disable_interrupts();
1130
+
out_dom:
1131
+
gicv5_free_domains();
1132
+
out_irs:
1133
+
gicv5_irs_remove();
1134
+
1135
+
return ret;
1136
+
}
1137
+
IRQCHIP_DECLARE(gic_v5, "arm,gic-v5", gicv5_of_init);
+1
-1
drivers/irqchip/irq-gic.c
+1
-1
drivers/irqchip/irq-gic.c
+4
-1
drivers/irqchip/irq-msi-lib.c
+4
-1
drivers/irqchip/irq-msi-lib.c
···
133
133
{
134
134
const struct msi_parent_ops *ops = d->msi_parent_ops;
135
135
u32 busmask = BIT(bus_token);
136
+
struct fwnode_handle *fwh;
136
137
137
138
if (!ops)
138
139
return 0;
139
140
140
-
if (fwspec->fwnode != d->fwnode || fwspec->param_count != 0)
141
+
fwh = d->flags & IRQ_DOMAIN_FLAG_FWNODE_PARENT ? fwnode_get_parent(fwspec->fwnode)
142
+
: fwspec->fwnode;
143
+
if (fwh != d->fwnode || fwspec->param_count != 0)
141
144
return 0;
142
145
143
146
/* Handle pure domain searches */
+17
-5
drivers/of/irq.c
+17
-5
drivers/of/irq.c
···
670
670
}
671
671
}
672
672
673
-
static u32 __of_msi_map_id(struct device *dev, struct device_node **np,
674
-
u32 id_in)
673
+
/**
674
+
* of_msi_xlate - map a MSI ID and find relevant MSI controller node
675
+
* @dev: device for which the mapping is to be done.
676
+
* @msi_np: Pointer to store the MSI controller node
677
+
* @id_in: Device ID.
678
+
*
679
+
* Walk up the device hierarchy looking for devices with a "msi-map"
680
+
* property. If found, apply the mapping to @id_in. @msi_np pointed
681
+
* value must be NULL on entry, if an MSI controller is found @msi_np is
682
+
* initialized to the MSI controller node with a reference held.
683
+
*
684
+
* Returns: The mapped MSI id.
685
+
*/
686
+
u32 of_msi_xlate(struct device *dev, struct device_node **msi_np, u32 id_in)
675
687
{
676
688
struct device *parent_dev;
677
689
u32 id_out = id_in;
···
694
682
*/
695
683
for (parent_dev = dev; parent_dev; parent_dev = parent_dev->parent)
696
684
if (!of_map_id(parent_dev->of_node, id_in, "msi-map",
697
-
"msi-map-mask", np, &id_out))
685
+
"msi-map-mask", msi_np, &id_out))
698
686
break;
699
687
return id_out;
700
688
}
···
712
700
*/
713
701
u32 of_msi_map_id(struct device *dev, struct device_node *msi_np, u32 id_in)
714
702
{
715
-
return __of_msi_map_id(dev, &msi_np, id_in);
703
+
return of_msi_xlate(dev, &msi_np, id_in);
716
704
}
717
705
718
706
/**
···
731
719
{
732
720
struct device_node *np = NULL;
733
721
734
-
__of_msi_map_id(dev, &np, id);
722
+
of_msi_xlate(dev, &np, id);
735
723
return irq_find_matching_host(np, bus_token);
736
724
}
737
725
+20
drivers/pci/msi/irqdomain.c
+20
drivers/pci/msi/irqdomain.c
···
428
428
}
429
429
430
430
/**
431
+
* pci_msi_map_rid_ctlr_node - Get the MSI controller node and MSI requester id (RID)
432
+
* @pdev: The PCI device
433
+
* @node: Pointer to store the MSI controller device node
434
+
*
435
+
* Use the firmware data to find the MSI controller node for @pdev.
436
+
* If found map the RID and initialize @node with it. @node value must
437
+
* be set to NULL on entry.
438
+
*
439
+
* Returns: The RID.
440
+
*/
441
+
u32 pci_msi_map_rid_ctlr_node(struct pci_dev *pdev, struct device_node **node)
442
+
{
443
+
u32 rid = pci_dev_id(pdev);
444
+
445
+
pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
446
+
447
+
return of_msi_xlate(&pdev->dev, node, rid);
448
+
}
449
+
450
+
/**
431
451
* pci_msi_get_device_domain - Get the MSI domain for a given PCI device
432
452
* @pdev: The PCI device
433
453
*
+1
include/asm-generic/msi.h
+1
include/asm-generic/msi.h
+8
-1
include/kvm/arm_vgic.h
+8
-1
include/kvm/arm_vgic.h
···
38
38
enum vgic_type {
39
39
VGIC_V2, /* Good ol' GICv2 */
40
40
VGIC_V3, /* New fancy GICv3 */
41
+
VGIC_V5, /* Newer, fancier GICv5 */
41
42
};
42
43
43
44
/* same for all guests, as depending only on the _host's_ GIC model */
···
78
77
/* Pseudo GICv3 from outer space */
79
78
bool no_hw_deactivation;
80
79
81
-
/* GIC system register CPU interface */
80
+
/* GICv3 system register CPU interface */
82
81
struct static_key_false gicv3_cpuif;
82
+
83
+
/* GICv3 compat mode on a GICv5 host */
84
+
bool has_gcie_v3_compat;
83
85
84
86
u32 ich_vtr_el2;
85
87
};
···
267
263
268
264
/* distributor enabled */
269
265
bool enabled;
266
+
267
+
/* Supports SGIs without active state */
268
+
bool nassgicap;
270
269
271
270
/* Wants SGIs without active state */
272
271
bool nassgireq;
+394
include/linux/irqchip/arm-gic-v5.h
+394
include/linux/irqchip/arm-gic-v5.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* Copyright (C) 2025 ARM Limited, All Rights Reserved.
4
+
*/
5
+
#ifndef __LINUX_IRQCHIP_ARM_GIC_V5_H
6
+
#define __LINUX_IRQCHIP_ARM_GIC_V5_H
7
+
8
+
#include <linux/iopoll.h>
9
+
10
+
#include <asm/cacheflush.h>
11
+
#include <asm/smp.h>
12
+
#include <asm/sysreg.h>
13
+
14
+
#define GICV5_IPIS_PER_CPU MAX_IPI
15
+
16
+
/*
17
+
* INTID handling
18
+
*/
19
+
#define GICV5_HWIRQ_ID GENMASK(23, 0)
20
+
#define GICV5_HWIRQ_TYPE GENMASK(31, 29)
21
+
#define GICV5_HWIRQ_INTID GENMASK_ULL(31, 0)
22
+
23
+
#define GICV5_HWIRQ_TYPE_PPI UL(0x1)
24
+
#define GICV5_HWIRQ_TYPE_LPI UL(0x2)
25
+
#define GICV5_HWIRQ_TYPE_SPI UL(0x3)
26
+
27
+
/*
28
+
* Tables attributes
29
+
*/
30
+
#define GICV5_NO_READ_ALLOC 0b0
31
+
#define GICV5_READ_ALLOC 0b1
32
+
#define GICV5_NO_WRITE_ALLOC 0b0
33
+
#define GICV5_WRITE_ALLOC 0b1
34
+
35
+
#define GICV5_NON_CACHE 0b00
36
+
#define GICV5_WB_CACHE 0b01
37
+
#define GICV5_WT_CACHE 0b10
38
+
39
+
#define GICV5_NON_SHARE 0b00
40
+
#define GICV5_OUTER_SHARE 0b10
41
+
#define GICV5_INNER_SHARE 0b11
42
+
43
+
/*
44
+
* IRS registers and tables structures
45
+
*/
46
+
#define GICV5_IRS_IDR1 0x0004
47
+
#define GICV5_IRS_IDR2 0x0008
48
+
#define GICV5_IRS_IDR5 0x0014
49
+
#define GICV5_IRS_IDR6 0x0018
50
+
#define GICV5_IRS_IDR7 0x001c
51
+
#define GICV5_IRS_CR0 0x0080
52
+
#define GICV5_IRS_CR1 0x0084
53
+
#define GICV5_IRS_SYNCR 0x00c0
54
+
#define GICV5_IRS_SYNC_STATUSR 0x00c4
55
+
#define GICV5_IRS_SPI_SELR 0x0108
56
+
#define GICV5_IRS_SPI_CFGR 0x0114
57
+
#define GICV5_IRS_SPI_STATUSR 0x0118
58
+
#define GICV5_IRS_PE_SELR 0x0140
59
+
#define GICV5_IRS_PE_STATUSR 0x0144
60
+
#define GICV5_IRS_PE_CR0 0x0148
61
+
#define GICV5_IRS_IST_BASER 0x0180
62
+
#define GICV5_IRS_IST_CFGR 0x0190
63
+
#define GICV5_IRS_IST_STATUSR 0x0194
64
+
#define GICV5_IRS_MAP_L2_ISTR 0x01c0
65
+
66
+
#define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20)
67
+
#define GICV5_IRS_IDR1_IAFFID_BITS GENMASK(19, 16)
68
+
69
+
#define GICV5_IRS_IDR1_PRIORITY_BITS_1BITS 0b000
70
+
#define GICV5_IRS_IDR1_PRIORITY_BITS_2BITS 0b001
71
+
#define GICV5_IRS_IDR1_PRIORITY_BITS_3BITS 0b010
72
+
#define GICV5_IRS_IDR1_PRIORITY_BITS_4BITS 0b011
73
+
#define GICV5_IRS_IDR1_PRIORITY_BITS_5BITS 0b100
74
+
75
+
#define GICV5_IRS_IDR2_ISTMD_SZ GENMASK(19, 15)
76
+
#define GICV5_IRS_IDR2_ISTMD BIT(14)
77
+
#define GICV5_IRS_IDR2_IST_L2SZ GENMASK(13, 11)
78
+
#define GICV5_IRS_IDR2_IST_LEVELS BIT(10)
79
+
#define GICV5_IRS_IDR2_MIN_LPI_ID_BITS GENMASK(9, 6)
80
+
#define GICV5_IRS_IDR2_LPI BIT(5)
81
+
#define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0)
82
+
83
+
#define GICV5_IRS_IDR5_SPI_RANGE GENMASK(24, 0)
84
+
#define GICV5_IRS_IDR6_SPI_IRS_RANGE GENMASK(24, 0)
85
+
#define GICV5_IRS_IDR7_SPI_BASE GENMASK(23, 0)
86
+
87
+
#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r))
88
+
#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r))
89
+
#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r))
90
+
91
+
#define GICV5_IRS_CR0_IDLE BIT(1)
92
+
#define GICV5_IRS_CR0_IRSEN BIT(0)
93
+
94
+
#define GICV5_IRS_CR1_VPED_WA BIT(15)
95
+
#define GICV5_IRS_CR1_VPED_RA BIT(14)
96
+
#define GICV5_IRS_CR1_VMD_WA BIT(13)
97
+
#define GICV5_IRS_CR1_VMD_RA BIT(12)
98
+
#define GICV5_IRS_CR1_VPET_WA BIT(11)
99
+
#define GICV5_IRS_CR1_VPET_RA BIT(10)
100
+
#define GICV5_IRS_CR1_VMT_WA BIT(9)
101
+
#define GICV5_IRS_CR1_VMT_RA BIT(8)
102
+
#define GICV5_IRS_CR1_IST_WA BIT(7)
103
+
#define GICV5_IRS_CR1_IST_RA BIT(6)
104
+
#define GICV5_IRS_CR1_IC GENMASK(5, 4)
105
+
#define GICV5_IRS_CR1_OC GENMASK(3, 2)
106
+
#define GICV5_IRS_CR1_SH GENMASK(1, 0)
107
+
108
+
#define GICV5_IRS_SYNCR_SYNC BIT(31)
109
+
110
+
#define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0)
111
+
112
+
#define GICV5_IRS_SPI_STATUSR_V BIT(1)
113
+
#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0)
114
+
115
+
#define GICV5_IRS_SPI_SELR_ID GENMASK(23, 0)
116
+
117
+
#define GICV5_IRS_SPI_CFGR_TM BIT(0)
118
+
119
+
#define GICV5_IRS_PE_SELR_IAFFID GENMASK(15, 0)
120
+
121
+
#define GICV5_IRS_PE_STATUSR_V BIT(1)
122
+
#define GICV5_IRS_PE_STATUSR_IDLE BIT(0)
123
+
124
+
#define GICV5_IRS_PE_CR0_DPS BIT(0)
125
+
126
+
#define GICV5_IRS_IST_STATUSR_IDLE BIT(0)
127
+
128
+
#define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16)
129
+
#define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7)
130
+
#define GICV5_IRS_IST_CFGR_L2SZ GENMASK(6, 5)
131
+
#define GICV5_IRS_IST_CFGR_LPI_ID_BITS GENMASK(4, 0)
132
+
133
+
#define GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR 0b0
134
+
#define GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL 0b1
135
+
136
+
#define GICV5_IRS_IST_CFGR_ISTSZ_4 0b00
137
+
#define GICV5_IRS_IST_CFGR_ISTSZ_8 0b01
138
+
#define GICV5_IRS_IST_CFGR_ISTSZ_16 0b10
139
+
140
+
#define GICV5_IRS_IST_CFGR_L2SZ_4K 0b00
141
+
#define GICV5_IRS_IST_CFGR_L2SZ_16K 0b01
142
+
#define GICV5_IRS_IST_CFGR_L2SZ_64K 0b10
143
+
144
+
#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6)
145
+
#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0)
146
+
147
+
#define GICV5_IRS_MAP_L2_ISTR_ID GENMASK(23, 0)
148
+
149
+
#define GICV5_ISTL1E_VALID BIT_ULL(0)
150
+
151
+
#define GICV5_ISTL1E_L2_ADDR_MASK GENMASK_ULL(55, 12)
152
+
153
+
/*
154
+
* ITS registers and tables structures
155
+
*/
156
+
#define GICV5_ITS_IDR1 0x0004
157
+
#define GICV5_ITS_IDR2 0x0008
158
+
#define GICV5_ITS_CR0 0x0080
159
+
#define GICV5_ITS_CR1 0x0084
160
+
#define GICV5_ITS_DT_BASER 0x00c0
161
+
#define GICV5_ITS_DT_CFGR 0x00d0
162
+
#define GICV5_ITS_DIDR 0x0100
163
+
#define GICV5_ITS_EIDR 0x0108
164
+
#define GICV5_ITS_INV_EVENTR 0x010c
165
+
#define GICV5_ITS_INV_DEVICER 0x0110
166
+
#define GICV5_ITS_STATUSR 0x0120
167
+
#define GICV5_ITS_SYNCR 0x0140
168
+
#define GICV5_ITS_SYNC_STATUSR 0x0148
169
+
170
+
#define GICV5_ITS_IDR1_L2SZ GENMASK(10, 8)
171
+
#define GICV5_ITS_IDR1_ITT_LEVELS BIT(7)
172
+
#define GICV5_ITS_IDR1_DT_LEVELS BIT(6)
173
+
#define GICV5_ITS_IDR1_DEVICEID_BITS GENMASK(5, 0)
174
+
175
+
#define GICV5_ITS_IDR1_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(8), (r))
176
+
#define GICV5_ITS_IDR1_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(9), (r))
177
+
#define GICV5_ITS_IDR1_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(10), (r))
178
+
179
+
#define GICV5_ITS_IDR2_XDMN_EVENTs GENMASK(6, 5)
180
+
#define GICV5_ITS_IDR2_EVENTID_BITS GENMASK(4, 0)
181
+
182
+
#define GICV5_ITS_CR0_IDLE BIT(1)
183
+
#define GICV5_ITS_CR0_ITSEN BIT(0)
184
+
185
+
#define GICV5_ITS_CR1_ITT_RA BIT(7)
186
+
#define GICV5_ITS_CR1_DT_RA BIT(6)
187
+
#define GICV5_ITS_CR1_IC GENMASK(5, 4)
188
+
#define GICV5_ITS_CR1_OC GENMASK(3, 2)
189
+
#define GICV5_ITS_CR1_SH GENMASK(1, 0)
190
+
191
+
#define GICV5_ITS_DT_CFGR_STRUCTURE BIT(16)
192
+
#define GICV5_ITS_DT_CFGR_L2SZ GENMASK(7, 6)
193
+
#define GICV5_ITS_DT_CFGR_DEVICEID_BITS GENMASK(5, 0)
194
+
195
+
#define GICV5_ITS_DT_BASER_ADDR_MASK GENMASK_ULL(55, 3)
196
+
197
+
#define GICV5_ITS_INV_DEVICER_I BIT(31)
198
+
#define GICV5_ITS_INV_DEVICER_EVENTID_BITS GENMASK(5, 1)
199
+
#define GICV5_ITS_INV_DEVICER_L1 BIT(0)
200
+
201
+
#define GICV5_ITS_DIDR_DEVICEID GENMASK_ULL(31, 0)
202
+
203
+
#define GICV5_ITS_EIDR_EVENTID GENMASK(15, 0)
204
+
205
+
#define GICV5_ITS_INV_EVENTR_I BIT(31)
206
+
#define GICV5_ITS_INV_EVENTR_ITT_L2SZ GENMASK(2, 1)
207
+
#define GICV5_ITS_INV_EVENTR_L1 BIT(0)
208
+
209
+
#define GICV5_ITS_STATUSR_IDLE BIT(0)
210
+
211
+
#define GICV5_ITS_SYNCR_SYNC BIT_ULL(63)
212
+
#define GICV5_ITS_SYNCR_SYNCALL BIT_ULL(32)
213
+
#define GICV5_ITS_SYNCR_DEVICEID GENMASK_ULL(31, 0)
214
+
215
+
#define GICV5_ITS_SYNC_STATUSR_IDLE BIT(0)
216
+
217
+
#define GICV5_DTL1E_VALID BIT_ULL(0)
218
+
/* Note that there is no shift for the address by design */
219
+
#define GICV5_DTL1E_L2_ADDR_MASK GENMASK_ULL(55, 3)
220
+
#define GICV5_DTL1E_SPAN GENMASK_ULL(63, 60)
221
+
222
+
#define GICV5_DTL2E_VALID BIT_ULL(0)
223
+
#define GICV5_DTL2E_ITT_L2SZ GENMASK_ULL(2, 1)
224
+
/* Note that there is no shift for the address by design */
225
+
#define GICV5_DTL2E_ITT_ADDR_MASK GENMASK_ULL(55, 3)
226
+
#define GICV5_DTL2E_ITT_DSWE BIT_ULL(57)
227
+
#define GICV5_DTL2E_ITT_STRUCTURE BIT_ULL(58)
228
+
#define GICV5_DTL2E_EVENT_ID_BITS GENMASK_ULL(63, 59)
229
+
230
+
#define GICV5_ITTL1E_VALID BIT_ULL(0)
231
+
/* Note that there is no shift for the address by design */
232
+
#define GICV5_ITTL1E_L2_ADDR_MASK GENMASK_ULL(55, 3)
233
+
#define GICV5_ITTL1E_SPAN GENMASK_ULL(63, 60)
234
+
235
+
#define GICV5_ITTL2E_LPI_ID GENMASK_ULL(23, 0)
236
+
#define GICV5_ITTL2E_DAC GENMASK_ULL(29, 28)
237
+
#define GICV5_ITTL2E_VIRTUAL BIT_ULL(30)
238
+
#define GICV5_ITTL2E_VALID BIT_ULL(31)
239
+
#define GICV5_ITTL2E_VM_ID GENMASK_ULL(47, 32)
240
+
241
+
#define GICV5_ITS_DT_ITT_CFGR_L2SZ_4k 0b00
242
+
#define GICV5_ITS_DT_ITT_CFGR_L2SZ_16k 0b01
243
+
#define GICV5_ITS_DT_ITT_CFGR_L2SZ_64k 0b10
244
+
245
+
#define GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR 0
246
+
#define GICV5_ITS_DT_ITT_CFGR_STRUCTURE_TWO_LEVEL 1
247
+
248
+
#define GICV5_ITS_HWIRQ_DEVICE_ID GENMASK_ULL(31, 0)
249
+
#define GICV5_ITS_HWIRQ_EVENT_ID GENMASK_ULL(63, 32)
250
+
251
+
/*
252
+
* IWB registers
253
+
*/
254
+
#define GICV5_IWB_IDR0 0x0000
255
+
#define GICV5_IWB_CR0 0x0080
256
+
#define GICV5_IWB_WENABLE_STATUSR 0x00c0
257
+
#define GICV5_IWB_WENABLER 0x2000
258
+
#define GICV5_IWB_WTMR 0x4000
259
+
260
+
#define GICV5_IWB_IDR0_INT_DOMS GENMASK(14, 11)
261
+
#define GICV5_IWB_IDR0_IW_RANGE GENMASK(10, 0)
262
+
263
+
#define GICV5_IWB_CR0_IDLE BIT(1)
264
+
#define GICV5_IWB_CR0_IWBEN BIT(0)
265
+
266
+
#define GICV5_IWB_WENABLE_STATUSR_IDLE BIT(0)
267
+
268
+
/*
269
+
* Global Data structures and functions
270
+
*/
271
+
struct gicv5_chip_data {
272
+
struct fwnode_handle *fwnode;
273
+
struct irq_domain *ppi_domain;
274
+
struct irq_domain *spi_domain;
275
+
struct irq_domain *lpi_domain;
276
+
struct irq_domain *ipi_domain;
277
+
u32 global_spi_count;
278
+
u8 cpuif_pri_bits;
279
+
u8 cpuif_id_bits;
280
+
u8 irs_pri_bits;
281
+
struct {
282
+
__le64 *l1ist_addr;
283
+
u32 l2_size;
284
+
u8 l2_bits;
285
+
bool l2;
286
+
} ist;
287
+
};
288
+
289
+
extern struct gicv5_chip_data gicv5_global_data __read_mostly;
290
+
291
+
struct gicv5_irs_chip_data {
292
+
struct list_head entry;
293
+
struct fwnode_handle *fwnode;
294
+
void __iomem *irs_base;
295
+
u32 flags;
296
+
u32 spi_min;
297
+
u32 spi_range;
298
+
raw_spinlock_t spi_config_lock;
299
+
};
300
+
301
+
static inline int gicv5_wait_for_op_s_atomic(void __iomem *addr, u32 offset,
302
+
const char *reg_s, u32 mask,
303
+
u32 *val)
304
+
{
305
+
void __iomem *reg = addr + offset;
306
+
u32 tmp;
307
+
int ret;
308
+
309
+
ret = readl_poll_timeout_atomic(reg, tmp, tmp & mask, 1, 10 * USEC_PER_MSEC);
310
+
if (unlikely(ret == -ETIMEDOUT)) {
311
+
pr_err_ratelimited("%s timeout...\n", reg_s);
312
+
return ret;
313
+
}
314
+
315
+
if (val)
316
+
*val = tmp;
317
+
318
+
return 0;
319
+
}
320
+
321
+
static inline int gicv5_wait_for_op_s(void __iomem *addr, u32 offset,
322
+
const char *reg_s, u32 mask)
323
+
{
324
+
void __iomem *reg = addr + offset;
325
+
u32 val;
326
+
int ret;
327
+
328
+
ret = readl_poll_timeout(reg, val, val & mask, 1, 10 * USEC_PER_MSEC);
329
+
if (unlikely(ret == -ETIMEDOUT)) {
330
+
pr_err_ratelimited("%s timeout...\n", reg_s);
331
+
return ret;
332
+
}
333
+
334
+
return 0;
335
+
}
336
+
337
+
#define gicv5_wait_for_op_atomic(base, reg, mask, val) \
338
+
gicv5_wait_for_op_s_atomic(base, reg, #reg, mask, val)
339
+
340
+
#define gicv5_wait_for_op(base, reg, mask) \
341
+
gicv5_wait_for_op_s(base, reg, #reg, mask)
342
+
343
+
void __init gicv5_init_lpi_domain(void);
344
+
void __init gicv5_free_lpi_domain(void);
345
+
346
+
int gicv5_irs_of_probe(struct device_node *parent);
347
+
void gicv5_irs_remove(void);
348
+
int gicv5_irs_enable(void);
349
+
void gicv5_irs_its_probe(void);
350
+
int gicv5_irs_register_cpu(int cpuid);
351
+
int gicv5_irs_cpu_to_iaffid(int cpu_id, u16 *iaffid);
352
+
struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id);
353
+
int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type);
354
+
int gicv5_irs_iste_alloc(u32 lpi);
355
+
void gicv5_irs_syncr(void);
356
+
357
+
struct gicv5_its_devtab_cfg {
358
+
union {
359
+
struct {
360
+
__le64 *devtab;
361
+
} linear;
362
+
struct {
363
+
__le64 *l1devtab;
364
+
__le64 **l2ptrs;
365
+
} l2;
366
+
};
367
+
u32 cfgr;
368
+
};
369
+
370
+
struct gicv5_its_itt_cfg {
371
+
union {
372
+
struct {
373
+
__le64 *itt;
374
+
unsigned int num_ents;
375
+
} linear;
376
+
struct {
377
+
__le64 *l1itt;
378
+
__le64 **l2ptrs;
379
+
unsigned int num_l1_ents;
380
+
u8 l2sz;
381
+
} l2;
382
+
};
383
+
u8 event_id_bits;
384
+
bool l2itt;
385
+
};
386
+
387
+
void gicv5_init_lpis(u32 max);
388
+
void gicv5_deinit_lpis(void);
389
+
390
+
int gicv5_alloc_lpi(void);
391
+
void gicv5_free_lpi(u32 lpi);
392
+
393
+
void __init gicv5_its_of_probe(struct device_node *parent);
394
+
#endif
+4
include/linux/irqchip/arm-vgic-info.h
+4
include/linux/irqchip/arm-vgic-info.h
···
15
15
GIC_V2,
16
16
/* Full GICv3, optionally with v2 compat */
17
17
GIC_V3,
18
+
/* Full GICv5, optionally with v3 compat */
19
+
GIC_V5,
18
20
};
19
21
20
22
struct gic_kvm_info {
···
36
34
bool has_v4_1;
37
35
/* Deactivation impared, subpar stuff */
38
36
bool no_hw_deactivation;
37
+
/* v3 compat support (GICv5 hosts, only) */
38
+
bool has_gcie_v3_compat;
39
39
};
40
40
41
41
#ifdef CONFIG_KVM
+3
include/linux/irqdomain.h
+3
include/linux/irqdomain.h
···
212
212
/* Address and data pair is mutable when irq_set_affinity() */
213
213
IRQ_DOMAIN_FLAG_MSI_IMMUTABLE = (1 << 11),
214
214
215
+
/* IRQ domain requires parent fwnode matching */
216
+
IRQ_DOMAIN_FLAG_FWNODE_PARENT = (1 << 12),
217
+
215
218
/*
216
219
* Flags starting from IRQ_DOMAIN_FLAG_NONCORE are reserved
217
220
* for implementation specific purposes and ignored by the
+1
include/linux/msi.h
+1
include/linux/msi.h
···
705
705
struct msi_domain_info *info,
706
706
struct irq_domain *parent);
707
707
u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev);
708
+
u32 pci_msi_map_rid_ctlr_node(struct pci_dev *pdev, struct device_node **node);
708
709
struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev);
709
710
#else /* CONFIG_PCI_MSI */
710
711
static inline struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)
+5
include/linux/of_irq.h
+5
include/linux/of_irq.h
···
54
54
u32 id,
55
55
u32 bus_token);
56
56
extern void of_msi_configure(struct device *dev, const struct device_node *np);
57
+
extern u32 of_msi_xlate(struct device *dev, struct device_node **msi_np, u32 id_in);
57
58
u32 of_msi_map_id(struct device *dev, struct device_node *msi_np, u32 id_in);
58
59
#else
59
60
static inline void of_irq_init(const struct of_device_id *matches)
···
100
99
}
101
100
static inline void of_msi_configure(struct device *dev, struct device_node *np)
102
101
{
102
+
}
103
+
static inline u32 of_msi_xlate(struct device *dev, struct device_node **msi_np, u32 id_in)
104
+
{
105
+
return id_in;
103
106
}
104
107
static inline u32 of_msi_map_id(struct device *dev,
105
108
struct device_node *msi_np, u32 id_in)
+1
include/uapi/linux/kvm.h
+1
include/uapi/linux/kvm.h
+1
-1
tools/testing/selftests/kvm/Makefile.kvm
+1
-1
tools/testing/selftests/kvm/Makefile.kvm
···
158
158
TEST_GEN_PROGS_arm64 += arm64/debug-exceptions
159
159
TEST_GEN_PROGS_arm64 += arm64/host_sve
160
160
TEST_GEN_PROGS_arm64 += arm64/hypercalls
161
-
TEST_GEN_PROGS_arm64 += arm64/mmio_abort
161
+
TEST_GEN_PROGS_arm64 += arm64/external_aborts
162
162
TEST_GEN_PROGS_arm64 += arm64/page_fault_test
163
163
TEST_GEN_PROGS_arm64 += arm64/psci_test
164
164
TEST_GEN_PROGS_arm64 += arm64/set_id_regs
+330
tools/testing/selftests/kvm/arm64/external_aborts.c
+330
tools/testing/selftests/kvm/arm64/external_aborts.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* external_abort - Tests for userspace external abort injection
4
+
*
5
+
* Copyright (c) 2024 Google LLC
6
+
*/
7
+
#include "processor.h"
8
+
#include "test_util.h"
9
+
10
+
#define MMIO_ADDR 0x8000000ULL
11
+
#define EXPECTED_SERROR_ISS (ESR_ELx_ISV | 0x1d1ed)
12
+
13
+
static u64 expected_abort_pc;
14
+
15
+
static void expect_sea_handler(struct ex_regs *regs)
16
+
{
17
+
u64 esr = read_sysreg(esr_el1);
18
+
19
+
GUEST_ASSERT_EQ(regs->pc, expected_abort_pc);
20
+
GUEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_DABT_CUR);
21
+
GUEST_ASSERT_EQ(esr & ESR_ELx_FSC_TYPE, ESR_ELx_FSC_EXTABT);
22
+
23
+
GUEST_DONE();
24
+
}
25
+
26
+
static void unexpected_dabt_handler(struct ex_regs *regs)
27
+
{
28
+
GUEST_FAIL("Unexpected data abort at PC: %lx\n", regs->pc);
29
+
}
30
+
31
+
static struct kvm_vm *vm_create_with_dabt_handler(struct kvm_vcpu **vcpu, void *guest_code,
32
+
handler_fn dabt_handler)
33
+
{
34
+
struct kvm_vm *vm = vm_create_with_one_vcpu(vcpu, guest_code);
35
+
36
+
vm_init_descriptor_tables(vm);
37
+
vcpu_init_descriptor_tables(*vcpu);
38
+
vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, ESR_ELx_EC_DABT_CUR, dabt_handler);
39
+
40
+
virt_map(vm, MMIO_ADDR, MMIO_ADDR, 1);
41
+
42
+
return vm;
43
+
}
44
+
45
+
static void vcpu_inject_sea(struct kvm_vcpu *vcpu)
46
+
{
47
+
struct kvm_vcpu_events events = {};
48
+
49
+
events.exception.ext_dabt_pending = true;
50
+
vcpu_events_set(vcpu, &events);
51
+
}
52
+
53
+
static bool vcpu_has_ras(struct kvm_vcpu *vcpu)
54
+
{
55
+
u64 pfr0 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR0_EL1));
56
+
57
+
return SYS_FIELD_GET(ID_AA64PFR0_EL1, RAS, pfr0);
58
+
}
59
+
60
+
static bool guest_has_ras(void)
61
+
{
62
+
return SYS_FIELD_GET(ID_AA64PFR0_EL1, RAS, read_sysreg(id_aa64pfr0_el1));
63
+
}
64
+
65
+
static void vcpu_inject_serror(struct kvm_vcpu *vcpu)
66
+
{
67
+
struct kvm_vcpu_events events = {};
68
+
69
+
events.exception.serror_pending = true;
70
+
if (vcpu_has_ras(vcpu)) {
71
+
events.exception.serror_has_esr = true;
72
+
events.exception.serror_esr = EXPECTED_SERROR_ISS;
73
+
}
74
+
75
+
vcpu_events_set(vcpu, &events);
76
+
}
77
+
78
+
static void __vcpu_run_expect(struct kvm_vcpu *vcpu, unsigned int cmd)
79
+
{
80
+
struct ucall uc;
81
+
82
+
vcpu_run(vcpu);
83
+
switch (get_ucall(vcpu, &uc)) {
84
+
case UCALL_ABORT:
85
+
REPORT_GUEST_ASSERT(uc);
86
+
break;
87
+
default:
88
+
if (uc.cmd == cmd)
89
+
return;
90
+
91
+
TEST_FAIL("Unexpected ucall: %lu", uc.cmd);
92
+
}
93
+
}
94
+
95
+
static void vcpu_run_expect_done(struct kvm_vcpu *vcpu)
96
+
{
97
+
__vcpu_run_expect(vcpu, UCALL_DONE);
98
+
}
99
+
100
+
static void vcpu_run_expect_sync(struct kvm_vcpu *vcpu)
101
+
{
102
+
__vcpu_run_expect(vcpu, UCALL_SYNC);
103
+
}
104
+
105
+
extern char test_mmio_abort_insn;
106
+
107
+
static noinline void test_mmio_abort_guest(void)
108
+
{
109
+
WRITE_ONCE(expected_abort_pc, (u64)&test_mmio_abort_insn);
110
+
111
+
asm volatile("test_mmio_abort_insn:\n\t"
112
+
"ldr x0, [%0]\n\t"
113
+
: : "r" (MMIO_ADDR) : "x0", "memory");
114
+
115
+
GUEST_FAIL("MMIO instruction should not retire");
116
+
}
117
+
118
+
/*
119
+
* Test that KVM doesn't complete MMIO emulation when userspace has made an
120
+
* external abort pending for the instruction.
121
+
*/
122
+
static void test_mmio_abort(void)
123
+
{
124
+
struct kvm_vcpu *vcpu;
125
+
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_abort_guest,
126
+
expect_sea_handler);
127
+
struct kvm_run *run = vcpu->run;
128
+
129
+
vcpu_run(vcpu);
130
+
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO);
131
+
TEST_ASSERT_EQ(run->mmio.phys_addr, MMIO_ADDR);
132
+
TEST_ASSERT_EQ(run->mmio.len, sizeof(unsigned long));
133
+
TEST_ASSERT(!run->mmio.is_write, "Expected MMIO read");
134
+
135
+
vcpu_inject_sea(vcpu);
136
+
vcpu_run_expect_done(vcpu);
137
+
kvm_vm_free(vm);
138
+
}
139
+
140
+
extern char test_mmio_nisv_insn;
141
+
142
+
static void test_mmio_nisv_guest(void)
143
+
{
144
+
WRITE_ONCE(expected_abort_pc, (u64)&test_mmio_nisv_insn);
145
+
146
+
asm volatile("test_mmio_nisv_insn:\n\t"
147
+
"ldr x0, [%0], #8\n\t"
148
+
: : "r" (MMIO_ADDR) : "x0", "memory");
149
+
150
+
GUEST_FAIL("MMIO instruction should not retire");
151
+
}
152
+
153
+
/*
154
+
* Test that the KVM_RUN ioctl fails for ESR_EL2.ISV=0 MMIO aborts if userspace
155
+
* hasn't enabled KVM_CAP_ARM_NISV_TO_USER.
156
+
*/
157
+
static void test_mmio_nisv(void)
158
+
{
159
+
struct kvm_vcpu *vcpu;
160
+
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_nisv_guest,
161
+
unexpected_dabt_handler);
162
+
163
+
TEST_ASSERT(_vcpu_run(vcpu), "Expected nonzero return code from KVM_RUN");
164
+
TEST_ASSERT_EQ(errno, ENOSYS);
165
+
166
+
kvm_vm_free(vm);
167
+
}
168
+
169
+
/*
170
+
* Test that ESR_EL2.ISV=0 MMIO aborts reach userspace and that an injected SEA
171
+
* reaches the guest.
172
+
*/
173
+
static void test_mmio_nisv_abort(void)
174
+
{
175
+
struct kvm_vcpu *vcpu;
176
+
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_nisv_guest,
177
+
expect_sea_handler);
178
+
struct kvm_run *run = vcpu->run;
179
+
180
+
vm_enable_cap(vm, KVM_CAP_ARM_NISV_TO_USER, 1);
181
+
182
+
vcpu_run(vcpu);
183
+
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_ARM_NISV);
184
+
TEST_ASSERT_EQ(run->arm_nisv.fault_ipa, MMIO_ADDR);
185
+
186
+
vcpu_inject_sea(vcpu);
187
+
vcpu_run_expect_done(vcpu);
188
+
kvm_vm_free(vm);
189
+
}
190
+
191
+
static void unexpected_serror_handler(struct ex_regs *regs)
192
+
{
193
+
GUEST_FAIL("Took unexpected SError exception");
194
+
}
195
+
196
+
static void test_serror_masked_guest(void)
197
+
{
198
+
GUEST_ASSERT(read_sysreg(isr_el1) & ISR_EL1_A);
199
+
200
+
isb();
201
+
202
+
GUEST_DONE();
203
+
}
204
+
205
+
static void test_serror_masked(void)
206
+
{
207
+
struct kvm_vcpu *vcpu;
208
+
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_serror_masked_guest,
209
+
unexpected_dabt_handler);
210
+
211
+
vm_install_exception_handler(vm, VECTOR_ERROR_CURRENT, unexpected_serror_handler);
212
+
213
+
vcpu_inject_serror(vcpu);
214
+
vcpu_run_expect_done(vcpu);
215
+
kvm_vm_free(vm);
216
+
}
217
+
218
+
static void expect_serror_handler(struct ex_regs *regs)
219
+
{
220
+
u64 esr = read_sysreg(esr_el1);
221
+
222
+
GUEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_SERROR);
223
+
if (guest_has_ras())
224
+
GUEST_ASSERT_EQ(ESR_ELx_ISS(esr), EXPECTED_SERROR_ISS);
225
+
226
+
GUEST_DONE();
227
+
}
228
+
229
+
static void test_serror_guest(void)
230
+
{
231
+
GUEST_ASSERT(read_sysreg(isr_el1) & ISR_EL1_A);
232
+
233
+
local_serror_enable();
234
+
isb();
235
+
local_serror_disable();
236
+
237
+
GUEST_FAIL("Should've taken pending SError exception");
238
+
}
239
+
240
+
static void test_serror(void)
241
+
{
242
+
struct kvm_vcpu *vcpu;
243
+
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_serror_guest,
244
+
unexpected_dabt_handler);
245
+
246
+
vm_install_exception_handler(vm, VECTOR_ERROR_CURRENT, expect_serror_handler);
247
+
248
+
vcpu_inject_serror(vcpu);
249
+
vcpu_run_expect_done(vcpu);
250
+
kvm_vm_free(vm);
251
+
}
252
+
253
+
static void test_serror_emulated_guest(void)
254
+
{
255
+
GUEST_ASSERT(!(read_sysreg(isr_el1) & ISR_EL1_A));
256
+
257
+
local_serror_enable();
258
+
GUEST_SYNC(0);
259
+
local_serror_disable();
260
+
261
+
GUEST_FAIL("Should've taken unmasked SError exception");
262
+
}
263
+
264
+
static void test_serror_emulated(void)
265
+
{
266
+
struct kvm_vcpu *vcpu;
267
+
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_serror_emulated_guest,
268
+
unexpected_dabt_handler);
269
+
270
+
vm_install_exception_handler(vm, VECTOR_ERROR_CURRENT, expect_serror_handler);
271
+
272
+
vcpu_run_expect_sync(vcpu);
273
+
vcpu_inject_serror(vcpu);
274
+
vcpu_run_expect_done(vcpu);
275
+
kvm_vm_free(vm);
276
+
}
277
+
278
+
static void test_mmio_ease_guest(void)
279
+
{
280
+
sysreg_clear_set_s(SYS_SCTLR2_EL1, 0, SCTLR2_EL1_EASE);
281
+
isb();
282
+
283
+
test_mmio_abort_guest();
284
+
}
285
+
286
+
/*
287
+
* Test that KVM doesn't complete MMIO emulation when userspace has made an
288
+
* external abort pending for the instruction.
289
+
*/
290
+
static void test_mmio_ease(void)
291
+
{
292
+
struct kvm_vcpu *vcpu;
293
+
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_ease_guest,
294
+
unexpected_dabt_handler);
295
+
struct kvm_run *run = vcpu->run;
296
+
u64 pfr1;
297
+
298
+
pfr1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR1_EL1));
299
+
if (!SYS_FIELD_GET(ID_AA64PFR1_EL1, DF2, pfr1)) {
300
+
pr_debug("Skipping %s\n", __func__);
301
+
return;
302
+
}
303
+
304
+
/*
305
+
* SCTLR2_ELx.EASE changes the exception vector to the SError vector but
306
+
* doesn't further modify the exception context (e.g. ESR_ELx, FAR_ELx).
307
+
*/
308
+
vm_install_exception_handler(vm, VECTOR_ERROR_CURRENT, expect_sea_handler);
309
+
310
+
vcpu_run(vcpu);
311
+
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO);
312
+
TEST_ASSERT_EQ(run->mmio.phys_addr, MMIO_ADDR);
313
+
TEST_ASSERT_EQ(run->mmio.len, sizeof(unsigned long));
314
+
TEST_ASSERT(!run->mmio.is_write, "Expected MMIO read");
315
+
316
+
vcpu_inject_sea(vcpu);
317
+
vcpu_run_expect_done(vcpu);
318
+
kvm_vm_free(vm);
319
+
}
320
+
321
+
int main(void)
322
+
{
323
+
test_mmio_abort();
324
+
test_mmio_nisv();
325
+
test_mmio_nisv_abort();
326
+
test_serror();
327
+
test_serror_masked();
328
+
test_serror_emulated();
329
+
test_mmio_ease();
330
+
}
+173
-30
tools/testing/selftests/kvm/arm64/get-reg-list.c
+173
-30
tools/testing/selftests/kvm/arm64/get-reg-list.c
···
15
15
#include "test_util.h"
16
16
#include "processor.h"
17
17
18
+
#define SYS_REG(r) ARM64_SYS_REG(sys_reg_Op0(SYS_ ## r), \
19
+
sys_reg_Op1(SYS_ ## r), \
20
+
sys_reg_CRn(SYS_ ## r), \
21
+
sys_reg_CRm(SYS_ ## r), \
22
+
sys_reg_Op2(SYS_ ## r))
23
+
18
24
struct feature_id_reg {
19
25
__u64 reg;
20
26
__u64 id_reg;
···
28
22
__u64 feat_min;
29
23
};
30
24
31
-
static struct feature_id_reg feat_id_regs[] = {
32
-
{
33
-
ARM64_SYS_REG(3, 0, 2, 0, 3), /* TCR2_EL1 */
34
-
ARM64_SYS_REG(3, 0, 0, 7, 3), /* ID_AA64MMFR3_EL1 */
35
-
0,
36
-
1
37
-
},
38
-
{
39
-
ARM64_SYS_REG(3, 0, 10, 2, 2), /* PIRE0_EL1 */
40
-
ARM64_SYS_REG(3, 0, 0, 7, 3), /* ID_AA64MMFR3_EL1 */
41
-
8,
42
-
1
43
-
},
44
-
{
45
-
ARM64_SYS_REG(3, 0, 10, 2, 3), /* PIR_EL1 */
46
-
ARM64_SYS_REG(3, 0, 0, 7, 3), /* ID_AA64MMFR3_EL1 */
47
-
8,
48
-
1
49
-
},
50
-
{
51
-
ARM64_SYS_REG(3, 0, 10, 2, 4), /* POR_EL1 */
52
-
ARM64_SYS_REG(3, 0, 0, 7, 3), /* ID_AA64MMFR3_EL1 */
53
-
16,
54
-
1
55
-
},
56
-
{
57
-
ARM64_SYS_REG(3, 3, 10, 2, 4), /* POR_EL0 */
58
-
ARM64_SYS_REG(3, 0, 0, 7, 3), /* ID_AA64MMFR3_EL1 */
59
-
16,
60
-
1
25
+
#define FEAT(id, f, v) \
26
+
.id_reg = SYS_REG(id), \
27
+
.feat_shift = id ## _ ## f ## _SHIFT, \
28
+
.feat_min = id ## _ ## f ## _ ## v
29
+
30
+
#define REG_FEAT(r, id, f, v) \
31
+
{ \
32
+
.reg = SYS_REG(r), \
33
+
FEAT(id, f, v) \
61
34
}
35
+
36
+
static struct feature_id_reg feat_id_regs[] = {
37
+
REG_FEAT(TCR2_EL1, ID_AA64MMFR3_EL1, TCRX, IMP),
38
+
REG_FEAT(TCR2_EL2, ID_AA64MMFR3_EL1, TCRX, IMP),
39
+
REG_FEAT(PIRE0_EL1, ID_AA64MMFR3_EL1, S1PIE, IMP),
40
+
REG_FEAT(PIRE0_EL2, ID_AA64MMFR3_EL1, S1PIE, IMP),
41
+
REG_FEAT(PIR_EL1, ID_AA64MMFR3_EL1, S1PIE, IMP),
42
+
REG_FEAT(PIR_EL2, ID_AA64MMFR3_EL1, S1PIE, IMP),
43
+
REG_FEAT(POR_EL1, ID_AA64MMFR3_EL1, S1POE, IMP),
44
+
REG_FEAT(POR_EL0, ID_AA64MMFR3_EL1, S1POE, IMP),
45
+
REG_FEAT(POR_EL2, ID_AA64MMFR3_EL1, S1POE, IMP),
46
+
REG_FEAT(HCRX_EL2, ID_AA64MMFR1_EL1, HCX, IMP),
47
+
REG_FEAT(HFGRTR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
48
+
REG_FEAT(HFGWTR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
49
+
REG_FEAT(HFGITR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
50
+
REG_FEAT(HDFGRTR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
51
+
REG_FEAT(HDFGWTR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
52
+
REG_FEAT(HAFGRTR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
53
+
REG_FEAT(HFGRTR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2),
54
+
REG_FEAT(HFGWTR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2),
55
+
REG_FEAT(HFGITR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2),
56
+
REG_FEAT(HDFGRTR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2),
57
+
REG_FEAT(HDFGWTR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2),
58
+
REG_FEAT(ZCR_EL2, ID_AA64PFR0_EL1, SVE, IMP),
59
+
REG_FEAT(SCTLR2_EL1, ID_AA64MMFR3_EL1, SCTLRX, IMP),
60
+
REG_FEAT(VDISR_EL2, ID_AA64PFR0_EL1, RAS, IMP),
61
+
REG_FEAT(VSESR_EL2, ID_AA64PFR0_EL1, RAS, IMP),
62
62
};
63
63
64
64
bool filter_reg(__u64 reg)
···
481
469
ARM64_SYS_REG(3, 0, 1, 0, 0), /* SCTLR_EL1 */
482
470
ARM64_SYS_REG(3, 0, 1, 0, 1), /* ACTLR_EL1 */
483
471
ARM64_SYS_REG(3, 0, 1, 0, 2), /* CPACR_EL1 */
472
+
KVM_ARM64_SYS_REG(SYS_SCTLR2_EL1),
484
473
ARM64_SYS_REG(3, 0, 2, 0, 0), /* TTBR0_EL1 */
485
474
ARM64_SYS_REG(3, 0, 2, 0, 1), /* TTBR1_EL1 */
486
475
ARM64_SYS_REG(3, 0, 2, 0, 2), /* TCR_EL1 */
···
699
686
ARM64_SYS_REG(3, 0, 2, 3, 1), /* APGAKEYHI_EL1 */
700
687
};
701
688
689
+
static __u64 el2_regs[] = {
690
+
SYS_REG(VPIDR_EL2),
691
+
SYS_REG(VMPIDR_EL2),
692
+
SYS_REG(SCTLR_EL2),
693
+
SYS_REG(ACTLR_EL2),
694
+
SYS_REG(HCR_EL2),
695
+
SYS_REG(MDCR_EL2),
696
+
SYS_REG(CPTR_EL2),
697
+
SYS_REG(HSTR_EL2),
698
+
SYS_REG(HFGRTR_EL2),
699
+
SYS_REG(HFGWTR_EL2),
700
+
SYS_REG(HFGITR_EL2),
701
+
SYS_REG(HACR_EL2),
702
+
SYS_REG(ZCR_EL2),
703
+
SYS_REG(HCRX_EL2),
704
+
SYS_REG(TTBR0_EL2),
705
+
SYS_REG(TTBR1_EL2),
706
+
SYS_REG(TCR_EL2),
707
+
SYS_REG(TCR2_EL2),
708
+
SYS_REG(VTTBR_EL2),
709
+
SYS_REG(VTCR_EL2),
710
+
SYS_REG(VNCR_EL2),
711
+
SYS_REG(HDFGRTR2_EL2),
712
+
SYS_REG(HDFGWTR2_EL2),
713
+
SYS_REG(HFGRTR2_EL2),
714
+
SYS_REG(HFGWTR2_EL2),
715
+
SYS_REG(HDFGRTR_EL2),
716
+
SYS_REG(HDFGWTR_EL2),
717
+
SYS_REG(HAFGRTR_EL2),
718
+
SYS_REG(HFGITR2_EL2),
719
+
SYS_REG(SPSR_EL2),
720
+
SYS_REG(ELR_EL2),
721
+
SYS_REG(AFSR0_EL2),
722
+
SYS_REG(AFSR1_EL2),
723
+
SYS_REG(ESR_EL2),
724
+
SYS_REG(FAR_EL2),
725
+
SYS_REG(HPFAR_EL2),
726
+
SYS_REG(MAIR_EL2),
727
+
SYS_REG(PIRE0_EL2),
728
+
SYS_REG(PIR_EL2),
729
+
SYS_REG(POR_EL2),
730
+
SYS_REG(AMAIR_EL2),
731
+
SYS_REG(VBAR_EL2),
732
+
SYS_REG(CONTEXTIDR_EL2),
733
+
SYS_REG(TPIDR_EL2),
734
+
SYS_REG(CNTVOFF_EL2),
735
+
SYS_REG(CNTHCTL_EL2),
736
+
SYS_REG(CNTHP_CTL_EL2),
737
+
SYS_REG(CNTHP_CVAL_EL2),
738
+
SYS_REG(CNTHV_CTL_EL2),
739
+
SYS_REG(CNTHV_CVAL_EL2),
740
+
SYS_REG(SP_EL2),
741
+
SYS_REG(VDISR_EL2),
742
+
SYS_REG(VSESR_EL2),
743
+
};
744
+
702
745
#define BASE_SUBLIST \
703
746
{ "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), }
704
747
#define VREGS_SUBLIST \
···
780
711
.feature = KVM_ARM_VCPU_PTRAUTH_GENERIC, \
781
712
.regs = pauth_generic_regs, \
782
713
.regs_n = ARRAY_SIZE(pauth_generic_regs), \
714
+
}
715
+
#define EL2_SUBLIST \
716
+
{ \
717
+
.name = "EL2", \
718
+
.capability = KVM_CAP_ARM_EL2, \
719
+
.feature = KVM_ARM_VCPU_HAS_EL2, \
720
+
.regs = el2_regs, \
721
+
.regs_n = ARRAY_SIZE(el2_regs), \
783
722
}
784
723
785
724
static struct vcpu_reg_list vregs_config = {
···
838
761
},
839
762
};
840
763
764
+
static struct vcpu_reg_list el2_vregs_config = {
765
+
.sublists = {
766
+
BASE_SUBLIST,
767
+
EL2_SUBLIST,
768
+
VREGS_SUBLIST,
769
+
{0},
770
+
},
771
+
};
772
+
773
+
static struct vcpu_reg_list el2_vregs_pmu_config = {
774
+
.sublists = {
775
+
BASE_SUBLIST,
776
+
EL2_SUBLIST,
777
+
VREGS_SUBLIST,
778
+
PMU_SUBLIST,
779
+
{0},
780
+
},
781
+
};
782
+
783
+
static struct vcpu_reg_list el2_sve_config = {
784
+
.sublists = {
785
+
BASE_SUBLIST,
786
+
EL2_SUBLIST,
787
+
SVE_SUBLIST,
788
+
{0},
789
+
},
790
+
};
791
+
792
+
static struct vcpu_reg_list el2_sve_pmu_config = {
793
+
.sublists = {
794
+
BASE_SUBLIST,
795
+
EL2_SUBLIST,
796
+
SVE_SUBLIST,
797
+
PMU_SUBLIST,
798
+
{0},
799
+
},
800
+
};
801
+
802
+
static struct vcpu_reg_list el2_pauth_config = {
803
+
.sublists = {
804
+
BASE_SUBLIST,
805
+
EL2_SUBLIST,
806
+
VREGS_SUBLIST,
807
+
PAUTH_SUBLIST,
808
+
{0},
809
+
},
810
+
};
811
+
812
+
static struct vcpu_reg_list el2_pauth_pmu_config = {
813
+
.sublists = {
814
+
BASE_SUBLIST,
815
+
EL2_SUBLIST,
816
+
VREGS_SUBLIST,
817
+
PAUTH_SUBLIST,
818
+
PMU_SUBLIST,
819
+
{0},
820
+
},
821
+
};
822
+
841
823
struct vcpu_reg_list *vcpu_configs[] = {
842
824
&vregs_config,
843
825
&vregs_pmu_config,
···
904
768
&sve_pmu_config,
905
769
&pauth_config,
906
770
&pauth_pmu_config,
771
+
772
+
&el2_vregs_config,
773
+
&el2_vregs_pmu_config,
774
+
&el2_sve_config,
775
+
&el2_sve_pmu_config,
776
+
&el2_pauth_config,
777
+
&el2_pauth_pmu_config,
907
778
};
908
779
int vcpu_configs_n = ARRAY_SIZE(vcpu_configs);
-159
tools/testing/selftests/kvm/arm64/mmio_abort.c
-159
tools/testing/selftests/kvm/arm64/mmio_abort.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
/*
3
-
* mmio_abort - Tests for userspace MMIO abort injection
4
-
*
5
-
* Copyright (c) 2024 Google LLC
6
-
*/
7
-
#include "processor.h"
8
-
#include "test_util.h"
9
-
10
-
#define MMIO_ADDR 0x8000000ULL
11
-
12
-
static u64 expected_abort_pc;
13
-
14
-
static void expect_sea_handler(struct ex_regs *regs)
15
-
{
16
-
u64 esr = read_sysreg(esr_el1);
17
-
18
-
GUEST_ASSERT_EQ(regs->pc, expected_abort_pc);
19
-
GUEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_DABT_CUR);
20
-
GUEST_ASSERT_EQ(esr & ESR_ELx_FSC_TYPE, ESR_ELx_FSC_EXTABT);
21
-
22
-
GUEST_DONE();
23
-
}
24
-
25
-
static void unexpected_dabt_handler(struct ex_regs *regs)
26
-
{
27
-
GUEST_FAIL("Unexpected data abort at PC: %lx\n", regs->pc);
28
-
}
29
-
30
-
static struct kvm_vm *vm_create_with_dabt_handler(struct kvm_vcpu **vcpu, void *guest_code,
31
-
handler_fn dabt_handler)
32
-
{
33
-
struct kvm_vm *vm = vm_create_with_one_vcpu(vcpu, guest_code);
34
-
35
-
vm_init_descriptor_tables(vm);
36
-
vcpu_init_descriptor_tables(*vcpu);
37
-
vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, ESR_ELx_EC_DABT_CUR, dabt_handler);
38
-
39
-
virt_map(vm, MMIO_ADDR, MMIO_ADDR, 1);
40
-
41
-
return vm;
42
-
}
43
-
44
-
static void vcpu_inject_extabt(struct kvm_vcpu *vcpu)
45
-
{
46
-
struct kvm_vcpu_events events = {};
47
-
48
-
events.exception.ext_dabt_pending = true;
49
-
vcpu_events_set(vcpu, &events);
50
-
}
51
-
52
-
static void vcpu_run_expect_done(struct kvm_vcpu *vcpu)
53
-
{
54
-
struct ucall uc;
55
-
56
-
vcpu_run(vcpu);
57
-
switch (get_ucall(vcpu, &uc)) {
58
-
case UCALL_ABORT:
59
-
REPORT_GUEST_ASSERT(uc);
60
-
break;
61
-
case UCALL_DONE:
62
-
break;
63
-
default:
64
-
TEST_FAIL("Unexpected ucall: %lu", uc.cmd);
65
-
}
66
-
}
67
-
68
-
extern char test_mmio_abort_insn;
69
-
70
-
static void test_mmio_abort_guest(void)
71
-
{
72
-
WRITE_ONCE(expected_abort_pc, (u64)&test_mmio_abort_insn);
73
-
74
-
asm volatile("test_mmio_abort_insn:\n\t"
75
-
"ldr x0, [%0]\n\t"
76
-
: : "r" (MMIO_ADDR) : "x0", "memory");
77
-
78
-
GUEST_FAIL("MMIO instruction should not retire");
79
-
}
80
-
81
-
/*
82
-
* Test that KVM doesn't complete MMIO emulation when userspace has made an
83
-
* external abort pending for the instruction.
84
-
*/
85
-
static void test_mmio_abort(void)
86
-
{
87
-
struct kvm_vcpu *vcpu;
88
-
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_abort_guest,
89
-
expect_sea_handler);
90
-
struct kvm_run *run = vcpu->run;
91
-
92
-
vcpu_run(vcpu);
93
-
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO);
94
-
TEST_ASSERT_EQ(run->mmio.phys_addr, MMIO_ADDR);
95
-
TEST_ASSERT_EQ(run->mmio.len, sizeof(unsigned long));
96
-
TEST_ASSERT(!run->mmio.is_write, "Expected MMIO read");
97
-
98
-
vcpu_inject_extabt(vcpu);
99
-
vcpu_run_expect_done(vcpu);
100
-
kvm_vm_free(vm);
101
-
}
102
-
103
-
extern char test_mmio_nisv_insn;
104
-
105
-
static void test_mmio_nisv_guest(void)
106
-
{
107
-
WRITE_ONCE(expected_abort_pc, (u64)&test_mmio_nisv_insn);
108
-
109
-
asm volatile("test_mmio_nisv_insn:\n\t"
110
-
"ldr x0, [%0], #8\n\t"
111
-
: : "r" (MMIO_ADDR) : "x0", "memory");
112
-
113
-
GUEST_FAIL("MMIO instruction should not retire");
114
-
}
115
-
116
-
/*
117
-
* Test that the KVM_RUN ioctl fails for ESR_EL2.ISV=0 MMIO aborts if userspace
118
-
* hasn't enabled KVM_CAP_ARM_NISV_TO_USER.
119
-
*/
120
-
static void test_mmio_nisv(void)
121
-
{
122
-
struct kvm_vcpu *vcpu;
123
-
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_nisv_guest,
124
-
unexpected_dabt_handler);
125
-
126
-
TEST_ASSERT(_vcpu_run(vcpu), "Expected nonzero return code from KVM_RUN");
127
-
TEST_ASSERT_EQ(errno, ENOSYS);
128
-
129
-
kvm_vm_free(vm);
130
-
}
131
-
132
-
/*
133
-
* Test that ESR_EL2.ISV=0 MMIO aborts reach userspace and that an injected SEA
134
-
* reaches the guest.
135
-
*/
136
-
static void test_mmio_nisv_abort(void)
137
-
{
138
-
struct kvm_vcpu *vcpu;
139
-
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_nisv_guest,
140
-
expect_sea_handler);
141
-
struct kvm_run *run = vcpu->run;
142
-
143
-
vm_enable_cap(vm, KVM_CAP_ARM_NISV_TO_USER, 1);
144
-
145
-
vcpu_run(vcpu);
146
-
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_ARM_NISV);
147
-
TEST_ASSERT_EQ(run->arm_nisv.fault_ipa, MMIO_ADDR);
148
-
149
-
vcpu_inject_extabt(vcpu);
150
-
vcpu_run_expect_done(vcpu);
151
-
kvm_vm_free(vm);
152
-
}
153
-
154
-
int main(void)
155
-
{
156
-
test_mmio_abort();
157
-
test_mmio_nisv();
158
-
test_mmio_nisv_abort();
159
-
}
+12
-2
tools/testing/selftests/kvm/arm64/set_id_regs.c
+12
-2
tools/testing/selftests/kvm/arm64/set_id_regs.c
···
139
139
};
140
140
141
141
static const struct reg_ftr_bits ftr_id_aa64pfr1_el1[] = {
142
+
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR1_EL1, DF2, 0),
142
143
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR1_EL1, CSV2_frac, 0),
143
144
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR1_EL1, SSBS, ID_AA64PFR1_EL1_SSBS_NI),
144
145
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR1_EL1, BT, 0),
···
188
187
REG_FTR_END,
189
188
};
190
189
190
+
static const struct reg_ftr_bits ftr_id_aa64mmfr3_el1[] = {
191
+
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR3_EL1, S1POE, 0),
192
+
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR3_EL1, S1PIE, 0),
193
+
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR3_EL1, SCTLRX, 0),
194
+
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR3_EL1, TCRX, 0),
195
+
REG_FTR_END,
196
+
};
197
+
191
198
static const struct reg_ftr_bits ftr_id_aa64zfr0_el1[] = {
192
199
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64ZFR0_EL1, F64MM, 0),
193
200
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64ZFR0_EL1, F32MM, 0),
···
226
217
TEST_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0_el1),
227
218
TEST_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1_el1),
228
219
TEST_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2_el1),
220
+
TEST_REG(SYS_ID_AA64MMFR3_EL1, ftr_id_aa64mmfr3_el1),
229
221
TEST_REG(SYS_ID_AA64ZFR0_EL1, ftr_id_aa64zfr0_el1),
230
222
};
231
223
···
784
774
ARRAY_SIZE(ftr_id_aa64isar2_el1) + ARRAY_SIZE(ftr_id_aa64pfr0_el1) +
785
775
ARRAY_SIZE(ftr_id_aa64pfr1_el1) + ARRAY_SIZE(ftr_id_aa64mmfr0_el1) +
786
776
ARRAY_SIZE(ftr_id_aa64mmfr1_el1) + ARRAY_SIZE(ftr_id_aa64mmfr2_el1) +
787
-
ARRAY_SIZE(ftr_id_aa64zfr0_el1) - ARRAY_SIZE(test_regs) + 3 +
788
-
MPAM_IDREG_TEST + MTE_IDREG_TEST;
777
+
ARRAY_SIZE(ftr_id_aa64mmfr3_el1) + ARRAY_SIZE(ftr_id_aa64zfr0_el1) -
778
+
ARRAY_SIZE(test_regs) + 3 + MPAM_IDREG_TEST + MTE_IDREG_TEST;
789
779
790
780
ksft_set_plan(test_cnt);
791
781
+257
-2
tools/testing/selftests/kvm/arm64/vgic_init.c
+257
-2
tools/testing/selftests/kvm/arm64/vgic_init.c
···
9
9
#include <asm/kvm.h>
10
10
#include <asm/kvm_para.h>
11
11
12
+
#include <arm64/gic_v3.h>
13
+
12
14
#include "test_util.h"
13
15
#include "kvm_util.h"
14
16
#include "processor.h"
15
17
#include "vgic.h"
18
+
#include "gic_v3.h"
16
19
17
20
#define NR_VCPUS 4
18
21
19
22
#define REG_OFFSET(vcpu, offset) (((uint64_t)vcpu << 32) | offset)
20
-
21
-
#define GICR_TYPER 0x8
22
23
23
24
#define VGIC_DEV_IS_V2(_d) ((_d) == KVM_DEV_TYPE_ARM_VGIC_V2)
24
25
#define VGIC_DEV_IS_V3(_d) ((_d) == KVM_DEV_TYPE_ARM_VGIC_V3)
···
676
675
vm_gic_destroy(&v);
677
676
}
678
677
678
+
static void test_v3_nassgicap(void)
679
+
{
680
+
struct kvm_vcpu *vcpus[NR_VCPUS];
681
+
bool has_nassgicap;
682
+
struct vm_gic vm;
683
+
u32 typer2;
684
+
int ret;
685
+
686
+
vm = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus);
687
+
kvm_device_attr_get(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
688
+
GICD_TYPER2, &typer2);
689
+
has_nassgicap = typer2 & GICD_TYPER2_nASSGIcap;
690
+
691
+
typer2 |= GICD_TYPER2_nASSGIcap;
692
+
ret = __kvm_device_attr_set(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
693
+
GICD_TYPER2, &typer2);
694
+
if (has_nassgicap)
695
+
TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_DEVICE_ATTR_SET, ret));
696
+
else
697
+
TEST_ASSERT(ret && errno == EINVAL,
698
+
"Enabled nASSGIcap even though it's unavailable");
699
+
700
+
typer2 &= ~GICD_TYPER2_nASSGIcap;
701
+
kvm_device_attr_set(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
702
+
GICD_TYPER2, &typer2);
703
+
704
+
kvm_device_attr_set(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
705
+
KVM_DEV_ARM_VGIC_CTRL_INIT, NULL);
706
+
707
+
typer2 ^= GICD_TYPER2_nASSGIcap;
708
+
ret = __kvm_device_attr_set(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
709
+
GICD_TYPER2, &typer2);
710
+
TEST_ASSERT(ret && errno == EBUSY,
711
+
"Changed nASSGIcap after initializing the VGIC");
712
+
713
+
vm_gic_destroy(&vm);
714
+
}
715
+
679
716
/*
680
717
* Returns 0 if it's possible to create GIC device of a given type (V2 or V3).
681
718
*/
···
754
715
return 0;
755
716
}
756
717
718
+
struct sr_def {
719
+
const char *name;
720
+
u32 encoding;
721
+
};
722
+
723
+
#define PACK_SR(r) \
724
+
((sys_reg_Op0(r) << 14) | \
725
+
(sys_reg_Op1(r) << 11) | \
726
+
(sys_reg_CRn(r) << 7) | \
727
+
(sys_reg_CRm(r) << 3) | \
728
+
(sys_reg_Op2(r)))
729
+
730
+
#define SR(r) \
731
+
{ \
732
+
.name = #r, \
733
+
.encoding = r, \
734
+
}
735
+
736
+
static const struct sr_def sysregs_el1[] = {
737
+
SR(SYS_ICC_PMR_EL1),
738
+
SR(SYS_ICC_BPR0_EL1),
739
+
SR(SYS_ICC_AP0R0_EL1),
740
+
SR(SYS_ICC_AP0R1_EL1),
741
+
SR(SYS_ICC_AP0R2_EL1),
742
+
SR(SYS_ICC_AP0R3_EL1),
743
+
SR(SYS_ICC_AP1R0_EL1),
744
+
SR(SYS_ICC_AP1R1_EL1),
745
+
SR(SYS_ICC_AP1R2_EL1),
746
+
SR(SYS_ICC_AP1R3_EL1),
747
+
SR(SYS_ICC_BPR1_EL1),
748
+
SR(SYS_ICC_CTLR_EL1),
749
+
SR(SYS_ICC_SRE_EL1),
750
+
SR(SYS_ICC_IGRPEN0_EL1),
751
+
SR(SYS_ICC_IGRPEN1_EL1),
752
+
};
753
+
754
+
static const struct sr_def sysregs_el2[] = {
755
+
SR(SYS_ICH_AP0R0_EL2),
756
+
SR(SYS_ICH_AP0R1_EL2),
757
+
SR(SYS_ICH_AP0R2_EL2),
758
+
SR(SYS_ICH_AP0R3_EL2),
759
+
SR(SYS_ICH_AP1R0_EL2),
760
+
SR(SYS_ICH_AP1R1_EL2),
761
+
SR(SYS_ICH_AP1R2_EL2),
762
+
SR(SYS_ICH_AP1R3_EL2),
763
+
SR(SYS_ICH_HCR_EL2),
764
+
SR(SYS_ICC_SRE_EL2),
765
+
SR(SYS_ICH_VTR_EL2),
766
+
SR(SYS_ICH_VMCR_EL2),
767
+
SR(SYS_ICH_LR0_EL2),
768
+
SR(SYS_ICH_LR1_EL2),
769
+
SR(SYS_ICH_LR2_EL2),
770
+
SR(SYS_ICH_LR3_EL2),
771
+
SR(SYS_ICH_LR4_EL2),
772
+
SR(SYS_ICH_LR5_EL2),
773
+
SR(SYS_ICH_LR6_EL2),
774
+
SR(SYS_ICH_LR7_EL2),
775
+
SR(SYS_ICH_LR8_EL2),
776
+
SR(SYS_ICH_LR9_EL2),
777
+
SR(SYS_ICH_LR10_EL2),
778
+
SR(SYS_ICH_LR11_EL2),
779
+
SR(SYS_ICH_LR12_EL2),
780
+
SR(SYS_ICH_LR13_EL2),
781
+
SR(SYS_ICH_LR14_EL2),
782
+
SR(SYS_ICH_LR15_EL2),
783
+
};
784
+
785
+
static void test_sysreg_array(int gic, const struct sr_def *sr, int nr,
786
+
int (*check)(int, const struct sr_def *, const char *))
787
+
{
788
+
for (int i = 0; i < nr; i++) {
789
+
u64 val;
790
+
u64 attr;
791
+
int ret;
792
+
793
+
/* Assume MPIDR_EL1.Aff*=0 */
794
+
attr = PACK_SR(sr[i].encoding);
795
+
796
+
/*
797
+
* The API is braindead. A register can be advertised as
798
+
* available, and yet not be readable or writable.
799
+
* ICC_APnR{1,2,3}_EL1 are examples of such non-sense, and
800
+
* ICH_APnR{1,2,3}_EL2 do follow suit for consistency.
801
+
*
802
+
* On the bright side, no known HW is implementing more than
803
+
* 5 bits of priority, so we're safe. Sort of...
804
+
*/
805
+
ret = __kvm_has_device_attr(gic, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
806
+
attr);
807
+
TEST_ASSERT(ret == 0, "%s unavailable", sr[i].name);
808
+
809
+
/* Check that we can write back what we read */
810
+
ret = __kvm_device_attr_get(gic, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
811
+
attr, &val);
812
+
TEST_ASSERT(ret == 0 || !check(gic, &sr[i], "read"), "%s unreadable", sr[i].name);
813
+
ret = __kvm_device_attr_set(gic, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
814
+
attr, &val);
815
+
TEST_ASSERT(ret == 0 || !check(gic, &sr[i], "write"), "%s unwritable", sr[i].name);
816
+
}
817
+
}
818
+
819
+
static u8 get_ctlr_pribits(int gic)
820
+
{
821
+
int ret;
822
+
u64 val;
823
+
u8 pri;
824
+
825
+
ret = __kvm_device_attr_get(gic, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
826
+
PACK_SR(SYS_ICC_CTLR_EL1), &val);
827
+
TEST_ASSERT(ret == 0, "ICC_CTLR_EL1 unreadable");
828
+
829
+
pri = FIELD_GET(ICC_CTLR_EL1_PRI_BITS_MASK, val) + 1;
830
+
TEST_ASSERT(pri >= 5 && pri <= 7, "Bad pribits %d", pri);
831
+
832
+
return pri;
833
+
}
834
+
835
+
static int check_unaccessible_el1_regs(int gic, const struct sr_def *sr, const char *what)
836
+
{
837
+
switch (sr->encoding) {
838
+
case SYS_ICC_AP0R1_EL1:
839
+
case SYS_ICC_AP1R1_EL1:
840
+
if (get_ctlr_pribits(gic) >= 6)
841
+
return -EINVAL;
842
+
break;
843
+
case SYS_ICC_AP0R2_EL1:
844
+
case SYS_ICC_AP0R3_EL1:
845
+
case SYS_ICC_AP1R2_EL1:
846
+
case SYS_ICC_AP1R3_EL1:
847
+
if (get_ctlr_pribits(gic) == 7)
848
+
return 0;
849
+
break;
850
+
default:
851
+
return -EINVAL;
852
+
}
853
+
854
+
pr_info("SKIP %s for %s\n", sr->name, what);
855
+
return 0;
856
+
}
857
+
858
+
static u8 get_vtr_pribits(int gic)
859
+
{
860
+
int ret;
861
+
u64 val;
862
+
u8 pri;
863
+
864
+
ret = __kvm_device_attr_get(gic, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
865
+
PACK_SR(SYS_ICH_VTR_EL2), &val);
866
+
TEST_ASSERT(ret == 0, "ICH_VTR_EL2 unreadable");
867
+
868
+
pri = FIELD_GET(ICH_VTR_EL2_PRIbits, val) + 1;
869
+
TEST_ASSERT(pri >= 5 && pri <= 7, "Bad pribits %d", pri);
870
+
871
+
return pri;
872
+
}
873
+
874
+
static int check_unaccessible_el2_regs(int gic, const struct sr_def *sr, const char *what)
875
+
{
876
+
switch (sr->encoding) {
877
+
case SYS_ICH_AP0R1_EL2:
878
+
case SYS_ICH_AP1R1_EL2:
879
+
if (get_vtr_pribits(gic) >= 6)
880
+
return -EINVAL;
881
+
break;
882
+
case SYS_ICH_AP0R2_EL2:
883
+
case SYS_ICH_AP0R3_EL2:
884
+
case SYS_ICH_AP1R2_EL2:
885
+
case SYS_ICH_AP1R3_EL2:
886
+
if (get_vtr_pribits(gic) == 7)
887
+
return -EINVAL;
888
+
break;
889
+
default:
890
+
return -EINVAL;
891
+
}
892
+
893
+
pr_info("SKIP %s for %s\n", sr->name, what);
894
+
return 0;
895
+
}
896
+
897
+
static void test_v3_sysregs(void)
898
+
{
899
+
struct kvm_vcpu_init init = {};
900
+
struct kvm_vcpu *vcpu;
901
+
struct kvm_vm *vm;
902
+
u32 feat = 0;
903
+
int gic;
904
+
905
+
if (kvm_check_cap(KVM_CAP_ARM_EL2))
906
+
feat |= BIT(KVM_ARM_VCPU_HAS_EL2);
907
+
908
+
vm = vm_create(1);
909
+
910
+
vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init);
911
+
init.features[0] |= feat;
912
+
913
+
vcpu = aarch64_vcpu_add(vm, 0, &init, NULL);
914
+
TEST_ASSERT(vcpu, "Can't create a vcpu?");
915
+
916
+
gic = kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V3);
917
+
TEST_ASSERT(gic >= 0, "No GIC???");
918
+
919
+
kvm_device_attr_set(gic, KVM_DEV_ARM_VGIC_GRP_CTRL,
920
+
KVM_DEV_ARM_VGIC_CTRL_INIT, NULL);
921
+
922
+
test_sysreg_array(gic, sysregs_el1, ARRAY_SIZE(sysregs_el1), check_unaccessible_el1_regs);
923
+
if (feat)
924
+
test_sysreg_array(gic, sysregs_el2, ARRAY_SIZE(sysregs_el2), check_unaccessible_el2_regs);
925
+
else
926
+
pr_info("SKIP EL2 registers, not available\n");
927
+
928
+
close(gic);
929
+
kvm_vm_free(vm);
930
+
}
931
+
757
932
void run_tests(uint32_t gic_dev_type)
758
933
{
759
934
test_vcpus_then_vgic(gic_dev_type);
···
983
730
test_v3_last_bit_single_rdist();
984
731
test_v3_redist_ipa_range_check_at_vcpu_run();
985
732
test_v3_its_region();
733
+
test_v3_sysregs();
734
+
test_v3_nassgicap();
986
735
}
987
736
}
988
737
+10
tools/testing/selftests/kvm/include/arm64/processor.h
+10
tools/testing/selftests/kvm/include/arm64/processor.h
···
254
254
asm volatile("msr daifset, #3" : : : "memory");
255
255
}
256
256
257
+
static inline void local_serror_enable(void)
258
+
{
259
+
asm volatile("msr daifclr, #4" : : : "memory");
260
+
}
261
+
262
+
static inline void local_serror_disable(void)
263
+
{
264
+
asm volatile("msr daifset, #4" : : : "memory");
265
+
}
266
+
257
267
/**
258
268
* struct arm_smccc_res - Result from SMC/HVC call
259
269
* @a0-a3 result values from registers 0 to 3