Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
irqchip/gic-v5: Add GICv5 LPI/IPI support
An IRS supports Logical Peripheral Interrupts (LPIs) and implement
Linux IPIs on top of it.
LPIs are used for interrupt signals that are translated by a
GICv5 ITS (Interrupt Translation Service) but also for software
generated IRQs - namely interrupts that are not driven by a HW
signal, ie IPIs.
LPIs rely on memory storage for interrupt routing and state.
LPIs state and routing information is kept in the Interrupt
State Table (IST).
IRSes provide support for 1- or 2-level IST tables configured
to support a maximum number of interrupts that depend on the
OS configuration and the HW capabilities.
On systems that provide 2-level IST support, always allow
the maximum number of LPIs; On systems with only 1-level
support, limit the number of LPIs to 2^12 to prevent
wasting memory (presumably a system that supports a 1-level
only IST is not expecting a large number of interrupts).
On a 2-level IST system, L2 entries are allocated on
demand.
The IST table memory is allocated using the kmalloc() interface;
the allocation required may be smaller than a page and must be
made up of contiguous physical pages if larger than a page.
On systems where the IRS is not cache-coherent with the CPUs,
cache mainteinance operations are executed to clean and
invalidate the allocated memory to the point of coherency
making it visible to the IRS components.
On GICv5 systems, IPIs are implemented using LPIs.
Add an LPI IRQ domain and implement an IPI-specific IRQ domain created
as a child/subdomain of the LPI domain to allocate the required number
of LPIs needed to implement the IPIs.
IPIs are backed by LPIs, add LPIs allocation/de-allocation
functions.
The LPI INTID namespace is managed using an IDA to alloc/free LPI INTIDs.
Associate an IPI irqchip with IPI IRQ descriptors to provide
core code with the irqchip.ipi_send_single() method required
to raise an IPI.
Co-developed-by: Sascha Bischoff <sascha.bischoff@arm.com>
Signed-off-by: Sascha Bischoff <sascha.bischoff@arm.com>
Co-developed-by: Timothy Hayes <timothy.hayes@arm.com>
Signed-off-by: Timothy Hayes <timothy.hayes@arm.com>
Signed-off-by: Lorenzo Pieralisi <lpieralisi@kernel.org>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Cc: Will Deacon <will@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20250703-gicv5-host-v7-22-12e71f1b3528@kernel.org
Signed-off-by: Marc Zyngier <maz@kernel.org>
authored by
Lorenzo Pieralisi
and committed by
Marc Zyngier
0f010132
5cb1b6da
+746
-20
+17
arch/arm64/include/asm/smp.h
+17
arch/arm64/include/asm/smp.h
···
50
50
*/
51
51
extern void smp_init_cpus(void);
52
52
53
+
enum ipi_msg_type {
54
+
IPI_RESCHEDULE,
55
+
IPI_CALL_FUNC,
56
+
IPI_CPU_STOP,
57
+
IPI_CPU_STOP_NMI,
58
+
IPI_TIMER,
59
+
IPI_IRQ_WORK,
60
+
NR_IPI,
61
+
/*
62
+
* Any enum >= NR_IPI and < MAX_IPI is special and not tracable
63
+
* with trace_ipi_*
64
+
*/
65
+
IPI_CPU_BACKTRACE = NR_IPI,
66
+
IPI_KGDB_ROUNDUP,
67
+
MAX_IPI
68
+
};
69
+
53
70
/*
54
71
* Register IPI interrupts with the arch SMP code
55
72
*/
+6
arch/arm64/include/asm/sysreg.h
+6
arch/arm64/include/asm/sysreg.h
···
1088
1088
#define GICV5_OP_GIC_CDAFF sys_insn(1, 0, 12, 1, 3)
1089
1089
#define GICV5_OP_GIC_CDDI sys_insn(1, 0, 12, 2, 0)
1090
1090
#define GICV5_OP_GIC_CDDIS sys_insn(1, 0, 12, 1, 0)
1091
+
#define GICV5_OP_GIC_CDHM sys_insn(1, 0, 12, 2, 1)
1091
1092
#define GICV5_OP_GIC_CDEN sys_insn(1, 0, 12, 1, 1)
1092
1093
#define GICV5_OP_GIC_CDEOI sys_insn(1, 0, 12, 1, 7)
1093
1094
#define GICV5_OP_GIC_CDPEND sys_insn(1, 0, 12, 1, 4)
···
1115
1114
/* Definitions for GIC CDEN */
1116
1115
#define GICV5_GIC_CDEN_TYPE_MASK GENMASK_ULL(31, 29)
1117
1116
#define GICV5_GIC_CDEN_ID_MASK GENMASK_ULL(23, 0)
1117
+
1118
+
/* Definitions for GIC CDHM */
1119
+
#define GICV5_GIC_CDHM_HM_MASK BIT_ULL(32)
1120
+
#define GICV5_GIC_CDHM_TYPE_MASK GENMASK_ULL(31, 29)
1121
+
#define GICV5_GIC_CDHM_ID_MASK GENMASK_ULL(23, 0)
1118
1122
1119
1123
/* Definitions for GIC CDPEND */
1120
1124
#define GICV5_GIC_CDPEND_PENDING_MASK BIT_ULL(32)
-17
arch/arm64/kernel/smp.c
-17
arch/arm64/kernel/smp.c
···
64
64
/* Number of CPUs which aren't online, but looping in kernel text. */
65
65
static int cpus_stuck_in_kernel;
66
66
67
-
enum ipi_msg_type {
68
-
IPI_RESCHEDULE,
69
-
IPI_CALL_FUNC,
70
-
IPI_CPU_STOP,
71
-
IPI_CPU_STOP_NMI,
72
-
IPI_TIMER,
73
-
IPI_IRQ_WORK,
74
-
NR_IPI,
75
-
/*
76
-
* Any enum >= NR_IPI and < MAX_IPI is special and not tracable
77
-
* with trace_ipi_*
78
-
*/
79
-
IPI_CPU_BACKTRACE = NR_IPI,
80
-
IPI_KGDB_ROUNDUP,
81
-
MAX_IPI
82
-
};
83
-
84
67
static int ipi_irq_base __ro_after_init;
85
68
static int nr_ipi __ro_after_init = NR_IPI;
86
69
+364
drivers/irqchip/irq-gic-v5-irs.c
+364
drivers/irqchip/irq-gic-v5-irs.c
···
5
5
6
6
#define pr_fmt(fmt) "GICv5 IRS: " fmt
7
7
8
+
#include <linux/log2.h>
8
9
#include <linux/of.h>
9
10
#include <linux/of_address.h>
10
11
11
12
#include <linux/irqchip.h>
12
13
#include <linux/irqchip/arm-gic-v5.h>
14
+
15
+
/*
16
+
* Hardcoded ID_BITS limit for systems supporting only a 1-level IST
17
+
* table. Systems supporting only a 1-level IST table aren't expected
18
+
* to require more than 2^12 LPIs. Tweak as required.
19
+
*/
20
+
#define LPI_ID_BITS_LINEAR 12
13
21
14
22
#define IRS_FLAGS_NON_COHERENT BIT(0)
15
23
···
34
26
const u32 val, const u32 reg_offset)
35
27
{
36
28
writel_relaxed(val, irs_data->irs_base + reg_offset);
29
+
}
30
+
31
+
static u64 irs_readq_relaxed(struct gicv5_irs_chip_data *irs_data,
32
+
const u32 reg_offset)
33
+
{
34
+
return readq_relaxed(irs_data->irs_base + reg_offset);
35
+
}
36
+
37
+
static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data,
38
+
const u64 val, const u32 reg_offset)
39
+
{
40
+
writeq_relaxed(val, irs_data->irs_base + reg_offset);
41
+
}
42
+
43
+
/*
44
+
* The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register
45
+
* provides the memory barriers (through MMIO accessors)
46
+
* required to synchronize CPU and GIC access to IST memory.
47
+
*/
48
+
static int gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data *irs_data)
49
+
{
50
+
return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_IST_STATUSR,
51
+
GICV5_IRS_IST_STATUSR_IDLE, NULL);
52
+
}
53
+
54
+
static int __init gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data *irs_data,
55
+
unsigned int lpi_id_bits,
56
+
unsigned int istsz)
57
+
{
58
+
size_t l2istsz;
59
+
u32 n, cfgr;
60
+
void *ist;
61
+
u64 baser;
62
+
int ret;
63
+
64
+
/* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
65
+
n = max(5, lpi_id_bits + 1 + istsz);
66
+
67
+
l2istsz = BIT(n + 1);
68
+
/*
69
+
* Check memory requirements. For a linear IST we cap the
70
+
* number of ID bits to a value that should never exceed
71
+
* kmalloc interface memory allocation limits, so this
72
+
* check is really belt and braces.
73
+
*/
74
+
if (l2istsz > KMALLOC_MAX_SIZE) {
75
+
u8 lpi_id_cap = ilog2(KMALLOC_MAX_SIZE) - 2 + istsz;
76
+
77
+
pr_warn("Limiting LPI ID bits from %u to %u\n",
78
+
lpi_id_bits, lpi_id_cap);
79
+
lpi_id_bits = lpi_id_cap;
80
+
l2istsz = KMALLOC_MAX_SIZE;
81
+
}
82
+
83
+
ist = kzalloc(l2istsz, GFP_KERNEL);
84
+
if (!ist)
85
+
return -ENOMEM;
86
+
87
+
if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
88
+
dcache_clean_inval_poc((unsigned long)ist,
89
+
(unsigned long)ist + l2istsz);
90
+
else
91
+
dsb(ishst);
92
+
93
+
cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
94
+
GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR) |
95
+
FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) |
96
+
FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ,
97
+
GICV5_IRS_IST_CFGR_L2SZ_4K) |
98
+
FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
99
+
irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
100
+
101
+
gicv5_global_data.ist.l2 = false;
102
+
103
+
baser = (virt_to_phys(ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
104
+
FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
105
+
irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
106
+
107
+
ret = gicv5_irs_ist_synchronise(irs_data);
108
+
if (ret) {
109
+
kfree(ist);
110
+
return ret;
111
+
}
112
+
113
+
return 0;
114
+
}
115
+
116
+
static int __init gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data *irs_data,
117
+
unsigned int lpi_id_bits,
118
+
unsigned int istsz,
119
+
unsigned int l2sz)
120
+
{
121
+
__le64 *l1ist;
122
+
u32 cfgr, n;
123
+
size_t l1sz;
124
+
u64 baser;
125
+
int ret;
126
+
127
+
/* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
128
+
n = max(5, lpi_id_bits - ((10 - istsz) + (2 * l2sz)) + 2);
129
+
130
+
l1sz = BIT(n + 1);
131
+
132
+
l1ist = kzalloc(l1sz, GFP_KERNEL);
133
+
if (!l1ist)
134
+
return -ENOMEM;
135
+
136
+
if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
137
+
dcache_clean_inval_poc((unsigned long)l1ist,
138
+
(unsigned long)l1ist + l1sz);
139
+
else
140
+
dsb(ishst);
141
+
142
+
cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
143
+
GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL) |
144
+
FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) |
145
+
FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, l2sz) |
146
+
FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
147
+
irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
148
+
149
+
/*
150
+
* The L2SZ determine bits required at L2 level. Number of bytes
151
+
* required by metadata is reported through istsz - the number of bits
152
+
* covered by L2 entries scales accordingly.
153
+
*/
154
+
gicv5_global_data.ist.l2_size = BIT(11 + (2 * l2sz) + 1);
155
+
gicv5_global_data.ist.l2_bits = (10 - istsz) + (2 * l2sz);
156
+
gicv5_global_data.ist.l1ist_addr = l1ist;
157
+
gicv5_global_data.ist.l2 = true;
158
+
159
+
baser = (virt_to_phys(l1ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
160
+
FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
161
+
irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
162
+
163
+
ret = gicv5_irs_ist_synchronise(irs_data);
164
+
if (ret) {
165
+
kfree(l1ist);
166
+
return ret;
167
+
}
168
+
169
+
return 0;
170
+
}
171
+
172
+
/*
173
+
* Alloc L2 IST entries on demand.
174
+
*
175
+
* Locking/serialization is guaranteed by irqdomain core code by
176
+
* taking the hierarchical domain struct irq_domain.root->mutex.
177
+
*/
178
+
int gicv5_irs_iste_alloc(const u32 lpi)
179
+
{
180
+
struct gicv5_irs_chip_data *irs_data;
181
+
unsigned int index;
182
+
u32 l2istr, l2bits;
183
+
__le64 *l1ist;
184
+
size_t l2size;
185
+
void *l2ist;
186
+
int ret;
187
+
188
+
if (!gicv5_global_data.ist.l2)
189
+
return 0;
190
+
191
+
irs_data = per_cpu(per_cpu_irs_data, smp_processor_id());
192
+
if (!irs_data)
193
+
return -ENOENT;
194
+
195
+
l2size = gicv5_global_data.ist.l2_size;
196
+
l2bits = gicv5_global_data.ist.l2_bits;
197
+
l1ist = gicv5_global_data.ist.l1ist_addr;
198
+
index = lpi >> l2bits;
199
+
200
+
if (FIELD_GET(GICV5_ISTL1E_VALID, le64_to_cpu(l1ist[index])))
201
+
return 0;
202
+
203
+
l2ist = kzalloc(l2size, GFP_KERNEL);
204
+
if (!l2ist)
205
+
return -ENOMEM;
206
+
207
+
l1ist[index] = cpu_to_le64(virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK);
208
+
209
+
if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
210
+
dcache_clean_inval_poc((unsigned long)l2ist,
211
+
(unsigned long)l2ist + l2size);
212
+
dcache_clean_poc((unsigned long)(l1ist + index),
213
+
(unsigned long)(l1ist + index) + sizeof(*l1ist));
214
+
} else {
215
+
dsb(ishst);
216
+
}
217
+
218
+
l2istr = FIELD_PREP(GICV5_IRS_MAP_L2_ISTR_ID, lpi);
219
+
irs_writel_relaxed(irs_data, l2istr, GICV5_IRS_MAP_L2_ISTR);
220
+
221
+
ret = gicv5_irs_ist_synchronise(irs_data);
222
+
if (ret) {
223
+
l1ist[index] = 0;
224
+
kfree(l2ist);
225
+
return ret;
226
+
}
227
+
228
+
/*
229
+
* Make sure we invalidate the cache line pulled before the IRS
230
+
* had a chance to update the L1 entry and mark it valid.
231
+
*/
232
+
if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
233
+
/*
234
+
* gicv5_irs_ist_synchronise() includes memory
235
+
* barriers (MMIO accessors) required to guarantee that the
236
+
* following dcache invalidation is not executed before the
237
+
* IST mapping operation has completed.
238
+
*/
239
+
dcache_inval_poc((unsigned long)(l1ist + index),
240
+
(unsigned long)(l1ist + index) + sizeof(*l1ist));
241
+
}
242
+
243
+
return 0;
244
+
}
245
+
246
+
/*
247
+
* Try to match the L2 IST size to the pagesize, and if this is not possible
248
+
* pick the smallest supported L2 size in order to minimise the requirement for
249
+
* physically contiguous blocks of memory as page-sized allocations are
250
+
* guaranteed to be physically contiguous, and are by definition the easiest to
251
+
* find.
252
+
*
253
+
* Fall back to the smallest supported size (in the event that the pagesize
254
+
* itself is not supported) again serves to make it easier to find physically
255
+
* contiguous blocks of memory.
256
+
*/
257
+
static unsigned int gicv5_irs_l2_sz(u32 idr2)
258
+
{
259
+
switch (PAGE_SIZE) {
260
+
case SZ_64K:
261
+
if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2))
262
+
return GICV5_IRS_IST_CFGR_L2SZ_64K;
263
+
fallthrough;
264
+
case SZ_4K:
265
+
if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
266
+
return GICV5_IRS_IST_CFGR_L2SZ_4K;
267
+
fallthrough;
268
+
case SZ_16K:
269
+
if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2))
270
+
return GICV5_IRS_IST_CFGR_L2SZ_16K;
271
+
break;
272
+
}
273
+
274
+
if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
275
+
return GICV5_IRS_IST_CFGR_L2SZ_4K;
276
+
277
+
return GICV5_IRS_IST_CFGR_L2SZ_64K;
278
+
}
279
+
280
+
static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data)
281
+
{
282
+
u32 lpi_id_bits, idr2_id_bits, idr2_min_lpi_id_bits, l2_iste_sz, l2sz;
283
+
u32 l2_iste_sz_split, idr2;
284
+
bool two_levels, istmd;
285
+
u64 baser;
286
+
int ret;
287
+
288
+
baser = irs_readq_relaxed(irs_data, GICV5_IRS_IST_BASER);
289
+
if (FIELD_GET(GICV5_IRS_IST_BASER_VALID, baser)) {
290
+
pr_err("IST is marked as valid already; cannot allocate\n");
291
+
return -EPERM;
292
+
}
293
+
294
+
idr2 = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
295
+
two_levels = !!FIELD_GET(GICV5_IRS_IDR2_IST_LEVELS, idr2);
296
+
297
+
idr2_id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2);
298
+
idr2_min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2);
299
+
300
+
/*
301
+
* For two level tables we are always supporting the maximum allowed
302
+
* number of IDs.
303
+
*
304
+
* For 1-level tables, we should support a number of bits that
305
+
* is >= min_lpi_id_bits but cap it to LPI_ID_BITS_LINEAR lest
306
+
* the level 1-table gets too large and its memory allocation
307
+
* may fail.
308
+
*/
309
+
if (two_levels) {
310
+
lpi_id_bits = idr2_id_bits;
311
+
} else {
312
+
lpi_id_bits = max(LPI_ID_BITS_LINEAR, idr2_min_lpi_id_bits);
313
+
lpi_id_bits = min(lpi_id_bits, idr2_id_bits);
314
+
}
315
+
316
+
/*
317
+
* Cap the ID bits according to the CPUIF supported ID bits
318
+
*/
319
+
lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits);
320
+
321
+
if (two_levels)
322
+
l2sz = gicv5_irs_l2_sz(idr2);
323
+
324
+
istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2);
325
+
326
+
l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_4;
327
+
328
+
if (istmd) {
329
+
l2_iste_sz_split = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2);
330
+
331
+
if (lpi_id_bits < l2_iste_sz_split)
332
+
l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_8;
333
+
else
334
+
l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_16;
335
+
}
336
+
337
+
/*
338
+
* Follow GICv5 specification recommendation to opt in for two
339
+
* level tables (ref: 10.2.1.14 IRS_IST_CFGR).
340
+
*/
341
+
if (two_levels && (lpi_id_bits > ((10 - l2_iste_sz) + (2 * l2sz)))) {
342
+
ret = gicv5_irs_init_ist_two_level(irs_data, lpi_id_bits,
343
+
l2_iste_sz, l2sz);
344
+
} else {
345
+
ret = gicv5_irs_init_ist_linear(irs_data, lpi_id_bits,
346
+
l2_iste_sz);
347
+
}
348
+
if (ret)
349
+
return ret;
350
+
351
+
gicv5_init_lpis(BIT(lpi_id_bits));
352
+
353
+
return 0;
37
354
}
38
355
39
356
struct iaffid_entry {
···
695
362
goto out_iomem;
696
363
}
697
364
365
+
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
366
+
if (WARN(!FIELD_GET(GICV5_IRS_IDR2_LPI, idr),
367
+
"LPI support not available - no IPIs, can't proceed\n")) {
368
+
ret = -ENODEV;
369
+
goto out_iomem;
370
+
}
371
+
698
372
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR7);
699
373
irs_data->spi_min = FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, idr);
700
374
···
731
391
spi_count = FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, idr);
732
392
gicv5_global_data.global_spi_count = spi_count;
733
393
394
+
gicv5_init_lpi_domain();
395
+
734
396
pr_debug("Detected %u SPIs globally\n", spi_count);
735
397
}
736
398
···
751
409
{
752
410
struct gicv5_irs_chip_data *irs_data, *tmp_data;
753
411
412
+
gicv5_free_lpi_domain();
413
+
gicv5_deinit_lpis();
414
+
754
415
list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) {
755
416
iounmap(irs_data->irs_base);
756
417
list_del(&irs_data->entry);
757
418
kfree(irs_data);
758
419
}
420
+
}
421
+
422
+
int __init gicv5_irs_enable(void)
423
+
{
424
+
struct gicv5_irs_chip_data *irs_data;
425
+
int ret;
426
+
427
+
irs_data = list_first_entry_or_null(&irs_nodes,
428
+
struct gicv5_irs_chip_data, entry);
429
+
if (!irs_data)
430
+
return -ENODEV;
431
+
432
+
ret = gicv5_irs_init_ist(irs_data);
433
+
if (ret) {
434
+
pr_err("Failed to init IST\n");
435
+
return ret;
436
+
}
437
+
438
+
return 0;
759
439
}
760
440
761
441
int __init gicv5_irs_of_probe(struct device_node *parent)
+297
-2
drivers/irqchip/irq-gic-v5.c
+297
-2
drivers/irqchip/irq-gic-v5.c
···
5
5
6
6
#define pr_fmt(fmt) "GICv5: " fmt
7
7
8
+
#include <linux/idr.h>
8
9
#include <linux/irqdomain.h>
10
+
#include <linux/slab.h>
9
11
#include <linux/wordpart.h>
10
12
11
13
#include <linux/irqchip.h>
···
29
27
}
30
28
31
29
struct gicv5_chip_data gicv5_global_data __read_mostly;
30
+
31
+
static DEFINE_IDA(lpi_ida);
32
+
static u32 num_lpis __ro_after_init;
33
+
34
+
void __init gicv5_init_lpis(u32 lpis)
35
+
{
36
+
num_lpis = lpis;
37
+
}
38
+
39
+
void __init gicv5_deinit_lpis(void)
40
+
{
41
+
num_lpis = 0;
42
+
}
43
+
44
+
static int alloc_lpi(void)
45
+
{
46
+
if (!num_lpis)
47
+
return -ENOSPC;
48
+
49
+
return ida_alloc_max(&lpi_ida, num_lpis - 1, GFP_KERNEL);
50
+
}
51
+
52
+
static void release_lpi(u32 lpi)
53
+
{
54
+
ida_free(&lpi_ida, lpi);
55
+
}
56
+
57
+
int gicv5_alloc_lpi(void)
58
+
{
59
+
return alloc_lpi();
60
+
}
61
+
62
+
void gicv5_free_lpi(u32 lpi)
63
+
{
64
+
release_lpi(lpi);
65
+
}
32
66
33
67
static void gicv5_ppi_priority_init(void)
34
68
{
···
98
60
u16 iaffid;
99
61
int ret;
100
62
101
-
if (hwirq_type == GICV5_HWIRQ_TYPE_SPI) {
63
+
if (hwirq_type == GICV5_HWIRQ_TYPE_LPI || hwirq_type == GICV5_HWIRQ_TYPE_SPI) {
102
64
cdpri = FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, priority) |
103
65
FIELD_PREP(GICV5_GIC_CDPRI_TYPE_MASK, hwirq_type) |
104
66
FIELD_PREP(GICV5_GIC_CDPRI_ID_MASK, hwirq);
···
160
122
gicv5_iri_irq_mask(d, GICV5_HWIRQ_TYPE_SPI);
161
123
}
162
124
125
+
static void gicv5_lpi_irq_mask(struct irq_data *d)
126
+
{
127
+
gicv5_iri_irq_mask(d, GICV5_HWIRQ_TYPE_LPI);
128
+
}
129
+
163
130
static void gicv5_ppi_irq_unmask(struct irq_data *d)
164
131
{
165
132
u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64);
···
192
149
/*
193
150
* Rule R_XCLJC states that the effects of a GIC system instruction
194
151
* complete in finite time and that's the only requirement when
195
-
* unmasking an SPI IRQ.
152
+
* unmasking an SPI/LPI IRQ.
196
153
*/
197
154
gic_insn(cden, CDEN);
198
155
}
···
200
157
static void gicv5_spi_irq_unmask(struct irq_data *d)
201
158
{
202
159
gicv5_iri_irq_unmask(d, GICV5_HWIRQ_TYPE_SPI);
160
+
}
161
+
162
+
static void gicv5_lpi_irq_unmask(struct irq_data *d)
163
+
{
164
+
gicv5_iri_irq_unmask(d, GICV5_HWIRQ_TYPE_LPI);
203
165
}
204
166
205
167
static void gicv5_hwirq_eoi(u32 hwirq_id, u8 hwirq_type)
···
227
179
static void gicv5_spi_irq_eoi(struct irq_data *d)
228
180
{
229
181
gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_SPI);
182
+
}
183
+
184
+
static void gicv5_lpi_irq_eoi(struct irq_data *d)
185
+
{
186
+
gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_LPI);
230
187
}
231
188
232
189
static int gicv5_iri_irq_set_affinity(struct irq_data *d,
···
267
214
{
268
215
return gicv5_iri_irq_set_affinity(d, mask_val, force,
269
216
GICV5_HWIRQ_TYPE_SPI);
217
+
}
218
+
219
+
static int gicv5_lpi_irq_set_affinity(struct irq_data *d,
220
+
const struct cpumask *mask_val,
221
+
bool force)
222
+
{
223
+
return gicv5_iri_irq_set_affinity(d, mask_val, force,
224
+
GICV5_HWIRQ_TYPE_LPI);
270
225
}
271
226
272
227
enum ppi_reg {
···
397
336
GICV5_HWIRQ_TYPE_SPI);
398
337
}
399
338
339
+
static int gicv5_lpi_irq_get_irqchip_state(struct irq_data *d,
340
+
enum irqchip_irq_state which,
341
+
bool *state)
342
+
{
343
+
return gicv5_iri_irq_get_irqchip_state(d, which, state,
344
+
GICV5_HWIRQ_TYPE_LPI);
345
+
}
346
+
400
347
static int gicv5_ppi_irq_set_irqchip_state(struct irq_data *d,
401
348
enum irqchip_irq_state which,
402
349
bool state)
···
439
370
gicv5_iri_irq_write_pending_state(d, state, GICV5_HWIRQ_TYPE_SPI);
440
371
}
441
372
373
+
static void gicv5_lpi_irq_write_pending_state(struct irq_data *d, bool state)
374
+
{
375
+
gicv5_iri_irq_write_pending_state(d, state, GICV5_HWIRQ_TYPE_LPI);
376
+
}
377
+
442
378
static int gicv5_spi_irq_set_irqchip_state(struct irq_data *d,
443
379
enum irqchip_irq_state which,
444
380
bool state)
···
460
386
return 0;
461
387
}
462
388
389
+
static int gicv5_lpi_irq_set_irqchip_state(struct irq_data *d,
390
+
enum irqchip_irq_state which,
391
+
bool state)
392
+
{
393
+
switch (which) {
394
+
case IRQCHIP_STATE_PENDING:
395
+
gicv5_lpi_irq_write_pending_state(d, state);
396
+
break;
397
+
398
+
default:
399
+
pr_debug("Unexpected irqchip_irq_state\n");
400
+
return -EINVAL;
401
+
}
402
+
403
+
return 0;
404
+
}
405
+
463
406
static int gicv5_spi_irq_retrigger(struct irq_data *data)
464
407
{
465
408
return !gicv5_spi_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING,
466
409
true);
410
+
}
411
+
412
+
static int gicv5_lpi_irq_retrigger(struct irq_data *data)
413
+
{
414
+
return !gicv5_lpi_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING,
415
+
true);
416
+
}
417
+
418
+
static void gicv5_ipi_send_single(struct irq_data *d, unsigned int cpu)
419
+
{
420
+
/* Mark the LPI pending */
421
+
irq_chip_retrigger_hierarchy(d);
467
422
}
468
423
469
424
static bool gicv5_ppi_irq_is_level(irq_hw_number_t hwirq)
···
525
422
.irq_set_irqchip_state = gicv5_spi_irq_set_irqchip_state,
526
423
.flags = IRQCHIP_SET_TYPE_MASKED |
527
424
IRQCHIP_SKIP_SET_WAKE |
425
+
IRQCHIP_MASK_ON_SUSPEND,
426
+
};
427
+
428
+
static const struct irq_chip gicv5_lpi_irq_chip = {
429
+
.name = "GICv5-LPI",
430
+
.irq_mask = gicv5_lpi_irq_mask,
431
+
.irq_unmask = gicv5_lpi_irq_unmask,
432
+
.irq_eoi = gicv5_lpi_irq_eoi,
433
+
.irq_set_affinity = gicv5_lpi_irq_set_affinity,
434
+
.irq_retrigger = gicv5_lpi_irq_retrigger,
435
+
.irq_get_irqchip_state = gicv5_lpi_irq_get_irqchip_state,
436
+
.irq_set_irqchip_state = gicv5_lpi_irq_set_irqchip_state,
437
+
.flags = IRQCHIP_SKIP_SET_WAKE |
438
+
IRQCHIP_MASK_ON_SUSPEND,
439
+
};
440
+
441
+
static const struct irq_chip gicv5_ipi_irq_chip = {
442
+
.name = "GICv5-IPI",
443
+
.irq_mask = irq_chip_mask_parent,
444
+
.irq_unmask = irq_chip_unmask_parent,
445
+
.irq_eoi = irq_chip_eoi_parent,
446
+
.irq_set_affinity = irq_chip_set_affinity_parent,
447
+
.irq_get_irqchip_state = irq_chip_get_parent_state,
448
+
.irq_set_irqchip_state = irq_chip_set_parent_state,
449
+
.ipi_send_single = gicv5_ipi_send_single,
450
+
.flags = IRQCHIP_SKIP_SET_WAKE |
528
451
IRQCHIP_MASK_ON_SUSPEND,
529
452
};
530
453
···
714
585
.free = gicv5_irq_domain_free,
715
586
.select = gicv5_irq_spi_domain_select
716
587
};
588
+
589
+
static void gicv5_lpi_config_reset(struct irq_data *d)
590
+
{
591
+
u64 cdhm;
592
+
593
+
/*
594
+
* Reset LPIs handling mode to edge by default and clear pending
595
+
* state to make sure we start the LPI with a clean state from
596
+
* previous incarnations.
597
+
*/
598
+
cdhm = FIELD_PREP(GICV5_GIC_CDHM_HM_MASK, 0) |
599
+
FIELD_PREP(GICV5_GIC_CDHM_TYPE_MASK, GICV5_HWIRQ_TYPE_LPI) |
600
+
FIELD_PREP(GICV5_GIC_CDHM_ID_MASK, d->hwirq);
601
+
gic_insn(cdhm, CDHM);
602
+
603
+
gicv5_lpi_irq_write_pending_state(d, false);
604
+
}
605
+
606
+
static int gicv5_irq_lpi_domain_alloc(struct irq_domain *domain, unsigned int virq,
607
+
unsigned int nr_irqs, void *arg)
608
+
{
609
+
irq_hw_number_t hwirq;
610
+
struct irq_data *irqd;
611
+
u32 *lpi = arg;
612
+
int ret;
613
+
614
+
if (WARN_ON_ONCE(nr_irqs != 1))
615
+
return -EINVAL;
616
+
617
+
hwirq = *lpi;
618
+
619
+
irqd = irq_domain_get_irq_data(domain, virq);
620
+
621
+
irq_domain_set_info(domain, virq, hwirq, &gicv5_lpi_irq_chip, NULL,
622
+
handle_fasteoi_irq, NULL, NULL);
623
+
irqd_set_single_target(irqd);
624
+
625
+
ret = gicv5_irs_iste_alloc(hwirq);
626
+
if (ret < 0)
627
+
return ret;
628
+
629
+
gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_LPI);
630
+
gicv5_lpi_config_reset(irqd);
631
+
632
+
return 0;
633
+
}
634
+
635
+
static const struct irq_domain_ops gicv5_irq_lpi_domain_ops = {
636
+
.alloc = gicv5_irq_lpi_domain_alloc,
637
+
.free = gicv5_irq_domain_free,
638
+
};
639
+
640
+
void __init gicv5_init_lpi_domain(void)
641
+
{
642
+
struct irq_domain *d;
643
+
644
+
d = irq_domain_create_tree(NULL, &gicv5_irq_lpi_domain_ops, NULL);
645
+
gicv5_global_data.lpi_domain = d;
646
+
}
647
+
648
+
void __init gicv5_free_lpi_domain(void)
649
+
{
650
+
irq_domain_remove(gicv5_global_data.lpi_domain);
651
+
gicv5_global_data.lpi_domain = NULL;
652
+
}
653
+
654
+
static int gicv5_irq_ipi_domain_alloc(struct irq_domain *domain, unsigned int virq,
655
+
unsigned int nr_irqs, void *arg)
656
+
{
657
+
struct irq_data *irqd;
658
+
int ret, i;
659
+
u32 lpi;
660
+
661
+
for (i = 0; i < nr_irqs; i++) {
662
+
ret = gicv5_alloc_lpi();
663
+
if (ret < 0)
664
+
return ret;
665
+
666
+
lpi = ret;
667
+
668
+
ret = irq_domain_alloc_irqs_parent(domain, virq + i, 1, &lpi);
669
+
if (ret) {
670
+
gicv5_free_lpi(lpi);
671
+
return ret;
672
+
}
673
+
674
+
irqd = irq_domain_get_irq_data(domain, virq + i);
675
+
676
+
irq_domain_set_hwirq_and_chip(domain, virq + i, i,
677
+
&gicv5_ipi_irq_chip, NULL);
678
+
679
+
irqd_set_single_target(irqd);
680
+
681
+
irq_set_handler(virq + i, handle_percpu_irq);
682
+
}
683
+
684
+
return 0;
685
+
}
686
+
687
+
static void gicv5_irq_ipi_domain_free(struct irq_domain *domain, unsigned int virq,
688
+
unsigned int nr_irqs)
689
+
{
690
+
struct irq_data *d;
691
+
unsigned int i;
692
+
693
+
for (i = 0; i < nr_irqs; i++) {
694
+
d = irq_domain_get_irq_data(domain, virq + i);
695
+
696
+
if (!d)
697
+
return;
698
+
699
+
gicv5_free_lpi(d->parent_data->hwirq);
700
+
701
+
irq_set_handler(virq + i, NULL);
702
+
irq_domain_reset_irq_data(d);
703
+
irq_domain_free_irqs_parent(domain, virq + i, 1);
704
+
}
705
+
}
706
+
707
+
static const struct irq_domain_ops gicv5_irq_ipi_domain_ops = {
708
+
.alloc = gicv5_irq_ipi_domain_alloc,
709
+
.free = gicv5_irq_ipi_domain_free,
710
+
};
711
+
717
712
static void handle_irq_per_domain(u32 hwirq)
718
713
{
719
714
u8 hwirq_type = FIELD_GET(GICV5_HWIRQ_TYPE, hwirq);
···
850
597
break;
851
598
case GICV5_HWIRQ_TYPE_SPI:
852
599
domain = gicv5_global_data.spi_domain;
600
+
break;
601
+
case GICV5_HWIRQ_TYPE_LPI:
602
+
domain = gicv5_global_data.lpi_domain;
853
603
break;
854
604
default:
855
605
pr_err_once("Unknown IRQ type, bail out\n");
···
935
679
irq_domain_remove(gicv5_global_data.ppi_domain);
936
680
if (gicv5_global_data.spi_domain)
937
681
irq_domain_remove(gicv5_global_data.spi_domain);
682
+
if (gicv5_global_data.ipi_domain)
683
+
irq_domain_remove(gicv5_global_data.ipi_domain);
938
684
939
685
gicv5_global_data.ppi_domain = NULL;
940
686
gicv5_global_data.spi_domain = NULL;
687
+
gicv5_global_data.ipi_domain = NULL;
941
688
}
942
689
943
690
static int __init gicv5_init_domains(struct fwnode_handle *handle)
···
968
709
irq_domain_update_bus_token(d, DOMAIN_BUS_WIRED);
969
710
}
970
711
712
+
if (!WARN(!gicv5_global_data.lpi_domain,
713
+
"LPI domain uninitialized, can't set up IPIs")) {
714
+
d = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain,
715
+
0, GICV5_IPIS_PER_CPU * nr_cpu_ids,
716
+
NULL, &gicv5_irq_ipi_domain_ops,
717
+
NULL);
718
+
719
+
if (!d) {
720
+
gicv5_free_domains();
721
+
return -ENOMEM;
722
+
}
723
+
gicv5_global_data.ipi_domain = d;
724
+
}
971
725
gicv5_global_data.fwnode = handle;
972
726
973
727
return 0;
···
1004
732
}
1005
733
}
1006
734
735
+
static void gicv5_set_cpuif_idbits(void)
736
+
{
737
+
u32 icc_idr0 = read_sysreg_s(SYS_ICC_IDR0_EL1);
738
+
739
+
switch (FIELD_GET(ICC_IDR0_EL1_ID_BITS, icc_idr0)) {
740
+
case ICC_IDR0_EL1_ID_BITS_16BITS:
741
+
gicv5_global_data.cpuif_id_bits = 16;
742
+
break;
743
+
case ICC_IDR0_EL1_ID_BITS_24BITS:
744
+
gicv5_global_data.cpuif_id_bits = 24;
745
+
break;
746
+
default:
747
+
pr_err("Unexpected ICC_IDR0_EL1_ID_BITS value, default to 16");
748
+
gicv5_global_data.cpuif_id_bits = 16;
749
+
break;
750
+
}
751
+
}
752
+
1007
753
static int __init gicv5_of_init(struct device_node *node, struct device_node *parent)
1008
754
{
1009
755
int ret = gicv5_irs_of_probe(node);
···
1033
743
goto out_irs;
1034
744
1035
745
gicv5_set_cpuif_pribits();
746
+
gicv5_set_cpuif_idbits();
1036
747
1037
748
pri_bits = min_not_zero(gicv5_global_data.cpuif_pri_bits,
1038
749
gicv5_global_data.irs_pri_bits);
···
1043
752
goto out_dom;
1044
753
1045
754
ret = set_handle_irq(gicv5_handle_irq);
755
+
if (ret)
756
+
goto out_int;
757
+
758
+
ret = gicv5_irs_enable();
1046
759
if (ret)
1047
760
goto out_int;
1048
761
+62
-1
include/linux/irqchip/arm-gic-v5.h
+62
-1
include/linux/irqchip/arm-gic-v5.h
···
7
7
8
8
#include <linux/iopoll.h>
9
9
10
+
#include <asm/cacheflush.h>
11
+
#include <asm/smp.h>
10
12
#include <asm/sysreg.h>
13
+
14
+
#define GICV5_IPIS_PER_CPU MAX_IPI
11
15
12
16
/*
13
17
* INTID handling
···
21
17
#define GICV5_HWIRQ_INTID GENMASK_ULL(31, 0)
22
18
23
19
#define GICV5_HWIRQ_TYPE_PPI UL(0x1)
20
+
#define GICV5_HWIRQ_TYPE_LPI UL(0x2)
24
21
#define GICV5_HWIRQ_TYPE_SPI UL(0x3)
25
22
26
23
/*
···
41
36
#define GICV5_INNER_SHARE 0b11
42
37
43
38
/*
44
-
* IRS registers
39
+
* IRS registers and tables structures
45
40
*/
46
41
#define GICV5_IRS_IDR1 0x0004
47
42
#define GICV5_IRS_IDR2 0x0008
···
56
51
#define GICV5_IRS_PE_SELR 0x0140
57
52
#define GICV5_IRS_PE_STATUSR 0x0144
58
53
#define GICV5_IRS_PE_CR0 0x0148
54
+
#define GICV5_IRS_IST_BASER 0x0180
55
+
#define GICV5_IRS_IST_CFGR 0x0190
56
+
#define GICV5_IRS_IST_STATUSR 0x0194
57
+
#define GICV5_IRS_MAP_L2_ISTR 0x01c0
59
58
60
59
#define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20)
61
60
#define GICV5_IRS_IDR1_IAFFID_BITS GENMASK(19, 16)
···
81
72
#define GICV5_IRS_IDR5_SPI_RANGE GENMASK(24, 0)
82
73
#define GICV5_IRS_IDR6_SPI_IRS_RANGE GENMASK(24, 0)
83
74
#define GICV5_IRS_IDR7_SPI_BASE GENMASK(23, 0)
75
+
76
+
#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r))
77
+
#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r))
78
+
#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r))
79
+
84
80
#define GICV5_IRS_CR0_IDLE BIT(1)
85
81
#define GICV5_IRS_CR0_IRSEN BIT(0)
86
82
···
117
103
118
104
#define GICV5_IRS_PE_CR0_DPS BIT(0)
119
105
106
+
#define GICV5_IRS_IST_STATUSR_IDLE BIT(0)
107
+
108
+
#define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16)
109
+
#define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7)
110
+
#define GICV5_IRS_IST_CFGR_L2SZ GENMASK(6, 5)
111
+
#define GICV5_IRS_IST_CFGR_LPI_ID_BITS GENMASK(4, 0)
112
+
113
+
#define GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR 0b0
114
+
#define GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL 0b1
115
+
116
+
#define GICV5_IRS_IST_CFGR_ISTSZ_4 0b00
117
+
#define GICV5_IRS_IST_CFGR_ISTSZ_8 0b01
118
+
#define GICV5_IRS_IST_CFGR_ISTSZ_16 0b10
119
+
120
+
#define GICV5_IRS_IST_CFGR_L2SZ_4K 0b00
121
+
#define GICV5_IRS_IST_CFGR_L2SZ_16K 0b01
122
+
#define GICV5_IRS_IST_CFGR_L2SZ_64K 0b10
123
+
124
+
#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6)
125
+
#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0)
126
+
127
+
#define GICV5_IRS_MAP_L2_ISTR_ID GENMASK(23, 0)
128
+
129
+
#define GICV5_ISTL1E_VALID BIT_ULL(0)
130
+
131
+
#define GICV5_ISTL1E_L2_ADDR_MASK GENMASK_ULL(55, 12)
132
+
120
133
/*
121
134
* Global Data structures and functions
122
135
*/
···
151
110
struct fwnode_handle *fwnode;
152
111
struct irq_domain *ppi_domain;
153
112
struct irq_domain *spi_domain;
113
+
struct irq_domain *lpi_domain;
114
+
struct irq_domain *ipi_domain;
154
115
u32 global_spi_count;
155
116
u8 cpuif_pri_bits;
117
+
u8 cpuif_id_bits;
156
118
u8 irs_pri_bits;
119
+
struct {
120
+
__le64 *l1ist_addr;
121
+
u32 l2_size;
122
+
u8 l2_bits;
123
+
bool l2;
124
+
} ist;
157
125
};
158
126
159
127
extern struct gicv5_chip_data gicv5_global_data __read_mostly;
···
200
150
#define gicv5_wait_for_op_atomic(base, reg, mask, val) \
201
151
gicv5_wait_for_op_s_atomic(base, reg, #reg, mask, val)
202
152
153
+
void __init gicv5_init_lpi_domain(void);
154
+
void __init gicv5_free_lpi_domain(void);
155
+
203
156
int gicv5_irs_of_probe(struct device_node *parent);
204
157
void gicv5_irs_remove(void);
158
+
int gicv5_irs_enable(void);
205
159
int gicv5_irs_register_cpu(int cpuid);
206
160
int gicv5_irs_cpu_to_iaffid(int cpu_id, u16 *iaffid);
207
161
struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id);
208
162
int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type);
163
+
int gicv5_irs_iste_alloc(u32 lpi);
164
+
165
+
void gicv5_init_lpis(u32 max);
166
+
void gicv5_deinit_lpis(void);
167
+
168
+
int gicv5_alloc_lpi(void);
169
+
void gicv5_free_lpi(u32 lpi);
209
170
#endif