Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
arm64: smp: Support non-SGIs for IPIs
The arm64 arch has relied so far on GIC architectural software
generated interrupt (SGIs) to handle IPIs. Those are per-cpu
software generated interrupts.
arm64 architecture code that allocates the IPIs virtual IRQs and
IRQ descriptors was written accordingly.
On GICv5 systems, IPIs are implemented using LPIs that are not
per-cpu interrupts - they are just normal routable IRQs.
Add arch code to set-up IPIs on systems where they are handled
using normal routable IRQs.
For those systems, force the IRQ affinity (and make it immutable)
to the cpu a given IRQ was assigned to.
Signed-off-by: Timothy Hayes <timothy.hayes@arm.com>
[lpieralisi: changed affinity set-up, log]
Signed-off-by: Lorenzo Pieralisi <lpieralisi@kernel.org>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20250703-gicv5-host-v7-18-12e71f1b3528@kernel.org
Signed-off-by: Marc Zyngier <maz@kernel.org>
+99
-35
+6
-1
arch/arm64/include/asm/smp.h
+6
-1
arch/arm64/include/asm/smp.h
···
53
53
/*
54
54
* Register IPI interrupts with the arch SMP code
55
55
*/
56
-
extern void set_smp_ipi_range(int ipi_base, int nr_ipi);
56
+
extern void set_smp_ipi_range_percpu(int ipi_base, int nr_ipi, int ncpus);
57
+
58
+
static inline void set_smp_ipi_range(int ipi_base, int n)
59
+
{
60
+
set_smp_ipi_range_percpu(ipi_base, n, 0);
61
+
}
57
62
58
63
/*
59
64
* Called from the secondary holding pen, this is the secondary CPU entry point.
+93
-34
arch/arm64/kernel/smp.c
+93
-34
arch/arm64/kernel/smp.c
···
83
83
84
84
static int ipi_irq_base __ro_after_init;
85
85
static int nr_ipi __ro_after_init = NR_IPI;
86
-
static struct irq_desc *ipi_desc[MAX_IPI] __ro_after_init;
86
+
87
+
struct ipi_descs {
88
+
struct irq_desc *descs[MAX_IPI];
89
+
};
90
+
91
+
static DEFINE_PER_CPU_READ_MOSTLY(struct ipi_descs, pcpu_ipi_desc);
92
+
93
+
#define get_ipi_desc(__cpu, __ipi) (per_cpu_ptr(&pcpu_ipi_desc, __cpu)->descs[__ipi])
94
+
95
+
static bool percpu_ipi_descs __ro_after_init;
87
96
88
97
static bool crash_stop;
89
98
···
853
844
seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
854
845
prec >= 4 ? " " : "");
855
846
for_each_online_cpu(cpu)
856
-
seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu));
847
+
seq_printf(p, "%10u ", irq_desc_kstat_cpu(get_ipi_desc(cpu, i), cpu));
857
848
seq_printf(p, " %s\n", ipi_types[i]);
858
849
}
859
850
···
926
917
#endif
927
918
}
928
919
920
+
static void arm64_send_ipi(const cpumask_t *mask, unsigned int nr)
921
+
{
922
+
unsigned int cpu;
923
+
924
+
if (!percpu_ipi_descs)
925
+
__ipi_send_mask(get_ipi_desc(0, nr), mask);
926
+
else
927
+
for_each_cpu(cpu, mask)
928
+
__ipi_send_single(get_ipi_desc(cpu, nr), cpu);
929
+
}
930
+
929
931
static void arm64_backtrace_ipi(cpumask_t *mask)
930
932
{
931
-
__ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask);
933
+
arm64_send_ipi(mask, IPI_CPU_BACKTRACE);
932
934
}
933
935
934
936
void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu)
···
964
944
if (cpu == this_cpu)
965
945
continue;
966
946
967
-
__ipi_send_single(ipi_desc[IPI_KGDB_ROUNDUP], cpu);
947
+
__ipi_send_single(get_ipi_desc(cpu, IPI_KGDB_ROUNDUP), cpu);
968
948
}
969
949
}
970
950
#endif
···
1033
1013
1034
1014
static irqreturn_t ipi_handler(int irq, void *data)
1035
1015
{
1036
-
do_handle_IPI(irq - ipi_irq_base);
1016
+
unsigned int ipi = (irq - ipi_irq_base) % nr_ipi;
1017
+
1018
+
do_handle_IPI(ipi);
1037
1019
return IRQ_HANDLED;
1038
1020
}
1039
1021
1040
1022
static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
1041
1023
{
1042
1024
trace_ipi_raise(target, ipi_types[ipinr]);
1043
-
__ipi_send_mask(ipi_desc[ipinr], target);
1025
+
arm64_send_ipi(target, ipinr);
1044
1026
}
1045
1027
1046
1028
static bool ipi_should_be_nmi(enum ipi_msg_type ipi)
···
1068
1046
return;
1069
1047
1070
1048
for (i = 0; i < nr_ipi; i++) {
1071
-
if (ipi_should_be_nmi(i)) {
1072
-
prepare_percpu_nmi(ipi_irq_base + i);
1073
-
enable_percpu_nmi(ipi_irq_base + i, 0);
1049
+
if (!percpu_ipi_descs) {
1050
+
if (ipi_should_be_nmi(i)) {
1051
+
prepare_percpu_nmi(ipi_irq_base + i);
1052
+
enable_percpu_nmi(ipi_irq_base + i, 0);
1053
+
} else {
1054
+
enable_percpu_irq(ipi_irq_base + i, 0);
1055
+
}
1074
1056
} else {
1075
-
enable_percpu_irq(ipi_irq_base + i, 0);
1057
+
enable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));
1076
1058
}
1077
1059
}
1078
1060
}
···
1090
1064
return;
1091
1065
1092
1066
for (i = 0; i < nr_ipi; i++) {
1093
-
if (ipi_should_be_nmi(i)) {
1094
-
disable_percpu_nmi(ipi_irq_base + i);
1095
-
teardown_percpu_nmi(ipi_irq_base + i);
1067
+
if (!percpu_ipi_descs) {
1068
+
if (ipi_should_be_nmi(i)) {
1069
+
disable_percpu_nmi(ipi_irq_base + i);
1070
+
teardown_percpu_nmi(ipi_irq_base + i);
1071
+
} else {
1072
+
disable_percpu_irq(ipi_irq_base + i);
1073
+
}
1096
1074
} else {
1097
-
disable_percpu_irq(ipi_irq_base + i);
1075
+
disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));
1098
1076
}
1099
1077
}
1100
1078
}
1101
1079
#endif
1102
1080
1103
-
void __init set_smp_ipi_range(int ipi_base, int n)
1081
+
static void ipi_setup_sgi(int ipi)
1082
+
{
1083
+
int err, irq, cpu;
1084
+
1085
+
irq = ipi_irq_base + ipi;
1086
+
1087
+
if (ipi_should_be_nmi(irq)) {
1088
+
err = request_percpu_nmi(irq, ipi_handler, "IPI", &irq_stat);
1089
+
WARN(err, "Could not request IRQ %d as NMI, err=%d\n", irq, err);
1090
+
} else {
1091
+
err = request_percpu_irq(irq, ipi_handler, "IPI", &irq_stat);
1092
+
WARN(err, "Could not request IRQ %d as IRQ, err=%d\n", irq, err);
1093
+
}
1094
+
1095
+
for_each_possible_cpu(cpu)
1096
+
get_ipi_desc(cpu, ipi) = irq_to_desc(irq);
1097
+
1098
+
irq_set_status_flags(irq, IRQ_HIDDEN);
1099
+
}
1100
+
1101
+
static void ipi_setup_lpi(int ipi, int ncpus)
1102
+
{
1103
+
for (int cpu = 0; cpu < ncpus; cpu++) {
1104
+
int err, irq;
1105
+
1106
+
irq = ipi_irq_base + (cpu * nr_ipi) + ipi;
1107
+
1108
+
err = irq_force_affinity(irq, cpumask_of(cpu));
1109
+
WARN(err, "Could not force affinity IRQ %d, err=%d\n", irq, err);
1110
+
1111
+
err = request_irq(irq, ipi_handler, IRQF_NO_AUTOEN, "IPI",
1112
+
NULL);
1113
+
WARN(err, "Could not request IRQ %d, err=%d\n", irq, err);
1114
+
1115
+
irq_set_status_flags(irq, (IRQ_HIDDEN | IRQ_NO_BALANCING_MASK));
1116
+
1117
+
get_ipi_desc(cpu, ipi) = irq_to_desc(irq);
1118
+
}
1119
+
}
1120
+
1121
+
void __init set_smp_ipi_range_percpu(int ipi_base, int n, int ncpus)
1104
1122
{
1105
1123
int i;
1106
1124
1107
1125
WARN_ON(n < MAX_IPI);
1108
1126
nr_ipi = min(n, MAX_IPI);
1109
1127
1110
-
for (i = 0; i < nr_ipi; i++) {
1111
-
int err;
1112
-
1113
-
if (ipi_should_be_nmi(i)) {
1114
-
err = request_percpu_nmi(ipi_base + i, ipi_handler,
1115
-
"IPI", &irq_stat);
1116
-
WARN(err, "Could not request IPI %d as NMI, err=%d\n",
1117
-
i, err);
1118
-
} else {
1119
-
err = request_percpu_irq(ipi_base + i, ipi_handler,
1120
-
"IPI", &irq_stat);
1121
-
WARN(err, "Could not request IPI %d as IRQ, err=%d\n",
1122
-
i, err);
1123
-
}
1124
-
1125
-
ipi_desc[i] = irq_to_desc(ipi_base + i);
1126
-
irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);
1127
-
}
1128
-
1128
+
percpu_ipi_descs = !!ncpus;
1129
1129
ipi_irq_base = ipi_base;
1130
+
1131
+
for (i = 0; i < nr_ipi; i++) {
1132
+
if (!percpu_ipi_descs)
1133
+
ipi_setup_sgi(i);
1134
+
else
1135
+
ipi_setup_lpi(i, ncpus);
1136
+
}
1130
1137
1131
1138
/* Setup the boot CPU immediately */
1132
1139
ipi_setup(smp_processor_id());