Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
sched/isolation: Split housekeeping cpumask per isolation features
To prepare for supporting each housekeeping feature toward cpuset, split
the global housekeeping cpumask per HK_TYPE_* entry.
This will later allow, for example, to runtime modify the cpulist passed
through "isolcpus=", "nohz_full=" and "rcu_nocbs=" kernel boot
parameters.
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Juri Lelli <juri.lelli@redhat.com>
Reviewed-by: Phil Auld <pauld@redhat.com>
Link: https://lore.kernel.org/r/20220207155910.527133-9-frederic@kernel.org
authored by
Frederic Weisbecker
and committed by
Peter Zijlstra
ed3b362d
65e53f86
+62
-29
+62
-29
kernel/sched/isolation.c
+62
-29
kernel/sched/isolation.c
···
23
23
24
24
DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
25
25
EXPORT_SYMBOL_GPL(housekeeping_overridden);
26
-
static cpumask_var_t housekeeping_mask;
27
-
static unsigned int housekeeping_flags;
26
+
27
+
struct housekeeping {
28
+
cpumask_var_t cpumasks[HK_TYPE_MAX];
29
+
unsigned long flags;
30
+
};
31
+
32
+
static struct housekeeping housekeeping;
28
33
29
34
bool housekeeping_enabled(enum hk_type type)
30
35
{
31
-
return !!(housekeeping_flags & BIT(type));
36
+
return !!(housekeeping.flags & BIT(type));
32
37
}
33
38
EXPORT_SYMBOL_GPL(housekeeping_enabled);
34
39
···
42
37
int cpu;
43
38
44
39
if (static_branch_unlikely(&housekeeping_overridden)) {
45
-
if (housekeeping_flags & BIT(type)) {
46
-
cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id());
40
+
if (housekeeping.flags & BIT(type)) {
41
+
cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
47
42
if (cpu < nr_cpu_ids)
48
43
return cpu;
49
44
50
-
return cpumask_any_and(housekeeping_mask, cpu_online_mask);
45
+
return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
51
46
}
52
47
}
53
48
return smp_processor_id();
···
57
52
const struct cpumask *housekeeping_cpumask(enum hk_type type)
58
53
{
59
54
if (static_branch_unlikely(&housekeeping_overridden))
60
-
if (housekeeping_flags & BIT(type))
61
-
return housekeeping_mask;
55
+
if (housekeeping.flags & BIT(type))
56
+
return housekeeping.cpumasks[type];
62
57
return cpu_possible_mask;
63
58
}
64
59
EXPORT_SYMBOL_GPL(housekeeping_cpumask);
···
66
61
void housekeeping_affine(struct task_struct *t, enum hk_type type)
67
62
{
68
63
if (static_branch_unlikely(&housekeeping_overridden))
69
-
if (housekeeping_flags & BIT(type))
70
-
set_cpus_allowed_ptr(t, housekeeping_mask);
64
+
if (housekeeping.flags & BIT(type))
65
+
set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
71
66
}
72
67
EXPORT_SYMBOL_GPL(housekeeping_affine);
73
68
74
69
bool housekeeping_test_cpu(int cpu, enum hk_type type)
75
70
{
76
71
if (static_branch_unlikely(&housekeeping_overridden))
77
-
if (housekeeping_flags & BIT(type))
78
-
return cpumask_test_cpu(cpu, housekeeping_mask);
72
+
if (housekeeping.flags & BIT(type))
73
+
return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
79
74
return true;
80
75
}
81
76
EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
82
77
83
78
void __init housekeeping_init(void)
84
79
{
85
-
if (!housekeeping_flags)
80
+
enum hk_type type;
81
+
82
+
if (!housekeeping.flags)
86
83
return;
87
84
88
85
static_branch_enable(&housekeeping_overridden);
89
86
90
-
if (housekeeping_flags & HK_FLAG_TICK)
87
+
if (housekeeping.flags & HK_FLAG_TICK)
91
88
sched_tick_offload_init();
92
89
93
-
/* We need at least one CPU to handle housekeeping work */
94
-
WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
90
+
for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
91
+
/* We need at least one CPU to handle housekeeping work */
92
+
WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
93
+
}
95
94
}
96
95
97
-
static int __init housekeeping_setup(char *str, enum hk_flags flags)
96
+
static void __init housekeeping_setup_type(enum hk_type type,
97
+
cpumask_var_t housekeeping_staging)
98
+
{
99
+
100
+
alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
101
+
cpumask_copy(housekeeping.cpumasks[type],
102
+
housekeeping_staging);
103
+
}
104
+
105
+
static int __init housekeeping_setup(char *str, unsigned long flags)
98
106
{
99
107
cpumask_var_t non_housekeeping_mask, housekeeping_staging;
100
108
int err = 0;
101
109
102
-
if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
110
+
if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
103
111
if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
104
112
pr_warn("Housekeeping: nohz unsupported."
105
113
" Build with CONFIG_NO_HZ_FULL\n");
···
133
115
if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) {
134
116
__cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
135
117
__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
136
-
if (!housekeeping_flags) {
118
+
if (!housekeeping.flags) {
137
119
pr_warn("Housekeeping: must include one present CPU, "
138
120
"using boot CPU:%d\n", smp_processor_id());
139
121
}
140
122
}
141
123
142
-
if (!housekeeping_flags) {
143
-
alloc_bootmem_cpumask_var(&housekeeping_mask);
144
-
cpumask_copy(housekeeping_mask, housekeeping_staging);
124
+
if (!housekeeping.flags) {
125
+
/* First setup call ("nohz_full=" or "isolcpus=") */
126
+
enum hk_type type;
127
+
128
+
for_each_set_bit(type, &flags, HK_TYPE_MAX)
129
+
housekeeping_setup_type(type, housekeeping_staging);
145
130
} else {
146
-
if (!cpumask_equal(housekeeping_staging, housekeeping_mask)) {
147
-
pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
148
-
goto free_housekeeping_staging;
131
+
/* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
132
+
enum hk_type type;
133
+
unsigned long iter_flags = flags & housekeeping.flags;
134
+
135
+
for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
136
+
if (!cpumask_equal(housekeeping_staging,
137
+
housekeeping.cpumasks[type])) {
138
+
pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
139
+
goto free_housekeeping_staging;
140
+
}
149
141
}
142
+
143
+
iter_flags = flags & ~housekeeping.flags;
144
+
145
+
for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
146
+
housekeeping_setup_type(type, housekeeping_staging);
150
147
}
151
148
152
-
if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK))
149
+
if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
153
150
tick_nohz_full_setup(non_housekeeping_mask);
154
151
155
-
housekeeping_flags |= flags;
152
+
housekeeping.flags |= flags;
156
153
err = 1;
157
154
158
155
free_housekeeping_staging:
···
180
147
181
148
static int __init housekeeping_nohz_full_setup(char *str)
182
149
{
183
-
unsigned int flags;
150
+
unsigned long flags;
184
151
185
152
flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
186
153
HK_FLAG_MISC | HK_FLAG_KTHREAD;
···
191
158
192
159
static int __init housekeeping_isolcpus_setup(char *str)
193
160
{
194
-
unsigned int flags = 0;
161
+
unsigned long flags = 0;
195
162
bool illegal = false;
196
163
char *par;
197
164
int len;