Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
sched/isolation: Convert housekeeping cpumasks to rcu pointers
HK_TYPE_DOMAIN's cpumask will soon be made modifiable by cpuset.
A synchronization mechanism is then needed to synchronize the updates
with the housekeeping cpumask readers.
Turn the housekeeping cpumasks into RCU pointers. Once a housekeeping
cpumask will be modified, the update side will wait for an RCU grace
period and propagate the change to interested subsystem when deemed
necessary.
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Marco Crivellari <marco.crivellari@suse.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Waiman Long <longman@redhat.com>
+38
-23
+37
-23
kernel/sched/isolation.c
+37
-23
kernel/sched/isolation.c
···
21
21
EXPORT_SYMBOL_GPL(housekeeping_overridden);
22
22
23
23
struct housekeeping {
24
-
cpumask_var_t cpumasks[HK_TYPE_MAX];
24
+
struct cpumask __rcu *cpumasks[HK_TYPE_MAX];
25
25
unsigned long flags;
26
26
};
27
27
···
33
33
}
34
34
EXPORT_SYMBOL_GPL(housekeeping_enabled);
35
35
36
+
const struct cpumask *housekeeping_cpumask(enum hk_type type)
37
+
{
38
+
if (static_branch_unlikely(&housekeeping_overridden)) {
39
+
if (housekeeping.flags & BIT(type)) {
40
+
return rcu_dereference_check(housekeeping.cpumasks[type], 1);
41
+
}
42
+
}
43
+
return cpu_possible_mask;
44
+
}
45
+
EXPORT_SYMBOL_GPL(housekeeping_cpumask);
46
+
36
47
int housekeeping_any_cpu(enum hk_type type)
37
48
{
38
49
int cpu;
39
50
40
51
if (static_branch_unlikely(&housekeeping_overridden)) {
41
52
if (housekeeping.flags & BIT(type)) {
42
-
cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
53
+
cpu = sched_numa_find_closest(housekeeping_cpumask(type), smp_processor_id());
43
54
if (cpu < nr_cpu_ids)
44
55
return cpu;
45
56
46
-
cpu = cpumask_any_and_distribute(housekeeping.cpumasks[type], cpu_online_mask);
57
+
cpu = cpumask_any_and_distribute(housekeeping_cpumask(type), cpu_online_mask);
47
58
if (likely(cpu < nr_cpu_ids))
48
59
return cpu;
49
60
/*
···
70
59
}
71
60
EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
72
61
73
-
const struct cpumask *housekeeping_cpumask(enum hk_type type)
74
-
{
75
-
if (static_branch_unlikely(&housekeeping_overridden))
76
-
if (housekeeping.flags & BIT(type))
77
-
return housekeeping.cpumasks[type];
78
-
return cpu_possible_mask;
79
-
}
80
-
EXPORT_SYMBOL_GPL(housekeeping_cpumask);
81
-
82
62
void housekeeping_affine(struct task_struct *t, enum hk_type type)
83
63
{
84
64
if (static_branch_unlikely(&housekeeping_overridden))
85
65
if (housekeeping.flags & BIT(type))
86
-
set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
66
+
set_cpus_allowed_ptr(t, housekeeping_cpumask(type));
87
67
}
88
68
EXPORT_SYMBOL_GPL(housekeeping_affine);
89
69
90
70
bool housekeeping_test_cpu(int cpu, enum hk_type type)
91
71
{
92
-
if (static_branch_unlikely(&housekeeping_overridden))
93
-
if (housekeeping.flags & BIT(type))
94
-
return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
72
+
if (static_branch_unlikely(&housekeeping_overridden) && housekeeping.flags & BIT(type))
73
+
return cpumask_test_cpu(cpu, housekeeping_cpumask(type));
95
74
return true;
96
75
}
97
76
EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
···
97
96
98
97
if (housekeeping.flags & HK_FLAG_KERNEL_NOISE)
99
98
sched_tick_offload_init();
100
-
99
+
/*
100
+
* Realloc with a proper allocator so that any cpumask update
101
+
* can indifferently free the old version with kfree().
102
+
*/
101
103
for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
104
+
struct cpumask *omask, *nmask = kmalloc(cpumask_size(), GFP_KERNEL);
105
+
106
+
if (WARN_ON_ONCE(!nmask))
107
+
return;
108
+
109
+
omask = rcu_dereference(housekeeping.cpumasks[type]);
110
+
102
111
/* We need at least one CPU to handle housekeeping work */
103
-
WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
112
+
WARN_ON_ONCE(cpumask_empty(omask));
113
+
cpumask_copy(nmask, omask);
114
+
RCU_INIT_POINTER(housekeeping.cpumasks[type], nmask);
115
+
memblock_free(omask, cpumask_size());
104
116
}
105
117
}
106
118
107
119
static void __init housekeeping_setup_type(enum hk_type type,
108
120
cpumask_var_t housekeeping_staging)
109
121
{
122
+
struct cpumask *mask = memblock_alloc_or_panic(cpumask_size(), SMP_CACHE_BYTES);
110
123
111
-
alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
112
-
cpumask_copy(housekeeping.cpumasks[type],
113
-
housekeeping_staging);
124
+
cpumask_copy(mask, housekeeping_staging);
125
+
RCU_INIT_POINTER(housekeeping.cpumasks[type], mask);
114
126
}
115
127
116
128
static int __init housekeeping_setup(char *str, unsigned long flags)
···
176
162
177
163
for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
178
164
if (!cpumask_equal(housekeeping_staging,
179
-
housekeeping.cpumasks[type])) {
165
+
housekeeping_cpumask(type))) {
180
166
pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
181
167
goto free_housekeeping_staging;
182
168
}
···
197
183
iter_flags = flags & (HK_FLAG_KERNEL_NOISE | HK_FLAG_DOMAIN);
198
184
first_cpu = (type == HK_TYPE_MAX || !iter_flags) ? 0 :
199
185
cpumask_first_and_and(cpu_present_mask,
200
-
housekeeping_staging, housekeeping.cpumasks[type]);
186
+
housekeeping_staging, housekeeping_cpumask(type));
201
187
if (first_cpu >= min(nr_cpu_ids, setup_max_cpus)) {
202
188
pr_warn("Housekeeping: must include one present CPU "
203
189
"neither in nohz_full= nor in isolcpus=domain, "