Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
workqueue: add WQ_AFFN_CACHE_SHARD affinity scope
On systems where many CPUs share one LLC, unbound workqueues using
WQ_AFFN_CACHE collapse to a single worker pool, causing heavy spinlock
contention on pool->lock. For example, Chuck Lever measured 39% of
cycles lost to native_queued_spin_lock_slowpath on a 12-core shared-L3
NFS-over-RDMA system.
The existing affinity hierarchy (cpu, smt, cache, numa, system) offers
no intermediate option between per-LLC and per-SMT-core granularity.
Add WQ_AFFN_CACHE_SHARD, which subdivides each LLC into groups of at
most wq_cache_shard_size cores (default 8, tunable via boot parameter).
Shards are always split on core (SMT group) boundaries so that
Hyper-Threading siblings are never placed in different pods. Cores are
distributed across shards as evenly as possible -- for example, 36 cores
in a single LLC with max shard size 8 produces 5 shards of 8+7+7+7+7
cores.
The implementation follows the same comparator pattern as other affinity
scopes: precompute_cache_shard_ids() pre-fills the cpu_shard_id[] array
from the already-initialized WQ_AFFN_CACHE and WQ_AFFN_SMT topology,
and cpus_share_cache_shard() is passed to init_pod_type().
Benchmark on NVIDIA Grace (72 CPUs, single LLC, 50k items/thread), show
cache_shard delivers ~5x the throughput and ~6.5x lower p50 latency
compared to cache scope on this 72-core single-LLC system.
Suggested-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Breno Leitao <leitao@debian.org>
Signed-off-by: Tejun Heo <tj@kernel.org>
authored by
Breno Leitao
and committed by
Tejun Heo
5920d046
9dc42c90
+184
+1
include/linux/workqueue.h
+1
include/linux/workqueue.h
···
133
133
WQ_AFFN_CPU, /* one pod per CPU */
134
134
WQ_AFFN_SMT, /* one pod per SMT */
135
135
WQ_AFFN_CACHE, /* one pod per LLC */
136
+
WQ_AFFN_CACHE_SHARD, /* synthetic sub-LLC shards */
136
137
WQ_AFFN_NUMA, /* one pod per NUMA node */
137
138
WQ_AFFN_SYSTEM, /* one pod across the whole system */
138
139
+183
kernel/workqueue.c
+183
kernel/workqueue.c
···
131
131
WORKER_ID_LEN = 10 + WQ_NAME_LEN, /* "kworker/R-" + WQ_NAME_LEN */
132
132
};
133
133
134
+
/* Layout of shards within one LLC pod */
135
+
struct llc_shard_layout {
136
+
int nr_large_shards; /* number of large shards (cores_per_shard + 1) */
137
+
int cores_per_shard; /* base number of cores per default shard */
138
+
int nr_shards; /* total number of shards */
139
+
/* nr_default shards = (nr_shards - nr_large_shards) */
140
+
};
141
+
134
142
/*
135
143
* We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and
136
144
* MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because
···
418
410
[WQ_AFFN_CPU] = "cpu",
419
411
[WQ_AFFN_SMT] = "smt",
420
412
[WQ_AFFN_CACHE] = "cache",
413
+
[WQ_AFFN_CACHE_SHARD] = "cache_shard",
421
414
[WQ_AFFN_NUMA] = "numa",
422
415
[WQ_AFFN_SYSTEM] = "system",
423
416
};
···
440
431
/* see the comment above the definition of WQ_POWER_EFFICIENT */
441
432
static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
442
433
module_param_named(power_efficient, wq_power_efficient, bool, 0444);
434
+
435
+
static unsigned int wq_cache_shard_size = 8;
436
+
module_param_named(cache_shard_size, wq_cache_shard_size, uint, 0444);
443
437
444
438
static bool wq_online; /* can kworkers be created yet? */
445
439
static bool wq_topo_initialized __read_mostly = false;
···
8167
8155
return cpu_to_node(cpu0) == cpu_to_node(cpu1);
8168
8156
}
8169
8157
8158
+
/* Maps each CPU to its shard index within the LLC pod it belongs to */
8159
+
static int cpu_shard_id[NR_CPUS] __initdata;
8160
+
8161
+
/**
8162
+
* llc_count_cores - count distinct cores (SMT groups) within an LLC pod
8163
+
* @pod_cpus: the cpumask of CPUs in the LLC pod
8164
+
* @smt_pods: the SMT pod type, used to identify sibling groups
8165
+
*
8166
+
* A core is represented by the lowest-numbered CPU in its SMT group. Returns
8167
+
* the number of distinct cores found in @pod_cpus.
8168
+
*/
8169
+
static int __init llc_count_cores(const struct cpumask *pod_cpus,
8170
+
struct wq_pod_type *smt_pods)
8171
+
{
8172
+
const struct cpumask *sibling_cpus;
8173
+
int nr_cores = 0, c;
8174
+
8175
+
/*
8176
+
* Count distinct cores by only counting the first CPU in each
8177
+
* SMT sibling group.
8178
+
*/
8179
+
for_each_cpu(c, pod_cpus) {
8180
+
sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]];
8181
+
if (cpumask_first(sibling_cpus) == c)
8182
+
nr_cores++;
8183
+
}
8184
+
8185
+
return nr_cores;
8186
+
}
8187
+
8188
+
/*
8189
+
* llc_shard_size - number of cores in a given shard
8190
+
*
8191
+
* Cores are spread as evenly as possible. The first @nr_large_shards shards are
8192
+
* "large shards" with (cores_per_shard + 1) cores; the rest are "default
8193
+
* shards" with cores_per_shard cores.
8194
+
*/
8195
+
static int __init llc_shard_size(int shard_id, int cores_per_shard, int nr_large_shards)
8196
+
{
8197
+
/* The first @nr_large_shards shards are large shards */
8198
+
if (shard_id < nr_large_shards)
8199
+
return cores_per_shard + 1;
8200
+
8201
+
/* The remaining shards are default shards */
8202
+
return cores_per_shard;
8203
+
}
8204
+
8205
+
/*
8206
+
* llc_calc_shard_layout - compute the shard layout for an LLC pod
8207
+
* @nr_cores: number of distinct cores in the LLC pod
8208
+
*
8209
+
* Chooses the number of shards that keeps average shard size closest to
8210
+
* wq_cache_shard_size. Returns a struct describing the total number of shards,
8211
+
* the base size of each, and how many are large shards.
8212
+
*/
8213
+
static struct llc_shard_layout __init llc_calc_shard_layout(int nr_cores)
8214
+
{
8215
+
struct llc_shard_layout layout;
8216
+
8217
+
/* Ensure at least one shard; pick the count closest to the target size */
8218
+
layout.nr_shards = max(1, DIV_ROUND_CLOSEST(nr_cores, wq_cache_shard_size));
8219
+
layout.cores_per_shard = nr_cores / layout.nr_shards;
8220
+
layout.nr_large_shards = nr_cores % layout.nr_shards;
8221
+
8222
+
return layout;
8223
+
}
8224
+
8225
+
/*
8226
+
* llc_shard_is_full - check whether a shard has reached its core capacity
8227
+
* @cores_in_shard: number of cores already assigned to this shard
8228
+
* @shard_id: index of the shard being checked
8229
+
* @layout: the shard layout computed by llc_calc_shard_layout()
8230
+
*
8231
+
* Returns true if @cores_in_shard equals the expected size for @shard_id.
8232
+
*/
8233
+
static bool __init llc_shard_is_full(int cores_in_shard, int shard_id,
8234
+
const struct llc_shard_layout *layout)
8235
+
{
8236
+
return cores_in_shard == llc_shard_size(shard_id, layout->cores_per_shard,
8237
+
layout->nr_large_shards);
8238
+
}
8239
+
8240
+
/**
8241
+
* llc_populate_cpu_shard_id - populate cpu_shard_id[] for each CPU in an LLC pod
8242
+
* @pod_cpus: the cpumask of CPUs in the LLC pod
8243
+
* @smt_pods: the SMT pod type, used to identify sibling groups
8244
+
* @nr_cores: number of distinct cores in @pod_cpus (from llc_count_cores())
8245
+
*
8246
+
* Walks @pod_cpus in order. At each SMT group leader, advances to the next
8247
+
* shard once the current shard is full. Results are written to cpu_shard_id[].
8248
+
*/
8249
+
static void __init llc_populate_cpu_shard_id(const struct cpumask *pod_cpus,
8250
+
struct wq_pod_type *smt_pods,
8251
+
int nr_cores)
8252
+
{
8253
+
struct llc_shard_layout layout = llc_calc_shard_layout(nr_cores);
8254
+
const struct cpumask *sibling_cpus;
8255
+
/* Count the number of cores in the current shard_id */
8256
+
int cores_in_shard = 0;
8257
+
/* This is a cursor for the shards. Go from zero to nr_shards - 1*/
8258
+
int shard_id = 0;
8259
+
int c;
8260
+
8261
+
/* Iterate at every CPU for a given LLC pod, and assign it a shard */
8262
+
for_each_cpu(c, pod_cpus) {
8263
+
sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]];
8264
+
if (cpumask_first(sibling_cpus) == c) {
8265
+
/* This is the CPU leader for the siblings */
8266
+
if (llc_shard_is_full(cores_in_shard, shard_id, &layout)) {
8267
+
shard_id++;
8268
+
cores_in_shard = 0;
8269
+
}
8270
+
cores_in_shard++;
8271
+
cpu_shard_id[c] = shard_id;
8272
+
} else {
8273
+
/*
8274
+
* The siblings' shard MUST be the same as the leader.
8275
+
* never split threads in the same core.
8276
+
*/
8277
+
cpu_shard_id[c] = cpu_shard_id[cpumask_first(sibling_cpus)];
8278
+
}
8279
+
}
8280
+
8281
+
WARN_ON_ONCE(shard_id != (layout.nr_shards - 1));
8282
+
}
8283
+
8284
+
/**
8285
+
* precompute_cache_shard_ids - assign each CPU its shard index within its LLC
8286
+
*
8287
+
* Iterates over all LLC pods. For each pod, counts distinct cores then assigns
8288
+
* shard indices to all CPUs in the pod. Must be called after WQ_AFFN_CACHE and
8289
+
* WQ_AFFN_SMT have been initialized.
8290
+
*/
8291
+
static void __init precompute_cache_shard_ids(void)
8292
+
{
8293
+
struct wq_pod_type *llc_pods = &wq_pod_types[WQ_AFFN_CACHE];
8294
+
struct wq_pod_type *smt_pods = &wq_pod_types[WQ_AFFN_SMT];
8295
+
const struct cpumask *cpus_sharing_llc;
8296
+
int nr_cores;
8297
+
int pod;
8298
+
8299
+
if (!wq_cache_shard_size) {
8300
+
pr_warn("workqueue: cache_shard_size must be > 0, setting to 1\n");
8301
+
wq_cache_shard_size = 1;
8302
+
}
8303
+
8304
+
for (pod = 0; pod < llc_pods->nr_pods; pod++) {
8305
+
cpus_sharing_llc = llc_pods->pod_cpus[pod];
8306
+
8307
+
/* Number of cores in this given LLC */
8308
+
nr_cores = llc_count_cores(cpus_sharing_llc, smt_pods);
8309
+
llc_populate_cpu_shard_id(cpus_sharing_llc, smt_pods, nr_cores);
8310
+
}
8311
+
}
8312
+
8313
+
/*
8314
+
* cpus_share_cache_shard - test whether two CPUs belong to the same cache shard
8315
+
*
8316
+
* Two CPUs share a cache shard if they are in the same LLC and have the same
8317
+
* shard index. Used as the pod affinity callback for WQ_AFFN_CACHE_SHARD.
8318
+
*/
8319
+
static bool __init cpus_share_cache_shard(int cpu0, int cpu1)
8320
+
{
8321
+
if (!cpus_share_cache(cpu0, cpu1))
8322
+
return false;
8323
+
8324
+
return cpu_shard_id[cpu0] == cpu_shard_id[cpu1];
8325
+
}
8326
+
8170
8327
/**
8171
8328
* workqueue_init_topology - initialize CPU pods for unbound workqueues
8172
8329
*
···
8351
8170
init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share);
8352
8171
init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt);
8353
8172
init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache);
8173
+
precompute_cache_shard_ids();
8174
+
init_pod_type(&wq_pod_types[WQ_AFFN_CACHE_SHARD], cpus_share_cache_shard);
8354
8175
init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa);
8355
8176
8356
8177
wq_topo_initialized = true;