Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
workqueue: RCU protect wq->dfl_pwq and implement accessors for it
wq->cpu_pwq is RCU protected but wq->dfl_pwq isn't. This is okay because
currently wq->dfl_pwq is used only accessed to install it into wq->cpu_pwq
which doesn't require RCU access. However, we want to be able to access
wq->dfl_pwq under RCU in the future to access its __pod_cpumask and the code
can be made easier to read by making the two pwq fields behave in the same
way.
- Make wq->dfl_pwq RCU protected.
- Add unbound_pwq_slot() and unbound_pwq() which can access both ->dfl_pwq
and ->cpu_pwq. The former returns the double pointer that can be used
access and update the pwqs. The latter performs locking check and
dereferences the double pointer.
- pwq accesses and updates are converted to use unbound_pwq[_slot]().
Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Lai Jiangshan <jiangshanlai@gmail.com>
+40
-24
+40
-24
kernel/workqueue.c
+40
-24
kernel/workqueue.c
···
308
308
int saved_max_active; /* WQ: saved max_active */
309
309
310
310
struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */
311
-
struct pool_workqueue *dfl_pwq; /* PW: only for unbound wqs */
311
+
struct pool_workqueue __rcu *dfl_pwq; /* PW: only for unbound wqs */
312
312
313
313
#ifdef CONFIG_SYSFS
314
314
struct wq_device *wq_dev; /* I: for sysfs interface */
···
637
637
return 0;
638
638
}
639
639
return ret;
640
+
}
641
+
642
+
static struct pool_workqueue __rcu **
643
+
unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
644
+
{
645
+
if (cpu >= 0)
646
+
return per_cpu_ptr(wq->cpu_pwq, cpu);
647
+
else
648
+
return &wq->dfl_pwq;
649
+
}
650
+
651
+
/* @cpu < 0 for dfl_pwq */
652
+
static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
653
+
{
654
+
return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
655
+
lockdep_is_held(&wq_pool_mutex) ||
656
+
lockdep_is_held(&wq->mutex));
640
657
}
641
658
642
659
static unsigned int work_color_to_flags(int color)
···
4345
4328
"possible intersect\n");
4346
4329
}
4347
4330
4348
-
/* install @pwq into @wq's cpu_pwq and return the old pwq */
4331
+
/* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
4349
4332
static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
4350
4333
int cpu, struct pool_workqueue *pwq)
4351
4334
{
4335
+
struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
4352
4336
struct pool_workqueue *old_pwq;
4353
4337
4354
4338
lockdep_assert_held(&wq_pool_mutex);
···
4358
4340
/* link_pwq() can handle duplicate calls */
4359
4341
link_pwq(pwq);
4360
4342
4361
-
old_pwq = rcu_access_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu));
4362
-
rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq);
4343
+
old_pwq = rcu_access_pointer(*slot);
4344
+
rcu_assign_pointer(*slot, pwq);
4363
4345
return old_pwq;
4364
4346
}
4365
4347
···
4459
4441
4460
4442
copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
4461
4443
4462
-
/* save the previous pwq and install the new one */
4444
+
/* save the previous pwqs and install the new ones */
4463
4445
for_each_possible_cpu(cpu)
4464
4446
ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
4465
4447
ctx->pwq_tbl[cpu]);
4466
-
4467
-
/* @dfl_pwq might not have been used, ensure it's linked */
4468
-
link_pwq(ctx->dfl_pwq);
4469
-
swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
4448
+
ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);
4470
4449
4471
4450
mutex_unlock(&ctx->wq->mutex);
4472
4451
}
···
4573
4558
4574
4559
/* nothing to do if the target cpumask matches the current pwq */
4575
4560
wq_calc_pod_cpumask(target_attrs, cpu, off_cpu);
4576
-
pwq = rcu_dereference_protected(*per_cpu_ptr(wq->cpu_pwq, cpu),
4577
-
lockdep_is_held(&wq_pool_mutex));
4578
-
if (wqattrs_equal(target_attrs, pwq->pool->attrs))
4561
+
if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
4579
4562
return;
4580
4563
4581
4564
/* create a new pwq */
···
4591
4578
4592
4579
use_dfl_pwq:
4593
4580
mutex_lock(&wq->mutex);
4594
-
raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
4595
-
get_pwq(wq->dfl_pwq);
4596
-
raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4597
-
old_pwq = install_unbound_pwq(wq, cpu, wq->dfl_pwq);
4581
+
pwq = unbound_pwq(wq, -1);
4582
+
raw_spin_lock_irq(&pwq->pool->lock);
4583
+
get_pwq(pwq);
4584
+
raw_spin_unlock_irq(&pwq->pool->lock);
4585
+
old_pwq = install_unbound_pwq(wq, cpu, pwq);
4598
4586
out_unlock:
4599
4587
mutex_unlock(&wq->mutex);
4600
4588
put_pwq_unlocked(old_pwq);
···
4633
4619
4634
4620
cpus_read_lock();
4635
4621
if (wq->flags & __WQ_ORDERED) {
4622
+
struct pool_workqueue *dfl_pwq;
4623
+
4636
4624
ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
4637
4625
/* there should only be single pwq for ordering guarantee */
4638
-
WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
4639
-
wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
4626
+
dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
4627
+
WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
4628
+
wq->pwqs.prev != &dfl_pwq->pwqs_node),
4640
4629
"ordering guarantee broken for workqueue %s\n", wq->name);
4641
4630
} else {
4642
4631
ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
···
4873
4856
if (pwq->nr_in_flight[i])
4874
4857
return true;
4875
4858
4876
-
if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
4859
+
if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
4877
4860
return true;
4878
4861
if (!pwq_is_empty(pwq))
4879
4862
return true;
···
4957
4940
rcu_read_lock();
4958
4941
4959
4942
for_each_possible_cpu(cpu) {
4960
-
pwq = rcu_access_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu));
4961
-
RCU_INIT_POINTER(*per_cpu_ptr(wq->cpu_pwq, cpu), NULL);
4962
-
put_pwq_unlocked(pwq);
4943
+
put_pwq_unlocked(unbound_pwq(wq, cpu));
4944
+
RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
4963
4945
}
4964
4946
4965
-
put_pwq_unlocked(wq->dfl_pwq);
4966
-
wq->dfl_pwq = NULL;
4947
+
put_pwq_unlocked(unbound_pwq(wq, -1));
4948
+
RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);
4967
4949
4968
4950
rcu_read_unlock();
4969
4951
}