Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'wq-for-6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq
Pull workqueue updates from Tejun Heo:
- Rescuer affinity management: Affinity is now updated only when
detached using wq_unbound_cpumask consistently. DISASSOCIATED workers
also follow unbound cpumask changes to avoid breaking CPU isolation
- Rescuer cleanups preparing for fetching work items one by one from
pool list: Work assignment factored out, optimized to skip pwqs no
longer needing rescue, and shutdown logic simplified
- Unused assert_rcu_or_wq_mutex_or_pool_mutex() removed
* tag 'wq-for-6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq:
workqueue: Don't rely on wq->rescuer to stop rescuer
workqueue: Only assign rescuer work when really needed
workqueue: Factor out assign_rescuer_work()
workqueue: Init rescuer's affinities as wq_unbound_cpumask
workqueue: Let DISASSOCIATED workers follow unbound wq cpumask changes
workqueue: Update the rescuer's affinity only when it is detached
workqueue: Remove unused assert_rcu_or_wq_mutex_or_pool_mutex
+50
-36
+50
-36
kernel/workqueue.c
+50
-36
kernel/workqueue.c
···
541
541
!lockdep_is_held(&wq_pool_mutex), \
542
542
"RCU or wq_pool_mutex should be held")
543
543
544
-
#define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
545
-
RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() && \
546
-
!lockdep_is_held(&wq->mutex) && \
547
-
!lockdep_is_held(&wq_pool_mutex), \
548
-
"RCU, wq->mutex or wq_pool_mutex should be held")
549
-
550
544
#define for_each_bh_worker_pool(pool, cpu) \
551
545
for ((pool) = &per_cpu(bh_worker_pools, cpu)[0]; \
552
546
(pool) < &per_cpu(bh_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
···
3437
3443
goto woke_up;
3438
3444
}
3439
3445
3446
+
static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescuer)
3447
+
{
3448
+
struct worker_pool *pool = pwq->pool;
3449
+
struct work_struct *work, *n;
3450
+
3451
+
/* need rescue? */
3452
+
if (!pwq->nr_active || !need_to_create_worker(pool))
3453
+
return false;
3454
+
3455
+
/*
3456
+
* Slurp in all works issued via this workqueue and
3457
+
* process'em.
3458
+
*/
3459
+
list_for_each_entry_safe(work, n, &pool->worklist, entry) {
3460
+
if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n))
3461
+
pwq->stats[PWQ_STAT_RESCUED]++;
3462
+
}
3463
+
3464
+
return !list_empty(&rescuer->scheduled);
3465
+
}
3466
+
3440
3467
/**
3441
3468
* rescuer_thread - the rescuer thread function
3442
3469
* @__rescuer: self
···
3512
3497
struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
3513
3498
struct pool_workqueue, mayday_node);
3514
3499
struct worker_pool *pool = pwq->pool;
3515
-
struct work_struct *work, *n;
3516
3500
3517
3501
__set_current_state(TASK_RUNNING);
3518
3502
list_del_init(&pwq->mayday_node);
···
3522
3508
3523
3509
raw_spin_lock_irq(&pool->lock);
3524
3510
3525
-
/*
3526
-
* Slurp in all works issued via this workqueue and
3527
-
* process'em.
3528
-
*/
3529
3511
WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
3530
-
list_for_each_entry_safe(work, n, &pool->worklist, entry) {
3531
-
if (get_work_pwq(work) == pwq &&
3532
-
assign_work(work, rescuer, &n))
3533
-
pwq->stats[PWQ_STAT_RESCUED]++;
3534
-
}
3535
3512
3536
-
if (!list_empty(&rescuer->scheduled)) {
3513
+
if (assign_rescuer_work(pwq, rescuer)) {
3537
3514
process_scheduled_works(rescuer);
3538
3515
3539
3516
/*
···
3539
3534
if (pwq->nr_active && need_to_create_worker(pool)) {
3540
3535
raw_spin_lock(&wq_mayday_lock);
3541
3536
/*
3542
-
* Queue iff we aren't racing destruction
3543
-
* and somebody else hasn't queued it already.
3537
+
* Queue iff somebody else hasn't queued it already.
3544
3538
*/
3545
-
if (wq->rescuer && list_empty(&pwq->mayday_node)) {
3539
+
if (list_empty(&pwq->mayday_node)) {
3546
3540
get_pwq(pwq);
3547
3541
list_add_tail(&pwq->mayday_node, &wq->maydays);
3548
3542
}
···
5380
5376
/* update node_nr_active->max */
5381
5377
wq_update_node_max_active(ctx->wq, -1);
5382
5378
5383
-
/* rescuer needs to respect wq cpumask changes */
5384
-
if (ctx->wq->rescuer)
5385
-
set_cpus_allowed_ptr(ctx->wq->rescuer->task,
5386
-
unbound_effective_cpumask(ctx->wq));
5387
-
5388
5379
mutex_unlock(&ctx->wq->mutex);
5389
5380
}
5390
5381
···
5613
5614
}
5614
5615
5615
5616
wq->rescuer = rescuer;
5616
-
if (wq->flags & WQ_UNBOUND)
5617
-
kthread_bind_mask(rescuer->task, unbound_effective_cpumask(wq));
5617
+
5618
+
/* initial cpumask is consistent with the detached rescuer and unbind_worker() */
5619
+
if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))
5620
+
kthread_bind_mask(rescuer->task, wq_unbound_cpumask);
5618
5621
else
5619
5622
kthread_bind_mask(rescuer->task, cpu_possible_mask);
5623
+
5620
5624
wake_up_process(rescuer->task);
5621
5625
5622
5626
return 0;
···
5904
5902
5905
5903
/* kill rescuer, if sanity checks fail, leave it w/o rescuer */
5906
5904
if (wq->rescuer) {
5907
-
struct worker *rescuer = wq->rescuer;
5908
-
5909
-
/* this prevents new queueing */
5910
-
raw_spin_lock_irq(&wq_mayday_lock);
5911
-
wq->rescuer = NULL;
5912
-
raw_spin_unlock_irq(&wq_mayday_lock);
5913
-
5914
5905
/* rescuer will empty maydays list before exiting */
5915
-
kthread_stop(rescuer->task);
5916
-
kfree(rescuer);
5906
+
kthread_stop(wq->rescuer->task);
5907
+
kfree(wq->rescuer);
5908
+
wq->rescuer = NULL;
5917
5909
}
5918
5910
5919
5911
/*
···
6933
6937
}
6934
6938
6935
6939
if (!ret) {
6940
+
int cpu;
6941
+
struct worker_pool *pool;
6942
+
struct worker *worker;
6943
+
6936
6944
mutex_lock(&wq_pool_attach_mutex);
6937
6945
cpumask_copy(wq_unbound_cpumask, unbound_cpumask);
6946
+
/* rescuer needs to respect cpumask changes when it is not attached */
6947
+
list_for_each_entry(wq, &workqueues, list) {
6948
+
if (wq->rescuer && !wq->rescuer->pool)
6949
+
unbind_worker(wq->rescuer);
6950
+
}
6951
+
/* DISASSOCIATED worker needs to respect wq_unbound_cpumask */
6952
+
for_each_possible_cpu(cpu) {
6953
+
for_each_cpu_worker_pool(pool, cpu) {
6954
+
if (!(pool->flags & POOL_DISASSOCIATED))
6955
+
continue;
6956
+
for_each_pool_worker(worker, pool)
6957
+
unbind_worker(worker);
6958
+
}
6959
+
}
6938
6960
mutex_unlock(&wq_pool_attach_mutex);
6939
6961
}
6940
6962
return ret;