Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex
Prepare for the new CID management scheme which puts the CID ownership
transition into the fork() and exit() slow path by serializing
sched_mm_cid_fork()/exit() with it, so task list and cpu mask walks can be
done in interruptible and preemptible code.
The contention on it is not worse than on other concurrency controls in the
fork()/exit() machinery.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.895826703@linutronix.de
+24
+2
include/linux/rseq_types.h
+2
include/linux/rseq_types.h
···
125
125
* do not actually share the MM.
126
126
* @lock: Spinlock to protect all fields except @pcpu. It also protects
127
127
* the MM cid cpumask and the MM cidmask bitmap.
128
+
* @mutex: Mutex to serialize forks and exits related to this mm
128
129
*/
129
130
struct mm_mm_cid {
130
131
struct mm_cid_pcpu __percpu *pcpu;
···
133
132
unsigned int nr_cpus_allowed;
134
133
unsigned int users;
135
134
raw_spinlock_t lock;
135
+
struct mutex mutex;
136
136
}____cacheline_aligned_in_smp;
137
137
#else /* CONFIG_SCHED_MM_CID */
138
138
struct mm_mm_cid { };
+22
kernel/sched/core.c
+22
kernel/sched/core.c
···
10370
10370
10371
10371
#ifdef CONFIG_SCHED_MM_CID
10372
10372
/*
10373
+
* Concurrency IDentifier management
10374
+
*
10375
+
* Serialization rules:
10376
+
*
10377
+
* mm::mm_cid::mutex: Serializes fork() and exit() and therefore
10378
+
* protects mm::mm_cid::users.
10379
+
*
10380
+
* mm::mm_cid::lock: Serializes mm_update_max_cids() and
10381
+
* mm_update_cpus_allowed(). Nests in mm_cid::mutex
10382
+
* and runqueue lock.
10383
+
*
10384
+
* The mm_cidmask bitmap is not protected by any of the mm::mm_cid locks
10385
+
* and can only be modified with atomic operations.
10386
+
*
10387
+
* The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
10388
+
* lock.
10389
+
*/
10390
+
10391
+
/*
10373
10392
* Update the CID range properties when the constraints change. Invoked via
10374
10393
* fork(), exit() and affinity changes
10375
10394
*/
···
10431
10412
10432
10413
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
10433
10414
10415
+
guard(mutex)(&mm->mm_cid.mutex);
10434
10416
guard(raw_spinlock)(&mm->mm_cid.lock);
10435
10417
t->mm_cid.active = 1;
10436
10418
mm->mm_cid.users++;
···
10451
10431
if (!mm || !t->mm_cid.active)
10452
10432
return;
10453
10433
10434
+
guard(mutex)(&mm->mm_cid.mutex);
10454
10435
guard(raw_spinlock)(&mm->mm_cid.lock);
10455
10436
t->mm_cid.active = 0;
10456
10437
mm->mm_cid.users--;
···
10488
10467
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
10489
10468
mm->mm_cid.users = 0;
10490
10469
raw_spin_lock_init(&mm->mm_cid.lock);
10470
+
mutex_init(&mm->mm_cid.mutex);
10491
10471
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
10492
10472
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
10493
10473
}