sched/mmcid: Use proper data structures

+14 -42

include/linux/mm_types.h

··· 20 20 #include <linux/seqlock.h> 21 21 #include <linux/percpu_counter.h> 22 22 #include <linux/types.h> 23 + #include <linux/rseq_types.h> 23 24 #include <linux/bitmap.h> 24 25 25 26 #include <asm/mmu.h> ··· 923 922 #define vma_policy(vma) NULL 924 923 #endif 925 924 926 - struct mm_cid { 927 - unsigned int cid; 928 - }; 929 - 930 925 /* 931 926 * Opaque type representing current mm_struct flag state. Must be accessed via 932 927 * mm_flags_xxx() helper functions. ··· 984 987 */ 985 988 atomic_t mm_users; 986 989 987 - #ifdef CONFIG_SCHED_MM_CID 988 - /** 989 - * @pcpu_cid: Per-cpu current cid. 990 - * 991 - * Keep track of the currently allocated mm_cid for each cpu. 992 - * The per-cpu mm_cid values are serialized by their respective 993 - * runqueue locks. 994 - */ 995 - struct mm_cid __percpu *pcpu_cid; 996 - /** 997 - * @nr_cpus_allowed: Number of CPUs allowed for mm. 998 - * 999 - * Number of CPUs allowed in the union of all mm's 1000 - * threads allowed CPUs. 1001 - */ 1002 - unsigned int nr_cpus_allowed; 1003 - /** 1004 - * @cpus_allowed_lock: Lock protecting mm cpus_allowed. 1005 - * 1006 - * Provide mutual exclusion for mm cpus_allowed and 1007 - * mm nr_cpus_allowed updates. 1008 - */ 1009 - raw_spinlock_t cpus_allowed_lock; 1010 - #endif 990 + /* MM CID related storage */ 991 + struct mm_mm_cid mm_cid; 992 + 1011 993 #ifdef CONFIG_MMU 1012 994 atomic_long_t pgtables_bytes; /* size of all page tables */ 1013 995 #endif ··· 1328 1352 } 1329 1353 1330 1354 #ifdef CONFIG_SCHED_MM_CID 1331 - 1332 - #define MM_CID_UNSET (~0U) 1333 - 1334 1355 /* 1335 1356 * mm_cpus_allowed: Union of all mm's threads allowed CPUs. 1336 1357 */ ··· 1356 1383 int i; 1357 1384 1358 1385 for_each_possible_cpu(i) { 1359 - struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i); 1386 + struct mm_cid_pcpu *pcpu = per_cpu_ptr(mm->mm_cid.pcpu, i); 1360 1387 1361 - pcpu_cid->cid = MM_CID_UNSET; 1388 + pcpu->cid = MM_CID_UNSET; 1362 1389 } 1363 - mm->nr_cpus_allowed = p->nr_cpus_allowed; 1364 - raw_spin_lock_init(&mm->cpus_allowed_lock); 1390 + mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed; 1391 + raw_spin_lock_init(&mm->mm_cid.lock); 1365 1392 cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask); 1366 1393 cpumask_clear(mm_cidmask(mm)); 1367 1394 } 1368 1395 1369 1396 static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p) 1370 1397 { 1371 - mm->pcpu_cid = alloc_percpu_noprof(struct mm_cid); 1372 - if (!mm->pcpu_cid) 1398 + mm->mm_cid.pcpu = alloc_percpu_noprof(struct mm_cid_pcpu); 1399 + if (!mm->mm_cid.pcpu) 1373 1400 return -ENOMEM; 1374 1401 mm_init_cid(mm, p); 1375 1402 return 0; ··· 1378 1405 1379 1406 static inline void mm_destroy_cid(struct mm_struct *mm) 1380 1407 { 1381 - free_percpu(mm->pcpu_cid); 1382 - mm->pcpu_cid = NULL; 1408 + free_percpu(mm->mm_cid.pcpu); 1409 + mm->mm_cid.pcpu = NULL; 1383 1410 } 1384 1411 1385 1412 static inline unsigned int mm_cid_size(void) ··· 1394 1421 if (!mm) 1395 1422 return; 1396 1423 /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */ 1397 - raw_spin_lock(&mm->cpus_allowed_lock); 1424 + guard(raw_spinlock)(&mm->mm_cid.lock); 1398 1425 cpumask_or(mm_allowed, mm_allowed, cpumask); 1399 - WRITE_ONCE(mm->nr_cpus_allowed, cpumask_weight(mm_allowed)); 1400 - raw_spin_unlock(&mm->cpus_allowed_lock); 1426 + WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed)); 1401 1427 } 1402 1428 #else /* CONFIG_SCHED_MM_CID */ 1403 1429 static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) { }

+42

include/linux/rseq_types.h

··· 90 90 struct rseq_data { }; 91 91 #endif /* !CONFIG_RSEQ */ 92 92 93 + #ifdef CONFIG_SCHED_MM_CID 94 + 95 + #define MM_CID_UNSET (~0U) 96 + 97 + /** 98 + * struct sched_mm_cid - Storage for per task MM CID data 99 + * @active: MM CID is active for the task 100 + * @cid: The CID associated to the task 101 + * @last_cid: The last CID associated to the task 102 + */ 103 + struct sched_mm_cid { 104 + unsigned int active; 105 + unsigned int cid; 106 + unsigned int last_cid; 107 + }; 108 + 109 + /** 110 + * struct mm_cid_pcpu - Storage for per CPU MM_CID data 111 + * @cid: The CID associated to the CPU 112 + */ 113 + struct mm_cid_pcpu { 114 + unsigned int cid; 115 + }; 116 + 117 + /** 118 + * struct mm_mm_cid - Storage for per MM CID data 119 + * @pcpu: Per CPU storage for CIDs associated to a CPU 120 + * @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map 121 + * is growth only. 122 + * @lock: Spinlock to protect all fields except @pcpu. It also protects 123 + * the MM cid cpumask and the MM cidmask bitmap. 124 + */ 125 + struct mm_mm_cid { 126 + struct mm_cid_pcpu __percpu *pcpu; 127 + unsigned int nr_cpus_allowed; 128 + raw_spinlock_t lock; 129 + }; 130 + #else /* CONFIG_SCHED_MM_CID */ 131 + struct mm_mm_cid { }; 132 + struct sched_mm_cid { }; 133 + #endif /* !CONFIG_SCHED_MM_CID */ 134 + 93 135 #endif

+2 -9

include/linux/sched.h

··· 1407 1407 #endif /* CONFIG_NUMA_BALANCING */ 1408 1408 1409 1409 struct rseq_data rseq; 1410 - 1411 - #ifdef CONFIG_SCHED_MM_CID 1412 - int mm_cid; /* Current cid in mm */ 1413 - int last_mm_cid; /* Most recent cid in mm */ 1414 - int migrate_from_cpu; 1415 - int mm_cid_active; /* Whether cid bitmap is active */ 1416 - struct callback_head cid_work; 1417 - #endif 1410 + struct sched_mm_cid mm_cid; 1418 1411 1419 1412 struct tlbflush_unmap_batch tlb_ubc; 1420 1413 ··· 2301 2308 void sched_mm_cid_exit_signals(struct task_struct *t); 2302 2309 static inline int task_mm_cid(struct task_struct *t) 2303 2310 { 2304 - return t->mm_cid; 2311 + return t->mm_cid.cid; 2305 2312 } 2306 2313 #else 2307 2314 static inline void sched_mm_cid_before_execve(struct task_struct *t) { }

+3

init/init_task.c

··· 223 223 #ifdef CONFIG_SECCOMP_FILTER 224 224 .seccomp = { .filter_count = ATOMIC_INIT(0) }, 225 225 #endif 226 + #ifdef CONFIG_SCHED_MM_CID 227 + .mm_cid = { .cid = MM_CID_UNSET, }, 228 + #endif 226 229 }; 227 230 EXPORT_SYMBOL(init_task); 228 231

+3 -3

kernel/fork.c

··· 955 955 #endif 956 956 957 957 #ifdef CONFIG_SCHED_MM_CID 958 - tsk->mm_cid = MM_CID_UNSET; 959 - tsk->last_mm_cid = MM_CID_UNSET; 960 - tsk->mm_cid_active = 0; 958 + tsk->mm_cid.cid = MM_CID_UNSET; 959 + tsk->mm_cid.last_cid = MM_CID_UNSET; 960 + tsk->mm_cid.active = 0; 961 961 #endif 962 962 return tsk; 963 963

+8 -8

kernel/sched/core.c

··· 10376 10376 { 10377 10377 struct mm_struct *mm = t->mm; 10378 10378 10379 - if (!mm || !t->mm_cid_active) 10379 + if (!mm || !t->mm_cid.active) 10380 10380 return; 10381 10381 10382 10382 guard(preempt)(); 10383 - t->mm_cid_active = 0; 10384 - if (t->mm_cid != MM_CID_UNSET) { 10385 - cpumask_clear_cpu(t->mm_cid, mm_cidmask(mm)); 10386 - t->mm_cid = MM_CID_UNSET; 10383 + t->mm_cid.active = 0; 10384 + if (t->mm_cid.cid != MM_CID_UNSET) { 10385 + cpumask_clear_cpu(t->mm_cid.cid, mm_cidmask(mm)); 10386 + t->mm_cid.cid = MM_CID_UNSET; 10387 10387 } 10388 10388 } 10389 10389 ··· 10402 10402 return; 10403 10403 10404 10404 guard(preempt)(); 10405 - t->mm_cid_active = 1; 10405 + t->mm_cid.active = 1; 10406 10406 mm_cid_select(t); 10407 10407 } 10408 10408 10409 10409 void sched_mm_cid_fork(struct task_struct *t) 10410 10410 { 10411 - WARN_ON_ONCE(!t->mm || t->mm_cid != MM_CID_UNSET); 10412 - t->mm_cid_active = 1; 10411 + WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET); 10412 + t->mm_cid.active = 1; 10413 10413 } 10414 10414 #endif /* CONFIG_SCHED_MM_CID */ 10415 10415

+13 -13

kernel/sched/sched.h

··· 3549 3549 return; 3550 3550 3551 3551 /* Preset last_mm_cid */ 3552 - max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users)); 3553 - t->last_mm_cid = max_cid - 1; 3552 + max_cid = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users)); 3553 + t->mm_cid.last_cid = max_cid - 1; 3554 3554 } 3555 3555 3556 3556 static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids) ··· 3561 3561 return false; 3562 3562 if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm))) 3563 3563 return false; 3564 - t->mm_cid = t->last_mm_cid = cid; 3565 - __this_cpu_write(mm->pcpu_cid->cid, cid); 3564 + t->mm_cid.cid = t->mm_cid.last_cid = cid; 3565 + __this_cpu_write(mm->mm_cid.pcpu->cid, cid); 3566 3566 return true; 3567 3567 } 3568 3568 ··· 3571 3571 struct mm_struct *mm = t->mm; 3572 3572 unsigned int max_cids; 3573 3573 3574 - max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users)); 3574 + max_cids = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users)); 3575 3575 3576 3576 /* Try to reuse the last CID of this task */ 3577 - if (__mm_cid_get(t, t->last_mm_cid, max_cids)) 3577 + if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids)) 3578 3578 return true; 3579 3579 3580 3580 /* Try to reuse the last CID of this mm on this CPU */ 3581 - if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids)) 3581 + if (__mm_cid_get(t, __this_cpu_read(mm->mm_cid.pcpu->cid), max_cids)) 3582 3582 return true; 3583 3583 3584 3584 /* Try the first zero bit in the cidmask. */ ··· 3601 3601 3602 3602 static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) 3603 3603 { 3604 - if (prev->mm_cid_active) { 3605 - if (prev->mm_cid != MM_CID_UNSET) 3606 - cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm)); 3607 - prev->mm_cid = MM_CID_UNSET; 3604 + if (prev->mm_cid.active) { 3605 + if (prev->mm_cid.cid != MM_CID_UNSET) 3606 + cpumask_clear_cpu(prev->mm_cid.cid, mm_cidmask(prev->mm)); 3607 + prev->mm_cid.cid = MM_CID_UNSET; 3608 3608 } 3609 3609 3610 - if (next->mm_cid_active) { 3610 + if (next->mm_cid.active) { 3611 3611 mm_cid_select(next); 3612 - rseq_sched_set_task_mm_cid(next, next->mm_cid); 3612 + rseq_sched_set_task_mm_cid(next, next->mm_cid.cid); 3613 3613 } 3614 3614 } 3615 3615

Configure Feed

Configure Feed