tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge tag 'sched-urgent-2021-03-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fixes from Thomas Gleixner:
"A set of scheduler updates:

- Prevent a NULL pointer dereference in the migration_stop_cpu()
mechanims

- Prevent self concurrency of affine_move_task()

- Small fixes and cleanups related to task migration/affinity setting

- Ensure that sync_runqueues_membarrier_state() is invoked on the
current CPU when it is in the cpu mask"

* tag 'sched-urgent-2021-03-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
sched/membarrier: fix missing local execution of ipi_sync_rq_state()
sched: Simplify set_affinity_pending refcounts
sched: Fix affine_move_task() self-concurrency
sched: Optimize migration_cpu_stop()
sched: Collate affine_move_task() stoppers
sched: Simplify migration_cpu_stop()
sched: Fix migration_cpu_stop() requeueing

Linus Torvalds c72cbc93 19469d2a

+64 -68
+63 -65
kernel/sched/core.c
··· 1862 1862 struct set_affinity_pending *pending; 1863 1863 }; 1864 1864 1865 + /* 1866 + * @refs: number of wait_for_completion() 1867 + * @stop_pending: is @stop_work in use 1868 + */ 1865 1869 struct set_affinity_pending { 1866 1870 refcount_t refs; 1871 + unsigned int stop_pending; 1867 1872 struct completion done; 1868 1873 struct cpu_stop_work stop_work; 1869 1874 struct migration_arg arg; ··· 1903 1898 */ 1904 1899 static int migration_cpu_stop(void *data) 1905 1900 { 1906 - struct set_affinity_pending *pending; 1907 1901 struct migration_arg *arg = data; 1902 + struct set_affinity_pending *pending = arg->pending; 1908 1903 struct task_struct *p = arg->task; 1909 1904 int dest_cpu = arg->dest_cpu; 1910 1905 struct rq *rq = this_rq(); ··· 1926 1921 raw_spin_lock(&p->pi_lock); 1927 1922 rq_lock(rq, &rf); 1928 1923 1929 - pending = p->migration_pending; 1930 1924 /* 1931 1925 * If task_rq(p) != rq, it cannot be migrated here, because we're 1932 1926 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because ··· 1936 1932 goto out; 1937 1933 1938 1934 if (pending) { 1939 - p->migration_pending = NULL; 1935 + if (p->migration_pending == pending) 1936 + p->migration_pending = NULL; 1940 1937 complete = true; 1941 1938 } 1942 1939 1943 - /* migrate_enable() -- we must not race against SCA */ 1944 1940 if (dest_cpu < 0) { 1945 - /* 1946 - * When this was migrate_enable() but we no longer 1947 - * have a @pending, a concurrent SCA 'fixed' things 1948 - * and we should be valid again. Nothing to do. 1949 - */ 1950 - if (!pending) { 1951 - WARN_ON_ONCE(!cpumask_test_cpu(task_cpu(p), &p->cpus_mask)); 1941 + if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) 1952 1942 goto out; 1953 - } 1954 1943 1955 1944 dest_cpu = cpumask_any_distribute(&p->cpus_mask); 1956 1945 } ··· 1953 1956 else 1954 1957 p->wake_cpu = dest_cpu; 1955 1958 1956 - } else if (dest_cpu < 0 || pending) { 1959 + /* 1960 + * XXX __migrate_task() can fail, at which point we might end 1961 + * up running on a dodgy CPU, AFAICT this can only happen 1962 + * during CPU hotplug, at which point we'll get pushed out 1963 + * anyway, so it's probably not a big deal. 1964 + */ 1965 + 1966 + } else if (pending) { 1957 1967 /* 1958 1968 * This happens when we get migrated between migrate_enable()'s 1959 1969 * preempt_enable() and scheduling the stopper task. At that ··· 1975 1971 * ->pi_lock, so the allowed mask is stable - if it got 1976 1972 * somewhere allowed, we're done. 1977 1973 */ 1978 - if (pending && cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) { 1979 - p->migration_pending = NULL; 1974 + if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) { 1975 + if (p->migration_pending == pending) 1976 + p->migration_pending = NULL; 1980 1977 complete = true; 1981 - goto out; 1982 - } 1983 - 1984 - /* 1985 - * When this was migrate_enable() but we no longer have an 1986 - * @pending, a concurrent SCA 'fixed' things and we should be 1987 - * valid again. Nothing to do. 1988 - */ 1989 - if (!pending) { 1990 - WARN_ON_ONCE(!cpumask_test_cpu(task_cpu(p), &p->cpus_mask)); 1991 1978 goto out; 1992 1979 } 1993 1980 ··· 1987 1992 * determine is_migration_disabled() and so have to chase after 1988 1993 * it. 1989 1994 */ 1995 + WARN_ON_ONCE(!pending->stop_pending); 1990 1996 task_rq_unlock(rq, p, &rf); 1991 1997 stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop, 1992 1998 &pending->arg, &pending->stop_work); 1993 1999 return 0; 1994 2000 } 1995 2001 out: 2002 + if (pending) 2003 + pending->stop_pending = false; 1996 2004 task_rq_unlock(rq, p, &rf); 1997 2005 1998 2006 if (complete) 1999 2007 complete_all(&pending->done); 2000 - 2001 - /* For pending->{arg,stop_work} */ 2002 - pending = arg->pending; 2003 - if (pending && refcount_dec_and_test(&pending->refs)) 2004 - wake_up_var(&pending->refs); 2005 2008 2006 2009 return 0; 2007 2010 } ··· 2187 2194 int dest_cpu, unsigned int flags) 2188 2195 { 2189 2196 struct set_affinity_pending my_pending = { }, *pending = NULL; 2190 - struct migration_arg arg = { 2191 - .task = p, 2192 - .dest_cpu = dest_cpu, 2193 - }; 2194 - bool complete = false; 2197 + bool stop_pending, complete = false; 2195 2198 2196 2199 /* Can the task run on the task's current CPU? If so, we're done */ 2197 2200 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) { ··· 2199 2210 push_task = get_task_struct(p); 2200 2211 } 2201 2212 2213 + /* 2214 + * If there are pending waiters, but no pending stop_work, 2215 + * then complete now. 2216 + */ 2202 2217 pending = p->migration_pending; 2203 - if (pending) { 2204 - refcount_inc(&pending->refs); 2218 + if (pending && !pending->stop_pending) { 2205 2219 p->migration_pending = NULL; 2206 2220 complete = true; 2207 2221 } 2222 + 2208 2223 task_rq_unlock(rq, p, rf); 2209 2224 2210 2225 if (push_task) { ··· 2217 2224 } 2218 2225 2219 2226 if (complete) 2220 - goto do_complete; 2227 + complete_all(&pending->done); 2221 2228 2222 2229 return 0; 2223 2230 } ··· 2228 2235 /* Install the request */ 2229 2236 refcount_set(&my_pending.refs, 1); 2230 2237 init_completion(&my_pending.done); 2238 + my_pending.arg = (struct migration_arg) { 2239 + .task = p, 2240 + .dest_cpu = -1, /* any */ 2241 + .pending = &my_pending, 2242 + }; 2243 + 2231 2244 p->migration_pending = &my_pending; 2232 2245 } else { 2233 2246 pending = p->migration_pending; ··· 2258 2259 return -EINVAL; 2259 2260 } 2260 2261 2261 - if (flags & SCA_MIGRATE_ENABLE) { 2262 - 2263 - refcount_inc(&pending->refs); /* pending->{arg,stop_work} */ 2264 - p->migration_flags &= ~MDF_PUSH; 2265 - task_rq_unlock(rq, p, rf); 2266 - 2267 - pending->arg = (struct migration_arg) { 2268 - .task = p, 2269 - .dest_cpu = -1, 2270 - .pending = pending, 2271 - }; 2272 - 2273 - stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, 2274 - &pending->arg, &pending->stop_work); 2275 - 2276 - return 0; 2277 - } 2278 - 2279 2262 if (task_running(rq, p) || p->state == TASK_WAKING) { 2280 2263 /* 2281 - * Lessen races (and headaches) by delegating 2282 - * is_migration_disabled(p) checks to the stopper, which will 2283 - * run on the same CPU as said p. 2264 + * MIGRATE_ENABLE gets here because 'p == current', but for 2265 + * anything else we cannot do is_migration_disabled(), punt 2266 + * and have the stopper function handle it all race-free. 2284 2267 */ 2285 - task_rq_unlock(rq, p, rf); 2286 - stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); 2268 + stop_pending = pending->stop_pending; 2269 + if (!stop_pending) 2270 + pending->stop_pending = true; 2287 2271 2272 + if (flags & SCA_MIGRATE_ENABLE) 2273 + p->migration_flags &= ~MDF_PUSH; 2274 + 2275 + task_rq_unlock(rq, p, rf); 2276 + 2277 + if (!stop_pending) { 2278 + stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, 2279 + &pending->arg, &pending->stop_work); 2280 + } 2281 + 2282 + if (flags & SCA_MIGRATE_ENABLE) 2283 + return 0; 2288 2284 } else { 2289 2285 2290 2286 if (!is_migration_disabled(p)) { 2291 2287 if (task_on_rq_queued(p)) 2292 2288 rq = move_queued_task(rq, rf, p, dest_cpu); 2293 2289 2294 - p->migration_pending = NULL; 2295 - complete = true; 2290 + if (!pending->stop_pending) { 2291 + p->migration_pending = NULL; 2292 + complete = true; 2293 + } 2296 2294 } 2297 2295 task_rq_unlock(rq, p, rf); 2298 2296 2299 - do_complete: 2300 2297 if (complete) 2301 2298 complete_all(&pending->done); 2302 2299 } ··· 2300 2305 wait_for_completion(&pending->done); 2301 2306 2302 2307 if (refcount_dec_and_test(&pending->refs)) 2303 - wake_up_var(&pending->refs); 2308 + wake_up_var(&pending->refs); /* No UaF, just an address */ 2304 2309 2305 2310 /* 2306 2311 * Block the original owner of &pending until all subsequent callers 2307 2312 * have seen the completion and decremented the refcount 2308 2313 */ 2309 2314 wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs)); 2315 + 2316 + /* ARGH */ 2317 + WARN_ON_ONCE(my_pending.stop_pending); 2310 2318 2311 2319 return 0; 2312 2320 }
+1 -3
kernel/sched/membarrier.c
··· 471 471 } 472 472 rcu_read_unlock(); 473 473 474 - preempt_disable(); 475 - smp_call_function_many(tmpmask, ipi_sync_rq_state, mm, 1); 476 - preempt_enable(); 474 + on_each_cpu_mask(tmpmask, ipi_sync_rq_state, mm, true); 477 475 478 476 free_cpumask_var(tmpmask); 479 477 cpus_read_unlock();