Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
i387: don't ever touch TS_USEDFPU directly, use helper functions
This creates three helper functions that do the TS_USEDFPU accesses, and
makes everybody that used to do it by hand use those helpers instead.
In addition, there's a couple of helper functions for the "change both
CR0.TS and TS_USEDFPU at the same time" case, and the places that do
that together have been changed to use those. That means that we have
fewer random places that open-code this situation.
The intent is partly to clarify the code without actually changing any
semantics yet (since we clearly still have some hard to reproduce bug in
this area), but also to make it much easier to use another approach
entirely to caching the CR0.TS bit for software accesses.
Right now we use a bit in the thread-info 'status' variable (this patch
does not change that), but we might want to make it a full field of its
own or even make it a per-cpu variable.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
+58
-23
+55
-20
arch/x86/include/asm/i387.h
+55
-20
arch/x86/include/asm/i387.h
···
280
280
}
281
281
282
282
/*
283
+
* Software FPU state helpers. Careful: these need to
284
+
* be preemption protection *and* they need to be
285
+
* properly paired with the CR0.TS changes!
286
+
*/
287
+
static inline int __thread_has_fpu(struct thread_info *ti)
288
+
{
289
+
return ti->status & TS_USEDFPU;
290
+
}
291
+
292
+
/* Must be paired with an 'stts' after! */
293
+
static inline void __thread_clear_has_fpu(struct thread_info *ti)
294
+
{
295
+
ti->status &= ~TS_USEDFPU;
296
+
}
297
+
298
+
/* Must be paired with a 'clts' before! */
299
+
static inline void __thread_set_has_fpu(struct thread_info *ti)
300
+
{
301
+
ti->status |= TS_USEDFPU;
302
+
}
303
+
304
+
/*
305
+
* Encapsulate the CR0.TS handling together with the
306
+
* software flag.
307
+
*
308
+
* These generally need preemption protection to work,
309
+
* do try to avoid using these on their own.
310
+
*/
311
+
static inline void __thread_fpu_end(struct thread_info *ti)
312
+
{
313
+
__thread_clear_has_fpu(ti);
314
+
stts();
315
+
}
316
+
317
+
static inline void __thread_fpu_begin(struct thread_info *ti)
318
+
{
319
+
clts();
320
+
__thread_set_has_fpu(ti);
321
+
}
322
+
323
+
/*
283
324
* Signal frame handlers...
284
325
*/
285
326
extern int save_i387_xstate(void __user *buf);
···
328
287
329
288
static inline void __unlazy_fpu(struct task_struct *tsk)
330
289
{
331
-
if (task_thread_info(tsk)->status & TS_USEDFPU) {
290
+
if (__thread_has_fpu(task_thread_info(tsk))) {
332
291
__save_init_fpu(tsk);
333
-
task_thread_info(tsk)->status &= ~TS_USEDFPU;
334
-
stts();
292
+
__thread_fpu_end(task_thread_info(tsk));
335
293
} else
336
294
tsk->fpu_counter = 0;
337
295
}
338
296
339
297
static inline void __clear_fpu(struct task_struct *tsk)
340
298
{
341
-
if (task_thread_info(tsk)->status & TS_USEDFPU) {
299
+
if (__thread_has_fpu(task_thread_info(tsk))) {
342
300
/* Ignore delayed exceptions from user space */
343
301
asm volatile("1: fwait\n"
344
302
"2:\n"
345
303
_ASM_EXTABLE(1b, 2b));
346
-
task_thread_info(tsk)->status &= ~TS_USEDFPU;
347
-
stts();
304
+
__thread_fpu_end(task_thread_info(tsk));
348
305
}
349
306
}
350
307
···
350
311
* Were we in an interrupt that interrupted kernel mode?
351
312
*
352
313
* We can do a kernel_fpu_begin/end() pair *ONLY* if that
353
-
* pair does nothing at all: TS_USEDFPU must be clear (so
314
+
* pair does nothing at all: the thread must not have fpu (so
354
315
* that we don't try to save the FPU state), and TS must
355
316
* be set (so that the clts/stts pair does nothing that is
356
317
* visible in the interrupted kernel thread).
357
318
*/
358
319
static inline bool interrupted_kernel_fpu_idle(void)
359
320
{
360
-
return !(current_thread_info()->status & TS_USEDFPU) &&
321
+
return !__thread_has_fpu(current_thread_info()) &&
361
322
(read_cr0() & X86_CR0_TS);
362
323
}
363
324
···
395
356
396
357
WARN_ON_ONCE(!irq_fpu_usable());
397
358
preempt_disable();
398
-
if (me->status & TS_USEDFPU) {
359
+
if (__thread_has_fpu(me)) {
399
360
__save_init_fpu(me->task);
400
-
me->status &= ~TS_USEDFPU;
361
+
__thread_clear_has_fpu(me);
401
362
/* We do 'stts()' in kernel_fpu_end() */
402
363
} else
403
364
clts();
···
461
422
*/
462
423
static inline int user_has_fpu(void)
463
424
{
464
-
return current_thread_info()->status & TS_USEDFPU;
425
+
return __thread_has_fpu(current_thread_info());
465
426
}
466
427
467
428
static inline void user_fpu_end(void)
468
429
{
469
430
preempt_disable();
470
-
current_thread_info()->status &= ~TS_USEDFPU;
471
-
stts();
431
+
__thread_fpu_end(current_thread_info());
472
432
preempt_enable();
473
433
}
474
434
475
435
static inline void user_fpu_begin(void)
476
436
{
477
437
preempt_disable();
478
-
if (!user_has_fpu()) {
479
-
clts();
480
-
current_thread_info()->status |= TS_USEDFPU;
481
-
}
438
+
if (!user_has_fpu())
439
+
__thread_fpu_begin(current_thread_info());
482
440
preempt_enable();
483
441
}
484
442
···
484
448
*/
485
449
static inline void save_init_fpu(struct task_struct *tsk)
486
450
{
487
-
WARN_ON_ONCE(!(task_thread_info(tsk)->status & TS_USEDFPU));
451
+
WARN_ON_ONCE(!__thread_has_fpu(task_thread_info(tsk)));
488
452
preempt_disable();
489
453
__save_init_fpu(tsk);
490
-
task_thread_info(tsk)->status &= ~TS_USEDFPU;
491
-
stts();
454
+
__thread_fpu_end(task_thread_info(tsk));
492
455
preempt_enable();
493
456
}
494
457
+1
-1
arch/x86/kernel/traps.c
+1
-1
arch/x86/kernel/traps.c
+1
-1
arch/x86/kernel/xsave.c
+1
-1
arch/x86/kernel/xsave.c
+1
-1
arch/x86/kvm/vmx.c
+1
-1
arch/x86/kvm/vmx.c
···
1457
1457
#ifdef CONFIG_X86_64
1458
1458
wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
1459
1459
#endif
1460
-
if (current_thread_info()->status & TS_USEDFPU)
1460
+
if (__thread_has_fpu(current_thread_info()))
1461
1461
clts();
1462
1462
load_gdt(&__get_cpu_var(host_gdt));
1463
1463
}