Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
i387: do not preload FPU state at task switch time
Yes, taking the trap to re-load the FPU/MMX state is expensive, but so
is spending several days looking for a bug in the state save/restore
code. And the preload code has some rather subtle interactions with
both paravirtualization support and segment state restore, so it's not
nearly as simple as it should be.
Also, now that we no longer necessarily depend on a single bit (ie
TS_USEDFPU) for keeping track of the state of the FPU, we migth be able
to do better. If we are really switching between two processes that
keep touching the FP state, save/restore is inevitable, but in the case
of having one process that does most of the FPU usage, we may actually
be able to do much better than the preloading.
In particular, we may be able to keep track of which CPU the process ran
on last, and also per CPU keep track of which process' FP state that CPU
has. For modern CPU's that don't destroy the FPU contents on save time,
that would allow us to do a lazy restore by just re-enabling the
existing FPU state - with no restore cost at all!
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
+11
-68
-1
arch/x86/include/asm/i387.h
-1
arch/x86/include/asm/i387.h
···
30
30
extern void mxcsr_feature_mask_init(void);
31
31
extern int init_fpu(struct task_struct *child);
32
32
extern void math_state_restore(void);
33
-
extern void __math_state_restore(void);
34
33
extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
35
34
36
35
extern user_regset_active_fn fpregs_active, xfpregs_active;
-20
arch/x86/kernel/process_32.c
-20
arch/x86/kernel/process_32.c
···
299
299
*next = &next_p->thread;
300
300
int cpu = smp_processor_id();
301
301
struct tss_struct *tss = &per_cpu(init_tss, cpu);
302
-
bool preload_fpu;
303
302
304
303
/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
305
304
306
-
/*
307
-
* If the task has used fpu the last 5 timeslices, just do a full
308
-
* restore of the math state immediately to avoid the trap; the
309
-
* chances of needing FPU soon are obviously high now
310
-
*/
311
-
preload_fpu = tsk_used_math(next_p) && next_p->fpu_counter > 5;
312
-
313
305
__unlazy_fpu(prev_p);
314
-
315
-
/* we're going to use this soon, after a few expensive things */
316
-
if (preload_fpu)
317
-
prefetch(next->fpu.state);
318
306
319
307
/*
320
308
* Reload esp0.
···
342
354
task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT))
343
355
__switch_to_xtra(prev_p, next_p, tss);
344
356
345
-
/* If we're going to preload the fpu context, make sure clts
346
-
is run while we're batching the cpu state updates. */
347
-
if (preload_fpu)
348
-
clts();
349
-
350
357
/*
351
358
* Leave lazy mode, flushing any hypercalls made here.
352
359
* This must be done before restoring TLS segments so
···
350
367
* to date.
351
368
*/
352
369
arch_end_context_switch(next_p);
353
-
354
-
if (preload_fpu)
355
-
__math_state_restore();
356
370
357
371
/*
358
372
* Restore %gs if needed (which is common)
-23
arch/x86/kernel/process_64.c
-23
arch/x86/kernel/process_64.c
···
386
386
int cpu = smp_processor_id();
387
387
struct tss_struct *tss = &per_cpu(init_tss, cpu);
388
388
unsigned fsindex, gsindex;
389
-
bool preload_fpu;
390
-
391
-
/*
392
-
* If the task has used fpu the last 5 timeslices, just do a full
393
-
* restore of the math state immediately to avoid the trap; the
394
-
* chances of needing FPU soon are obviously high now
395
-
*/
396
-
preload_fpu = tsk_used_math(next_p) && next_p->fpu_counter > 5;
397
-
398
-
/* we're going to use this soon, after a few expensive things */
399
-
if (preload_fpu)
400
-
prefetch(next->fpu.state);
401
389
402
390
/*
403
391
* Reload esp0, LDT and the page table pointer:
···
417
429
418
430
/* Must be after DS reload */
419
431
__unlazy_fpu(prev_p);
420
-
421
-
/* Make sure cpu is ready for new context */
422
-
if (preload_fpu)
423
-
clts();
424
432
425
433
/*
426
434
* Leave lazy mode, flushing any hypercalls made here.
···
475
491
if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
476
492
task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
477
493
__switch_to_xtra(prev_p, next_p, tss);
478
-
479
-
/*
480
-
* Preload the FPU context, now that we've determined that the
481
-
* task is likely to be using it.
482
-
*/
483
-
if (preload_fpu)
484
-
__math_state_restore();
485
494
486
495
return prev_p;
487
496
}
+11
-24
arch/x86/kernel/traps.c
+11
-24
arch/x86/kernel/traps.c
···
571
571
}
572
572
573
573
/*
574
-
* __math_state_restore assumes that cr0.TS is already clear and the
575
-
* fpu state is all ready for use. Used during context switch.
576
-
*/
577
-
void __math_state_restore(void)
578
-
{
579
-
struct thread_info *thread = current_thread_info();
580
-
struct task_struct *tsk = thread->task;
581
-
582
-
/*
583
-
* Paranoid restore. send a SIGSEGV if we fail to restore the state.
584
-
*/
585
-
if (unlikely(restore_fpu_checking(tsk))) {
586
-
stts();
587
-
force_sig(SIGSEGV, tsk);
588
-
return;
589
-
}
590
-
591
-
__thread_set_has_fpu(thread); /* clts in caller! */
592
-
tsk->fpu_counter++;
593
-
}
594
-
595
-
/*
596
574
* 'math_state_restore()' saves the current math information in the
597
575
* old math state array, and gets the new ones from the current task
598
576
*
···
600
622
local_irq_disable();
601
623
}
602
624
603
-
clts(); /* Allow maths ops (or we recurse) */
625
+
__thread_fpu_begin(thread);
604
626
605
-
__math_state_restore();
627
+
/*
628
+
* Paranoid restore. send a SIGSEGV if we fail to restore the state.
629
+
*/
630
+
if (unlikely(restore_fpu_checking(tsk))) {
631
+
__thread_fpu_end(thread);
632
+
force_sig(SIGSEGV, tsk);
633
+
return;
634
+
}
635
+
636
+
tsk->fpu_counter++;
606
637
}
607
638
EXPORT_SYMBOL_GPL(math_state_restore);
608
639