2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Copyright (C) 1995 Linus Torvalds
|
|
|
|
*
|
|
|
|
* Pentium III FXSR, SSE support
|
|
|
|
* Gareth Hughes <gareth@valinux.com>, May 2000
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This file handles the architecture-dependent parts of process handling..
|
|
|
|
*/
|
|
|
|
|
2005-06-25 21:54:50 +00:00
|
|
|
#include <linux/cpu.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/elfcore.h>
|
|
|
|
#include <linux/smp.h>
|
|
|
|
#include <linux/stddef.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/vmalloc.h>
|
|
|
|
#include <linux/user.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/delay.h>
|
|
|
|
#include <linux/reboot.h>
|
|
|
|
#include <linux/mc146818rtc.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/kallsyms.h>
|
|
|
|
#include <linux/ptrace.h>
|
2006-09-26 08:52:28 +00:00
|
|
|
#include <linux/personality.h>
|
2007-05-02 17:27:16 +00:00
|
|
|
#include <linux/percpu.h>
|
2008-04-13 22:24:18 +00:00
|
|
|
#include <linux/prctl.h>
|
2008-12-06 02:40:00 +00:00
|
|
|
#include <linux/ftrace.h>
|
2009-01-04 10:48:56 +00:00
|
|
|
#include <linux/uaccess.h>
|
|
|
|
#include <linux/io.h>
|
|
|
|
#include <linux/kdebug.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/ldt.h>
|
|
|
|
#include <asm/processor.h>
|
|
|
|
#include <asm/i387.h>
|
2012-02-21 21:19:22 +00:00
|
|
|
#include <asm/fpu-internal.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/desc.h>
|
|
|
|
#ifdef CONFIG_MATH_EMULATION
|
|
|
|
#include <asm/math_emu.h>
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#include <linux/err.h>
|
|
|
|
|
2005-06-25 21:54:50 +00:00
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
#include <asm/cpu.h>
|
2008-09-24 02:40:02 +00:00
|
|
|
#include <asm/idle.h>
|
2008-07-21 16:04:13 +00:00
|
|
|
#include <asm/syscalls.h>
|
2009-06-01 18:14:55 +00:00
|
|
|
#include <asm/debugreg.h>
|
2012-03-28 17:11:12 +00:00
|
|
|
#include <asm/switch_to.h>
|
2005-06-25 21:54:50 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
|
2012-09-10 20:44:54 +00:00
|
|
|
asmlinkage void ret_from_kernel_thread(void) __asm__("ret_from_kernel_thread");
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Return saved PC of a blocked thread.
|
|
|
|
*/
|
|
|
|
unsigned long thread_saved_pc(struct task_struct *tsk)
|
|
|
|
{
|
2008-01-30 12:31:02 +00:00
|
|
|
return ((unsigned long *)tsk->thread.sp)[3];
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2008-04-03 13:40:48 +00:00
|
|
|
void __show_regs(struct pt_regs *regs, int all)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L;
|
2007-07-21 15:10:42 +00:00
|
|
|
unsigned long d0, d1, d2, d3, d6, d7;
|
2008-01-30 12:30:56 +00:00
|
|
|
unsigned long sp;
|
2007-10-19 18:35:03 +00:00
|
|
|
unsigned short ss, gs;
|
|
|
|
|
|
|
|
if (user_mode_vm(regs)) {
|
2008-01-30 12:30:56 +00:00
|
|
|
sp = regs->sp;
|
|
|
|
ss = regs->ss & 0xffff;
|
2009-02-09 13:17:40 +00:00
|
|
|
gs = get_user_gs(regs);
|
2007-10-19 18:35:03 +00:00
|
|
|
} else {
|
2009-10-12 21:09:07 +00:00
|
|
|
sp = kernel_stack_pointer(regs);
|
2007-10-19 18:35:03 +00:00
|
|
|
savesegment(ss, ss);
|
|
|
|
savesegment(gs, gs);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-12-28 08:26:59 +00:00
|
|
|
printk(KERN_DEFAULT "EIP: %04x:[<%08lx>] EFLAGS: %08lx CPU: %d\n",
|
2008-02-08 20:09:56 +00:00
|
|
|
(u16)regs->cs, regs->ip, regs->flags,
|
2007-10-19 18:35:03 +00:00
|
|
|
smp_processor_id());
|
2008-01-30 12:30:56 +00:00
|
|
|
print_symbol("EIP is at %s\n", regs->ip);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-12-28 08:26:59 +00:00
|
|
|
printk(KERN_DEFAULT "EAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n",
|
2008-01-30 12:30:56 +00:00
|
|
|
regs->ax, regs->bx, regs->cx, regs->dx);
|
2009-12-28 08:26:59 +00:00
|
|
|
printk(KERN_DEFAULT "ESI: %08lx EDI: %08lx EBP: %08lx ESP: %08lx\n",
|
2008-01-30 12:30:56 +00:00
|
|
|
regs->si, regs->di, regs->bp, sp);
|
2009-12-28 08:26:59 +00:00
|
|
|
printk(KERN_DEFAULT " DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x\n",
|
2008-02-08 20:09:56 +00:00
|
|
|
(u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, ss);
|
2007-10-19 18:35:03 +00:00
|
|
|
|
|
|
|
if (!all)
|
|
|
|
return;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2005-09-03 22:56:36 +00:00
|
|
|
cr0 = read_cr0();
|
|
|
|
cr2 = read_cr2();
|
|
|
|
cr3 = read_cr3();
|
2014-10-24 22:58:08 +00:00
|
|
|
cr4 = __read_cr4_safe();
|
2009-12-28 08:26:59 +00:00
|
|
|
printk(KERN_DEFAULT "CR0: %08lx CR2: %08lx CR3: %08lx CR4: %08lx\n",
|
2007-10-19 18:35:03 +00:00
|
|
|
cr0, cr2, cr3, cr4);
|
2007-07-21 15:10:42 +00:00
|
|
|
|
|
|
|
get_debugreg(d0, 0);
|
|
|
|
get_debugreg(d1, 1);
|
|
|
|
get_debugreg(d2, 2);
|
|
|
|
get_debugreg(d3, 3);
|
|
|
|
get_debugreg(d6, 6);
|
|
|
|
get_debugreg(d7, 7);
|
2013-06-18 16:09:11 +00:00
|
|
|
|
|
|
|
/* Only print out debug registers if they are in their non-default state. */
|
|
|
|
if ((d0 == 0) && (d1 == 0) && (d2 == 0) && (d3 == 0) &&
|
|
|
|
(d6 == DR6_RESERVED) && (d7 == 0x400))
|
|
|
|
return;
|
|
|
|
|
|
|
|
printk(KERN_DEFAULT "DR0: %08lx DR1: %08lx DR2: %08lx DR3: %08lx\n",
|
|
|
|
d0, d1, d2, d3);
|
2009-12-28 08:26:59 +00:00
|
|
|
printk(KERN_DEFAULT "DR6: %08lx DR7: %08lx\n",
|
2007-10-19 18:35:03 +00:00
|
|
|
d6, d7);
|
|
|
|
}
|
2007-07-21 15:10:42 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
void release_thread(struct task_struct *dead_task)
|
|
|
|
{
|
2006-01-06 08:11:59 +00:00
|
|
|
BUG_ON(dead_task->mm);
|
2005-04-16 22:20:36 +00:00
|
|
|
release_vm86_irqs(dead_task);
|
|
|
|
}
|
|
|
|
|
2009-04-02 23:56:59 +00:00
|
|
|
int copy_thread(unsigned long clone_flags, unsigned long sp,
|
2012-10-23 02:51:14 +00:00
|
|
|
unsigned long arg, struct task_struct *p)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-09-10 20:44:54 +00:00
|
|
|
struct pt_regs *childregs = task_pt_regs(p);
|
2005-04-16 22:20:36 +00:00
|
|
|
struct task_struct *tsk;
|
|
|
|
int err;
|
|
|
|
|
2012-09-10 20:44:54 +00:00
|
|
|
p->thread.sp = (unsigned long) childregs;
|
|
|
|
p->thread.sp0 = (unsigned long) (childregs+1);
|
2014-09-02 17:57:33 +00:00
|
|
|
memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
|
2012-09-10 20:44:54 +00:00
|
|
|
|
2012-10-23 02:34:11 +00:00
|
|
|
if (unlikely(p->flags & PF_KTHREAD)) {
|
2012-09-10 20:44:54 +00:00
|
|
|
/* kernel thread */
|
|
|
|
memset(childregs, 0, sizeof(struct pt_regs));
|
|
|
|
p->thread.ip = (unsigned long) ret_from_kernel_thread;
|
|
|
|
task_user_gs(p) = __KERNEL_STACK_CANARY;
|
|
|
|
childregs->ds = __USER_DS;
|
|
|
|
childregs->es = __USER_DS;
|
|
|
|
childregs->fs = __KERNEL_PERCPU;
|
|
|
|
childregs->bx = sp; /* function */
|
|
|
|
childregs->bp = arg;
|
|
|
|
childregs->orig_ax = -1;
|
|
|
|
childregs->cs = __KERNEL_CS | get_kernel_rpl();
|
2013-04-27 23:10:11 +00:00
|
|
|
childregs->flags = X86_EFLAGS_IF | X86_EFLAGS_FIXED;
|
2012-09-10 20:44:54 +00:00
|
|
|
p->thread.io_bitmap_ptr = NULL;
|
|
|
|
return 0;
|
|
|
|
}
|
2012-10-23 02:34:11 +00:00
|
|
|
*childregs = *current_pt_regs();
|
2008-01-30 12:30:56 +00:00
|
|
|
childregs->ax = 0;
|
2012-10-23 02:34:11 +00:00
|
|
|
if (sp)
|
|
|
|
childregs->sp = sp;
|
2005-05-05 23:15:03 +00:00
|
|
|
|
2008-01-30 12:31:02 +00:00
|
|
|
p->thread.ip = (unsigned long) ret_from_fork;
|
2012-10-23 02:34:11 +00:00
|
|
|
task_user_gs(p) = get_user_gs(current_pt_regs());
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-06-01 18:14:55 +00:00
|
|
|
p->thread.io_bitmap_ptr = NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
tsk = current;
|
2009-06-01 18:14:55 +00:00
|
|
|
err = -ENOMEM;
|
2009-09-09 17:22:48 +00:00
|
|
|
|
2006-07-10 01:12:39 +00:00
|
|
|
if (unlikely(test_tsk_thread_flag(tsk, TIF_IO_BITMAP))) {
|
2006-10-01 06:27:21 +00:00
|
|
|
p->thread.io_bitmap_ptr = kmemdup(tsk->thread.io_bitmap_ptr,
|
|
|
|
IO_BITMAP_BYTES, GFP_KERNEL);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!p->thread.io_bitmap_ptr) {
|
|
|
|
p->thread.io_bitmap_max = 0;
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
2006-07-10 01:12:39 +00:00
|
|
|
set_tsk_thread_flag(p, TIF_IO_BITMAP);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2008-01-30 12:30:46 +00:00
|
|
|
err = 0;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Set a new TLS for the child thread?
|
|
|
|
*/
|
2008-01-30 12:30:46 +00:00
|
|
|
if (clone_flags & CLONE_SETTLS)
|
|
|
|
err = do_set_thread_area(p, -1,
|
2008-01-30 12:30:56 +00:00
|
|
|
(struct user_desc __user *)childregs->si, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (err && p->thread.io_bitmap_ptr) {
|
|
|
|
kfree(p->thread.io_bitmap_ptr);
|
|
|
|
p->thread.io_bitmap_max = 0;
|
|
|
|
}
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2008-02-21 04:18:40 +00:00
|
|
|
void
|
|
|
|
start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
|
|
|
|
{
|
2009-02-09 13:17:40 +00:00
|
|
|
set_user_gs(regs, 0);
|
2008-02-21 04:18:40 +00:00
|
|
|
regs->fs = 0;
|
|
|
|
regs->ds = __USER_DS;
|
|
|
|
regs->es = __USER_DS;
|
|
|
|
regs->ss = __USER_DS;
|
|
|
|
regs->cs = __USER_CS;
|
|
|
|
regs->ip = new_ip;
|
|
|
|
regs->sp = new_sp;
|
2012-08-02 19:05:11 +00:00
|
|
|
regs->flags = X86_EFLAGS_IF;
|
2012-08-02 18:12:06 +00:00
|
|
|
/*
|
|
|
|
* force it to the iret return path by making it look as if there was
|
|
|
|
* some work pending.
|
|
|
|
*/
|
|
|
|
set_thread_flag(TIF_NOTIFY_RESUME);
|
2008-02-21 04:18:40 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(start_thread);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
2011-04-28 09:02:08 +00:00
|
|
|
* switch_to(x,y) should switch tasks from x to y.
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* We fsave/fwait so that an exception goes off at the right time
|
|
|
|
* (as a call from the fsave or fwait in effect) rather than to
|
|
|
|
* the wrong process. Lazy FP saving no longer makes any sense
|
|
|
|
* with modern CPU's, and this simplifies a lot of things (SMP
|
|
|
|
* and UP become the same).
|
|
|
|
*
|
|
|
|
* NOTE! We used to use the x86 hardware context switching. The
|
|
|
|
* reason for not using it any more becomes apparent when you
|
|
|
|
* try to recover gracefully from saved state that is no longer
|
|
|
|
* valid (stale segment register values in particular). With the
|
|
|
|
* hardware task-switch, there is no way to fix up bad state in
|
|
|
|
* a reasonable manner.
|
|
|
|
*
|
|
|
|
* The fact that Intel documents the hardware task-switching to
|
|
|
|
* be slow is a fairly red herring - this code is not noticeably
|
|
|
|
* faster. However, there _is_ some room for improvement here,
|
|
|
|
* so the performance issues may eventually be a valid point.
|
|
|
|
* More important, however, is the fact that this allows us much
|
|
|
|
* more flexibility.
|
|
|
|
*
|
2008-01-30 12:30:56 +00:00
|
|
|
* The return value (in %ax) will be the "prev" task after
|
2005-04-16 22:20:36 +00:00
|
|
|
* the task-switch, and shows up in ret_from_fork in entry.S,
|
|
|
|
* for example.
|
|
|
|
*/
|
2013-08-05 22:02:39 +00:00
|
|
|
__visible __notrace_funcgraph struct task_struct *
|
2008-12-06 02:40:00 +00:00
|
|
|
__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct thread_struct *prev = &prev_p->thread,
|
|
|
|
*next = &next_p->thread;
|
|
|
|
int cpu = smp_processor_id();
|
2015-03-06 03:19:05 +00:00
|
|
|
struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
|
i387: re-introduce FPU state preloading at context switch time
After all the FPU state cleanups and finally finding the problem that
caused all our FPU save/restore problems, this re-introduces the
preloading of FPU state that was removed in commit b3b0870ef3ff ("i387:
do not preload FPU state at task switch time").
However, instead of simply reverting the removal, this reimplements
preloading with several fixes, most notably
- properly abstracted as a true FPU state switch, rather than as
open-coded save and restore with various hacks.
In particular, implementing it as a proper FPU state switch allows us
to optimize the CR0.TS flag accesses: there is no reason to set the
TS bit only to then almost immediately clear it again. CR0 accesses
are quite slow and expensive, don't flip the bit back and forth for
no good reason.
- Make sure that the same model works for both x86-32 and x86-64, so
that there are no gratuitous differences between the two due to the
way they save and restore segment state differently due to
architectural differences that really don't matter to the FPU state.
- Avoid exposing the "preload" state to the context switch routines,
and in particular allow the concept of lazy state restore: if nothing
else has used the FPU in the meantime, and the process is still on
the same CPU, we can avoid restoring state from memory entirely, just
re-expose the state that is still in the FPU unit.
That optimized lazy restore isn't actually implemented here, but the
infrastructure is set up for it. Of course, older CPU's that use
'fnsave' to save the state cannot take advantage of this, since the
state saving also trashes the state.
In other words, there is now an actual _design_ to the FPU state saving,
rather than just random historical baggage. Hopefully it's easier to
follow as a result.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-02-18 20:56:35 +00:00
|
|
|
fpu_switch_t fpu;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
|
|
|
|
|
i387: support lazy restore of FPU state
This makes us recognize when we try to restore FPU state that matches
what we already have in the FPU on this CPU, and avoids the restore
entirely if so.
To do this, we add two new data fields:
- a percpu 'fpu_owner_task' variable that gets written any time we
update the "has_fpu" field, and thus acts as a kind of back-pointer
to the task that owns the CPU. The exception is when we save the FPU
state as part of a context switch - if the save can keep the FPU
state around, we leave the 'fpu_owner_task' variable pointing at the
task whose FP state still remains on the CPU.
- a per-thread 'last_cpu' field, that indicates which CPU that thread
used its FPU on last. We update this on every context switch
(writing an invalid CPU number if the last context switch didn't
leave the FPU in a lazily usable state), so we know that *that*
thread has done nothing else with the FPU since.
These two fields together can be used when next switching back to the
task to see if the CPU still matches: if 'fpu_owner_task' matches the
task we are switching to, we know that no other task (or kernel FPU
usage) touched the FPU on this CPU in the meantime, and if the current
CPU number matches the 'last_cpu' field, we know that this thread did no
other FP work on any other CPU, so the FPU state on the CPU must match
what was saved on last context switch.
In that case, we can avoid the 'f[x]rstor' entirely, and just clear the
CR0.TS bit.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-02-19 21:27:00 +00:00
|
|
|
fpu = switch_fpu_prepare(prev_p, next_p, cpu);
|
2006-12-07 01:14:01 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2007-02-13 12:26:20 +00:00
|
|
|
* Save away %gs. No need to save %fs, as it was saved on the
|
[PATCH] i386: Use %gs as the PDA base-segment in the kernel
This patch is the meat of the PDA change. This patch makes several related
changes:
1: Most significantly, %gs is now used in the kernel. This means that on
entry, the old value of %gs is saved away, and it is reloaded with
__KERNEL_PDA.
2: entry.S constructs the stack in the shape of struct pt_regs, and this
is passed around the kernel so that the process's saved register
state can be accessed.
Unfortunately struct pt_regs doesn't currently have space for %gs
(or %fs). This patch extends pt_regs to add space for gs (no space
is allocated for %fs, since it won't be used, and it would just
complicate the code in entry.S to work around the space).
3: Because %gs is now saved on the stack like %ds, %es and the integer
registers, there are a number of places where it no longer needs to
be handled specially; namely context switch, and saving/restoring the
register state in a signal context.
4: And since kernel threads run in kernel space and call normal kernel
code, they need to be created with their %gs == __KERNEL_PDA.
Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com>
Signed-off-by: Andi Kleen <ak@suse.de>
Cc: Chuck Ebbert <76306.1226@compuserve.com>
Cc: Zachary Amsden <zach@vmware.com>
Cc: Jan Beulich <jbeulich@novell.com>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
2006-12-07 01:14:02 +00:00
|
|
|
* stack on entry. No need to save %es and %ds, as those are
|
|
|
|
* always kernel segments while inside the kernel. Doing this
|
|
|
|
* before setting the new TLS descriptors avoids the situation
|
|
|
|
* where we temporarily have non-reloadable segments in %fs
|
|
|
|
* and %gs. This could be an issue if the NMI handler ever
|
|
|
|
* used %fs or %gs (it does not today), or if the kernel is
|
|
|
|
* running inside of a hypervisor layer.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2009-02-09 13:17:40 +00:00
|
|
|
lazy_save_gs(prev->gs);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
2005-09-03 22:56:39 +00:00
|
|
|
* Load the per-thread Thread-Local Storage descriptor.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2005-09-03 22:56:39 +00:00
|
|
|
load_TLS(next, cpu);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-02-13 12:26:21 +00:00
|
|
|
/*
|
|
|
|
* Restore IOPL if needed. In normal use, the flags restore
|
|
|
|
* in the switch assembly will handle this. But if the kernel
|
|
|
|
* is running virtualized at a non-zero CPL, the popf will
|
|
|
|
* not restore flags, so it must be done in a separate step.
|
|
|
|
*/
|
|
|
|
if (get_kernel_rpl() && unlikely(prev->iopl != next->iopl))
|
|
|
|
set_iopl_mask(next->iopl);
|
|
|
|
|
2013-08-14 12:51:00 +00:00
|
|
|
/*
|
|
|
|
* If it were not for PREEMPT_ACTIVE we could guarantee that the
|
|
|
|
* preempt_count of all tasks was equal here and this would not be
|
|
|
|
* needed.
|
|
|
|
*/
|
|
|
|
task_thread_info(prev_p)->saved_preempt_count = this_cpu_read(__preempt_count);
|
|
|
|
this_cpu_write(__preempt_count, task_thread_info(next_p)->saved_preempt_count);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2006-07-10 01:12:39 +00:00
|
|
|
* Now maybe handle debug registers and/or IO bitmaps
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2007-07-16 06:41:33 +00:00
|
|
|
if (unlikely(task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV ||
|
|
|
|
task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT))
|
|
|
|
__switch_to_xtra(prev_p, next_p, tss);
|
2005-06-27 21:36:36 +00:00
|
|
|
|
2007-02-13 12:26:21 +00:00
|
|
|
/*
|
|
|
|
* Leave lazy mode, flushing any hypercalls made here.
|
|
|
|
* This must be done before restoring TLS segments so
|
|
|
|
* the GDT and LDT are properly updated, and must be
|
|
|
|
* done before math_state_restore, so the TS bit is up
|
|
|
|
* to date.
|
|
|
|
*/
|
2009-02-18 19:18:57 +00:00
|
|
|
arch_end_context_switch(next_p);
|
2007-02-13 12:26:21 +00:00
|
|
|
|
2015-03-07 01:50:18 +00:00
|
|
|
/*
|
|
|
|
* Reload esp0. This changes current_thread_info().
|
|
|
|
*/
|
|
|
|
load_sp0(tss, next);
|
|
|
|
|
2014-02-06 14:41:31 +00:00
|
|
|
this_cpu_write(kernel_stack,
|
|
|
|
(unsigned long)task_stack_page(next_p) +
|
|
|
|
THREAD_SIZE - KERNEL_STACK_OFFSET);
|
|
|
|
|
2007-02-13 12:26:21 +00:00
|
|
|
/*
|
|
|
|
* Restore %gs if needed (which is common)
|
|
|
|
*/
|
|
|
|
if (prev->gs | next->gs)
|
2009-02-09 13:17:40 +00:00
|
|
|
lazy_load_gs(next->gs);
|
2007-02-13 12:26:21 +00:00
|
|
|
|
i387: re-introduce FPU state preloading at context switch time
After all the FPU state cleanups and finally finding the problem that
caused all our FPU save/restore problems, this re-introduces the
preloading of FPU state that was removed in commit b3b0870ef3ff ("i387:
do not preload FPU state at task switch time").
However, instead of simply reverting the removal, this reimplements
preloading with several fixes, most notably
- properly abstracted as a true FPU state switch, rather than as
open-coded save and restore with various hacks.
In particular, implementing it as a proper FPU state switch allows us
to optimize the CR0.TS flag accesses: there is no reason to set the
TS bit only to then almost immediately clear it again. CR0 accesses
are quite slow and expensive, don't flip the bit back and forth for
no good reason.
- Make sure that the same model works for both x86-32 and x86-64, so
that there are no gratuitous differences between the two due to the
way they save and restore segment state differently due to
architectural differences that really don't matter to the FPU state.
- Avoid exposing the "preload" state to the context switch routines,
and in particular allow the concept of lazy state restore: if nothing
else has used the FPU in the meantime, and the process is still on
the same CPU, we can avoid restoring state from memory entirely, just
re-expose the state that is still in the FPU unit.
That optimized lazy restore isn't actually implemented here, but the
infrastructure is set up for it. Of course, older CPU's that use
'fnsave' to save the state cannot take advantage of this, since the
state saving also trashes the state.
In other words, there is now an actual _design_ to the FPU state saving,
rather than just random historical baggage. Hopefully it's easier to
follow as a result.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-02-18 20:56:35 +00:00
|
|
|
switch_fpu_finish(next_p, fpu);
|
|
|
|
|
2012-05-11 07:35:27 +00:00
|
|
|
this_cpu_write(current_task, next_p);
|
2007-02-13 12:26:21 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
return prev_p;
|
|
|
|
}
|
|
|
|
|
|
|
|
#define top_esp (THREAD_SIZE - sizeof(unsigned long))
|
|
|
|
#define top_ebp (THREAD_SIZE - 2*sizeof(unsigned long))
|
|
|
|
|
|
|
|
unsigned long get_wchan(struct task_struct *p)
|
|
|
|
{
|
2008-01-30 12:30:56 +00:00
|
|
|
unsigned long bp, sp, ip;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long stack_page;
|
|
|
|
int count = 0;
|
|
|
|
if (!p || p == current || p->state == TASK_RUNNING)
|
|
|
|
return 0;
|
2006-01-12 09:05:41 +00:00
|
|
|
stack_page = (unsigned long)task_stack_page(p);
|
2008-01-30 12:31:02 +00:00
|
|
|
sp = p->thread.sp;
|
2008-01-30 12:30:56 +00:00
|
|
|
if (!stack_page || sp < stack_page || sp > top_esp+stack_page)
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
2008-01-30 12:30:56 +00:00
|
|
|
/* include/asm-i386/system.h:switch_to() pushes bp last. */
|
|
|
|
bp = *(unsigned long *) sp;
|
2005-04-16 22:20:36 +00:00
|
|
|
do {
|
2008-01-30 12:30:56 +00:00
|
|
|
if (bp < stack_page || bp > top_ebp+stack_page)
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
2008-01-30 12:30:56 +00:00
|
|
|
ip = *(unsigned long *) (bp+4);
|
|
|
|
if (!in_sched_functions(ip))
|
|
|
|
return ip;
|
|
|
|
bp = *(unsigned long *) bp;
|
2005-04-16 22:20:36 +00:00
|
|
|
} while (count++ < 16);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|