mirror of
https://github.com/torvalds/linux.git
synced 2024-12-04 01:51:34 +00:00
7fef099702
The implementation of 'current' on x86 is very intentionally special: it is a very common thing to look up, and it uses 'this_cpu_read_stable()' to get the current thread pointer efficiently from per-cpu storage. And the keyword in there is 'stable': the current thread pointer never changes as far as a single thread is concerned. Even if when a thread is preempted, or moved to another CPU, or even across an explicit call 'schedule()' that thread will still have the same value for 'current'. It is, after all, the kernel base pointer to thread-local storage. That's why it's stable to begin with, but it's also why it's important enough that we have that special 'this_cpu_read_stable()' access for it. So this is all done very intentionally to allow the compiler to treat 'current' as a value that never visibly changes, so that the compiler can do CSE and combine multiple different 'current' accesses into one. However, there is obviously one very special situation when the currently running thread does actually change: inside the scheduler itself. So the scheduler code paths are special, and do not have a 'current' thread at all. Instead there are _two_ threads: the previous and the next thread - typically called 'prev' and 'next' (or prev_p/next_p) internally. So this is all actually quite straightforward and simple, and not all that complicated. Except for when you then have special code that is run in scheduler context, that code then has to be aware that 'current' isn't really a valid thing. Did you mean 'prev'? Did you mean 'next'? In fact, even if then look at the code, and you use 'current' after the new value has been assigned to the percpu variable, we have explicitly told the compiler that 'current' is magical and always stable. So the compiler is quite free to use an older (or newer) value of 'current', and the actual assignment to the percpu storage is not relevant even if it might look that way. Which is exactly what happened in the resctl code, that blithely used 'current' in '__resctrl_sched_in()' when it really wanted the new process state (as implied by the name: we're scheduling 'into' that new resctl state). And clang would end up just using the old thread pointer value at least in some configurations. This could have happened with gcc too, and purely depends on random compiler details. Clang just seems to have been more aggressive about moving the read of the per-cpu current_task pointer around. The fix is trivial: just make the resctl code adhere to the scheduler rules of using the prev/next thread pointer explicitly, instead of using 'current' in a situation where it just wasn't valid. That same code is then also used outside of the scheduler context (when a thread resctl state is explicitly changed), and then we will just pass in 'current' as that pointer, of course. There is no ambiguity in that case. The fix may be trivial, but noticing and figuring out what went wrong was not. The credit for that goes to Stephane Eranian. Reported-by: Stephane Eranian <eranian@google.com> Link: https://lore.kernel.org/lkml/20230303231133.1486085-1-eranian@google.com/ Link: https://lore.kernel.org/lkml/alpine.LFD.2.01.0908011214330.3304@localhost.localdomain/ Reviewed-by: Nick Desaulniers <ndesaulniers@google.com> Tested-by: Tony Luck <tony.luck@intel.com> Tested-by: Stephane Eranian <eranian@google.com> Tested-by: Babu Moger <babu.moger@amd.com> Cc: stable@kernel.org Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
224 lines
6.1 KiB
C
224 lines
6.1 KiB
C
/*
|
|
* Copyright (C) 1995 Linus Torvalds
|
|
*
|
|
* Pentium III FXSR, SSE support
|
|
* Gareth Hughes <gareth@valinux.com>, May 2000
|
|
*/
|
|
|
|
/*
|
|
* This file handles the architecture-dependent parts of process handling..
|
|
*/
|
|
|
|
#include <linux/cpu.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/sched/task.h>
|
|
#include <linux/sched/task_stack.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/elfcore.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/stddef.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/user.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/reboot.h>
|
|
#include <linux/mc146818rtc.h>
|
|
#include <linux/export.h>
|
|
#include <linux/kallsyms.h>
|
|
#include <linux/ptrace.h>
|
|
#include <linux/personality.h>
|
|
#include <linux/percpu.h>
|
|
#include <linux/prctl.h>
|
|
#include <linux/ftrace.h>
|
|
#include <linux/uaccess.h>
|
|
#include <linux/io.h>
|
|
#include <linux/kdebug.h>
|
|
#include <linux/syscalls.h>
|
|
|
|
#include <asm/ldt.h>
|
|
#include <asm/processor.h>
|
|
#include <asm/fpu/sched.h>
|
|
#include <asm/desc.h>
|
|
|
|
#include <linux/err.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
#include <asm/cpu.h>
|
|
#include <asm/debugreg.h>
|
|
#include <asm/switch_to.h>
|
|
#include <asm/vm86.h>
|
|
#include <asm/resctrl.h>
|
|
#include <asm/proto.h>
|
|
|
|
#include "process.h"
|
|
|
|
void __show_regs(struct pt_regs *regs, enum show_regs_mode mode,
|
|
const char *log_lvl)
|
|
{
|
|
unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L;
|
|
unsigned long d0, d1, d2, d3, d6, d7;
|
|
unsigned short gs;
|
|
|
|
savesegment(gs, gs);
|
|
|
|
show_ip(regs, log_lvl);
|
|
|
|
printk("%sEAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n",
|
|
log_lvl, regs->ax, regs->bx, regs->cx, regs->dx);
|
|
printk("%sESI: %08lx EDI: %08lx EBP: %08lx ESP: %08lx\n",
|
|
log_lvl, regs->si, regs->di, regs->bp, regs->sp);
|
|
printk("%sDS: %04x ES: %04x FS: %04x GS: %04x SS: %04x EFLAGS: %08lx\n",
|
|
log_lvl, (u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, regs->ss, regs->flags);
|
|
|
|
if (mode != SHOW_REGS_ALL)
|
|
return;
|
|
|
|
cr0 = read_cr0();
|
|
cr2 = read_cr2();
|
|
cr3 = __read_cr3();
|
|
cr4 = __read_cr4();
|
|
printk("%sCR0: %08lx CR2: %08lx CR3: %08lx CR4: %08lx\n",
|
|
log_lvl, cr0, cr2, cr3, cr4);
|
|
|
|
get_debugreg(d0, 0);
|
|
get_debugreg(d1, 1);
|
|
get_debugreg(d2, 2);
|
|
get_debugreg(d3, 3);
|
|
get_debugreg(d6, 6);
|
|
get_debugreg(d7, 7);
|
|
|
|
/* Only print out debug registers if they are in their non-default state. */
|
|
if ((d0 == 0) && (d1 == 0) && (d2 == 0) && (d3 == 0) &&
|
|
(d6 == DR6_RESERVED) && (d7 == 0x400))
|
|
return;
|
|
|
|
printk("%sDR0: %08lx DR1: %08lx DR2: %08lx DR3: %08lx\n",
|
|
log_lvl, d0, d1, d2, d3);
|
|
printk("%sDR6: %08lx DR7: %08lx\n",
|
|
log_lvl, d6, d7);
|
|
}
|
|
|
|
void release_thread(struct task_struct *dead_task)
|
|
{
|
|
BUG_ON(dead_task->mm);
|
|
release_vm86_irqs(dead_task);
|
|
}
|
|
|
|
void
|
|
start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
|
|
{
|
|
loadsegment(gs, 0);
|
|
regs->fs = 0;
|
|
regs->ds = __USER_DS;
|
|
regs->es = __USER_DS;
|
|
regs->ss = __USER_DS;
|
|
regs->cs = __USER_CS;
|
|
regs->ip = new_ip;
|
|
regs->sp = new_sp;
|
|
regs->flags = X86_EFLAGS_IF;
|
|
}
|
|
EXPORT_SYMBOL_GPL(start_thread);
|
|
|
|
|
|
/*
|
|
* switch_to(x,y) should switch tasks from x to y.
|
|
*
|
|
* We fsave/fwait so that an exception goes off at the right time
|
|
* (as a call from the fsave or fwait in effect) rather than to
|
|
* the wrong process. Lazy FP saving no longer makes any sense
|
|
* with modern CPU's, and this simplifies a lot of things (SMP
|
|
* and UP become the same).
|
|
*
|
|
* NOTE! We used to use the x86 hardware context switching. The
|
|
* reason for not using it any more becomes apparent when you
|
|
* try to recover gracefully from saved state that is no longer
|
|
* valid (stale segment register values in particular). With the
|
|
* hardware task-switch, there is no way to fix up bad state in
|
|
* a reasonable manner.
|
|
*
|
|
* The fact that Intel documents the hardware task-switching to
|
|
* be slow is a fairly red herring - this code is not noticeably
|
|
* faster. However, there _is_ some room for improvement here,
|
|
* so the performance issues may eventually be a valid point.
|
|
* More important, however, is the fact that this allows us much
|
|
* more flexibility.
|
|
*
|
|
* The return value (in %ax) will be the "prev" task after
|
|
* the task-switch, and shows up in ret_from_fork in entry.S,
|
|
* for example.
|
|
*/
|
|
__visible __notrace_funcgraph struct task_struct *
|
|
__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
|
|
{
|
|
struct thread_struct *prev = &prev_p->thread,
|
|
*next = &next_p->thread;
|
|
struct fpu *prev_fpu = &prev->fpu;
|
|
int cpu = smp_processor_id();
|
|
|
|
/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
|
|
|
|
if (!test_thread_flag(TIF_NEED_FPU_LOAD))
|
|
switch_fpu_prepare(prev_fpu, cpu);
|
|
|
|
/*
|
|
* Save away %gs. No need to save %fs, as it was saved on the
|
|
* stack on entry. No need to save %es and %ds, as those are
|
|
* always kernel segments while inside the kernel. Doing this
|
|
* before setting the new TLS descriptors avoids the situation
|
|
* where we temporarily have non-reloadable segments in %fs
|
|
* and %gs. This could be an issue if the NMI handler ever
|
|
* used %fs or %gs (it does not today), or if the kernel is
|
|
* running inside of a hypervisor layer.
|
|
*/
|
|
savesegment(gs, prev->gs);
|
|
|
|
/*
|
|
* Load the per-thread Thread-Local Storage descriptor.
|
|
*/
|
|
load_TLS(next, cpu);
|
|
|
|
switch_to_extra(prev_p, next_p);
|
|
|
|
/*
|
|
* Leave lazy mode, flushing any hypercalls made here.
|
|
* This must be done before restoring TLS segments so
|
|
* the GDT and LDT are properly updated.
|
|
*/
|
|
arch_end_context_switch(next_p);
|
|
|
|
/*
|
|
* Reload esp0 and pcpu_hot.top_of_stack. This changes
|
|
* current_thread_info(). Refresh the SYSENTER configuration in
|
|
* case prev or next is vm86.
|
|
*/
|
|
update_task_stack(next_p);
|
|
refresh_sysenter_cs(next);
|
|
this_cpu_write(pcpu_hot.top_of_stack,
|
|
(unsigned long)task_stack_page(next_p) +
|
|
THREAD_SIZE);
|
|
|
|
/*
|
|
* Restore %gs if needed (which is common)
|
|
*/
|
|
if (prev->gs | next->gs)
|
|
loadsegment(gs, next->gs);
|
|
|
|
raw_cpu_write(pcpu_hot.current_task, next_p);
|
|
|
|
switch_fpu_finish();
|
|
|
|
/* Load the Intel cache allocation PQR MSR. */
|
|
resctrl_sched_in(next_p);
|
|
|
|
return prev_p;
|
|
}
|
|
|
|
SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2)
|
|
{
|
|
return do_arch_prctl_common(option, arg2);
|
|
}
|