linux/arch/x86/include/asm/syscall.h
Andy Lutomirski 37a8f7c383 x86/asm: Move 'status' from thread_struct to thread_info
The TS_COMPAT bit is very hot and is accessed from code paths that mostly
also touch thread_info::flags.  Move it into struct thread_info to improve
cache locality.

The only reason it was in thread_struct is that there was a brief period
during which arch-specific fields were not allowed in struct thread_info.

Linus suggested further changing:

  ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);

to:

  if (unlikely(ti->status & (TS_COMPAT|TS_I386_REGS_POKED)))
          ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);

on the theory that frequently dirtying the cacheline even in pure 64-bit
code that never needs to modify status hurts performance.  That could be a
reasonable followup patch, but I suspect it matters less on top of this
patch.

Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Kernel Hardening <kernel-hardening@lists.openwall.com>
Link: https://lkml.kernel.org/r/03148bcc1b217100e6e8ecf6a5468c45cf4304b6.1517164461.git.luto@kernel.org
2018-01-30 15:30:36 +01:00

243 lines
5.1 KiB
C

/*
* Access to user system call parameters and results
*
* Copyright (C) 2008-2009 Red Hat, Inc. All rights reserved.
*
* This copyrighted material is made available to anyone wishing to use,
* modify, copy, or redistribute it subject to the terms and conditions
* of the GNU General Public License v.2.
*
* See asm-generic/syscall.h for descriptions of what we must do here.
*/
#ifndef _ASM_X86_SYSCALL_H
#define _ASM_X86_SYSCALL_H
#include <uapi/linux/audit.h>
#include <linux/sched.h>
#include <linux/err.h>
#include <asm/asm-offsets.h> /* For NR_syscalls */
#include <asm/thread_info.h> /* for TS_COMPAT */
#include <asm/unistd.h>
typedef asmlinkage long (*sys_call_ptr_t)(unsigned long, unsigned long,
unsigned long, unsigned long,
unsigned long, unsigned long);
extern const sys_call_ptr_t sys_call_table[];
#if defined(CONFIG_X86_32)
#define ia32_sys_call_table sys_call_table
#define __NR_syscall_compat_max __NR_syscall_max
#define IA32_NR_syscalls NR_syscalls
#endif
#if defined(CONFIG_IA32_EMULATION)
extern const sys_call_ptr_t ia32_sys_call_table[];
#endif
/*
* Only the low 32 bits of orig_ax are meaningful, so we return int.
* This importantly ignores the high bits on 64-bit, so comparisons
* sign-extend the low 32 bits.
*/
static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs)
{
return regs->orig_ax;
}
static inline void syscall_rollback(struct task_struct *task,
struct pt_regs *regs)
{
regs->ax = regs->orig_ax;
}
static inline long syscall_get_error(struct task_struct *task,
struct pt_regs *regs)
{
unsigned long error = regs->ax;
#ifdef CONFIG_IA32_EMULATION
/*
* TS_COMPAT is set for 32-bit syscall entries and then
* remains set until we return to user mode.
*/
if (task->thread_info.status & (TS_COMPAT|TS_I386_REGS_POKED))
/*
* Sign-extend the value so (int)-EFOO becomes (long)-EFOO
* and will match correctly in comparisons.
*/
error = (long) (int) error;
#endif
return IS_ERR_VALUE(error) ? error : 0;
}
static inline long syscall_get_return_value(struct task_struct *task,
struct pt_regs *regs)
{
return regs->ax;
}
static inline void syscall_set_return_value(struct task_struct *task,
struct pt_regs *regs,
int error, long val)
{
regs->ax = (long) error ?: val;
}
#ifdef CONFIG_X86_32
static inline void syscall_get_arguments(struct task_struct *task,
struct pt_regs *regs,
unsigned int i, unsigned int n,
unsigned long *args)
{
BUG_ON(i + n > 6);
memcpy(args, &regs->bx + i, n * sizeof(args[0]));
}
static inline void syscall_set_arguments(struct task_struct *task,
struct pt_regs *regs,
unsigned int i, unsigned int n,
const unsigned long *args)
{
BUG_ON(i + n > 6);
memcpy(&regs->bx + i, args, n * sizeof(args[0]));
}
static inline int syscall_get_arch(void)
{
return AUDIT_ARCH_I386;
}
#else /* CONFIG_X86_64 */
static inline void syscall_get_arguments(struct task_struct *task,
struct pt_regs *regs,
unsigned int i, unsigned int n,
unsigned long *args)
{
# ifdef CONFIG_IA32_EMULATION
if (task->thread_info.status & TS_COMPAT)
switch (i) {
case 0:
if (!n--) break;
*args++ = regs->bx;
case 1:
if (!n--) break;
*args++ = regs->cx;
case 2:
if (!n--) break;
*args++ = regs->dx;
case 3:
if (!n--) break;
*args++ = regs->si;
case 4:
if (!n--) break;
*args++ = regs->di;
case 5:
if (!n--) break;
*args++ = regs->bp;
case 6:
if (!n--) break;
default:
BUG();
break;
}
else
# endif
switch (i) {
case 0:
if (!n--) break;
*args++ = regs->di;
case 1:
if (!n--) break;
*args++ = regs->si;
case 2:
if (!n--) break;
*args++ = regs->dx;
case 3:
if (!n--) break;
*args++ = regs->r10;
case 4:
if (!n--) break;
*args++ = regs->r8;
case 5:
if (!n--) break;
*args++ = regs->r9;
case 6:
if (!n--) break;
default:
BUG();
break;
}
}
static inline void syscall_set_arguments(struct task_struct *task,
struct pt_regs *regs,
unsigned int i, unsigned int n,
const unsigned long *args)
{
# ifdef CONFIG_IA32_EMULATION
if (task->thread_info.status & TS_COMPAT)
switch (i) {
case 0:
if (!n--) break;
regs->bx = *args++;
case 1:
if (!n--) break;
regs->cx = *args++;
case 2:
if (!n--) break;
regs->dx = *args++;
case 3:
if (!n--) break;
regs->si = *args++;
case 4:
if (!n--) break;
regs->di = *args++;
case 5:
if (!n--) break;
regs->bp = *args++;
case 6:
if (!n--) break;
default:
BUG();
break;
}
else
# endif
switch (i) {
case 0:
if (!n--) break;
regs->di = *args++;
case 1:
if (!n--) break;
regs->si = *args++;
case 2:
if (!n--) break;
regs->dx = *args++;
case 3:
if (!n--) break;
regs->r10 = *args++;
case 4:
if (!n--) break;
regs->r8 = *args++;
case 5:
if (!n--) break;
regs->r9 = *args++;
case 6:
if (!n--) break;
default:
BUG();
break;
}
}
static inline int syscall_get_arch(void)
{
/* x32 tasks should be considered AUDIT_ARCH_X86_64. */
return in_ia32_syscall() ? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64;
}
#endif /* CONFIG_X86_32 */
#endif /* _ASM_X86_SYSCALL_H */