2008-10-23 05:26:29 +00:00
|
|
|
#ifndef _ASM_X86_PROCESSOR_H
|
|
|
|
#define _ASM_X86_PROCESSOR_H
|
2008-01-30 12:31:03 +00:00
|
|
|
|
2008-01-30 12:31:27 +00:00
|
|
|
#include <asm/processor-flags.h>
|
|
|
|
|
2008-01-30 12:31:27 +00:00
|
|
|
/* Forward declaration, a strange C thing */
|
|
|
|
struct task_struct;
|
|
|
|
struct mm_struct;
|
2015-07-29 05:41:16 +00:00
|
|
|
struct vm86;
|
2008-01-30 12:31:27 +00:00
|
|
|
|
2008-01-30 12:31:57 +00:00
|
|
|
#include <asm/math_emu.h>
|
|
|
|
#include <asm/segment.h>
|
|
|
|
#include <asm/types.h>
|
2015-09-05 07:32:43 +00:00
|
|
|
#include <uapi/asm/sigcontext.h>
|
2008-01-30 12:31:57 +00:00
|
|
|
#include <asm/current.h>
|
2016-01-26 21:12:04 +00:00
|
|
|
#include <asm/cpufeatures.h>
|
2008-01-30 12:31:57 +00:00
|
|
|
#include <asm/page.h>
|
2009-02-11 18:20:05 +00:00
|
|
|
#include <asm/pgtable_types.h>
|
2008-01-30 12:31:33 +00:00
|
|
|
#include <asm/percpu.h>
|
2008-01-30 12:31:57 +00:00
|
|
|
#include <asm/msr.h>
|
|
|
|
#include <asm/desc_defs.h>
|
2008-01-30 12:32:38 +00:00
|
|
|
#include <asm/nops.h>
|
2012-03-28 17:11:12 +00:00
|
|
|
#include <asm/special_insns.h>
|
2015-04-22 07:57:24 +00:00
|
|
|
#include <asm/fpu/types.h>
|
2008-02-21 03:24:40 +00:00
|
|
|
|
2008-01-30 12:31:57 +00:00
|
|
|
#include <linux/personality.h>
|
2008-01-30 12:31:33 +00:00
|
|
|
#include <linux/cache.h>
|
2008-01-30 12:31:57 +00:00
|
|
|
#include <linux/threads.h>
|
2009-09-02 09:49:52 +00:00
|
|
|
#include <linux/math64.h>
|
2010-03-25 13:51:50 +00:00
|
|
|
#include <linux/err.h>
|
2012-03-28 17:11:12 +00:00
|
|
|
#include <linux/irqflags.h>
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We handle most unaligned accesses in hardware. On the other hand
|
|
|
|
* unaligned DMA can be quite expensive on some Nehalem processors.
|
|
|
|
*
|
|
|
|
* Based on this we disable the IP header alignment in network drivers.
|
|
|
|
*/
|
|
|
|
#define NET_IP_ALIGN 0
|
2008-01-30 12:31:27 +00:00
|
|
|
|
2009-06-01 18:13:10 +00:00
|
|
|
#define HBP_NUM 4
|
2008-01-30 12:31:27 +00:00
|
|
|
/*
|
|
|
|
* Default implementation of macro that returns current
|
|
|
|
* instruction pointer ("program counter").
|
|
|
|
*/
|
|
|
|
static inline void *current_text_addr(void)
|
|
|
|
{
|
|
|
|
void *pc;
|
2008-02-21 03:24:40 +00:00
|
|
|
|
|
|
|
asm volatile("mov $1f, %0; 1:":"=r" (pc));
|
|
|
|
|
2008-01-30 12:31:27 +00:00
|
|
|
return pc;
|
|
|
|
}
|
|
|
|
|
2015-05-24 07:58:12 +00:00
|
|
|
/*
|
|
|
|
* These alignment constraints are for performance in the vSMP case,
|
|
|
|
* but in the task_struct case we must also meet hardware imposed
|
|
|
|
* alignment requirements of the FPU state:
|
|
|
|
*/
|
2008-01-30 12:31:31 +00:00
|
|
|
#ifdef CONFIG_X86_VSMP
|
2008-02-21 03:24:40 +00:00
|
|
|
# define ARCH_MIN_TASKALIGN (1 << INTERNODE_CACHE_SHIFT)
|
|
|
|
# define ARCH_MIN_MMSTRUCT_ALIGN (1 << INTERNODE_CACHE_SHIFT)
|
2008-01-30 12:31:31 +00:00
|
|
|
#else
|
2015-05-24 07:58:12 +00:00
|
|
|
# define ARCH_MIN_TASKALIGN __alignof__(union fpregs_state)
|
2008-02-21 03:24:40 +00:00
|
|
|
# define ARCH_MIN_MMSTRUCT_ALIGN 0
|
2008-01-30 12:31:31 +00:00
|
|
|
#endif
|
|
|
|
|
x86/tlb_info: get last level TLB entry number of CPU
For 4KB pages, x86 CPU has 2 or 1 level TLB, first level is data TLB and
instruction TLB, second level is shared TLB for both data and instructions.
For hupe page TLB, usually there is just one level and seperated by 2MB/4MB
and 1GB.
Although each levels TLB size is important for performance tuning, but for
genernal and rude optimizing, last level TLB entry number is suitable. And
in fact, last level TLB always has the biggest entry number.
This patch will get the biggest TLB entry number and use it in furture TLB
optimizing.
Accroding Borislav's suggestion, except tlb_ll[i/d]_* array, other
function and data will be released after system boot up.
For all kinds of x86 vendor friendly, vendor specific code was moved to its
specific files.
Signed-off-by: Alex Shi <alex.shi@intel.com>
Link: http://lkml.kernel.org/r/1340845344-27557-2-git-send-email-alex.shi@intel.com
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2012-06-28 01:02:16 +00:00
|
|
|
enum tlb_infos {
|
|
|
|
ENTRIES,
|
|
|
|
NR_INFO
|
|
|
|
};
|
|
|
|
|
|
|
|
extern u16 __read_mostly tlb_lli_4k[NR_INFO];
|
|
|
|
extern u16 __read_mostly tlb_lli_2m[NR_INFO];
|
|
|
|
extern u16 __read_mostly tlb_lli_4m[NR_INFO];
|
|
|
|
extern u16 __read_mostly tlb_lld_4k[NR_INFO];
|
|
|
|
extern u16 __read_mostly tlb_lld_2m[NR_INFO];
|
|
|
|
extern u16 __read_mostly tlb_lld_4m[NR_INFO];
|
x86, cpu: Detect more TLB configuration
The Intel Software Developer’s Manual covers few more TLB
configurations exposed as CPUID 2 descriptors:
61H Instruction TLB: 4 KByte pages, fully associative, 48 entries
63H Data TLB: 1 GByte pages, 4-way set associative, 4 entries
76H Instruction TLB: 2M/4M pages, fully associative, 8 entries
B5H Instruction TLB: 4KByte pages, 8-way set associative, 64 entries
B6H Instruction TLB: 4KByte pages, 8-way set associative, 128 entries
C1H Shared 2nd-Level TLB: 4 KByte/2MByte pages, 8-way associative, 1024 entries
C2H DTLB DTLB: 2 MByte/$MByte pages, 4-way associative, 16 entries
Let's detect them as well.
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Link: http://lkml.kernel.org/r/1387801018-14499-1-git-send-email-kirill.shutemov@linux.intel.com
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2013-12-23 12:16:58 +00:00
|
|
|
extern u16 __read_mostly tlb_lld_1g[NR_INFO];
|
2012-06-28 01:02:19 +00:00
|
|
|
|
2008-01-30 12:31:33 +00:00
|
|
|
/*
|
|
|
|
* CPU type and hardware bug flags. Kept separately for each CPU.
|
|
|
|
* Members of this structure are referenced in head.S, so think twice
|
|
|
|
* before touching them. [mj]
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct cpuinfo_x86 {
|
2008-02-21 03:24:40 +00:00
|
|
|
__u8 x86; /* CPU family */
|
|
|
|
__u8 x86_vendor; /* CPU vendor */
|
|
|
|
__u8 x86_model;
|
|
|
|
__u8 x86_mask;
|
2008-01-30 12:31:33 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-02-21 03:24:40 +00:00
|
|
|
char wp_works_ok; /* It doesn't on 386's */
|
|
|
|
|
|
|
|
/* Problems on some 486Dx4's and old 386's: */
|
|
|
|
char rfu;
|
|
|
|
char pad0;
|
2013-04-29 14:04:20 +00:00
|
|
|
char pad1;
|
2008-01-30 12:31:33 +00:00
|
|
|
#else
|
2008-02-21 03:24:40 +00:00
|
|
|
/* Number of 4K pages in DTLB/ITLB combined(in pages): */
|
2009-01-24 01:18:52 +00:00
|
|
|
int x86_tlbsize;
|
2009-03-12 12:37:34 +00:00
|
|
|
#endif
|
2008-02-21 03:24:40 +00:00
|
|
|
__u8 x86_virt_bits;
|
|
|
|
__u8 x86_phys_bits;
|
|
|
|
/* CPUID returned core id bits: */
|
|
|
|
__u8 x86_coreid_bits;
|
|
|
|
/* Max extended CPUID function supported: */
|
|
|
|
__u32 extended_cpuid_level;
|
|
|
|
/* Maximum supported CPUID level, -1=no CPUID: */
|
|
|
|
int cpuid_level;
|
2013-03-20 14:07:23 +00:00
|
|
|
__u32 x86_capability[NCAPINTS + NBUGINTS];
|
2008-02-21 03:24:40 +00:00
|
|
|
char x86_vendor_id[16];
|
|
|
|
char x86_model_id[64];
|
|
|
|
/* in KB - valid for CPUS which support this call: */
|
|
|
|
int x86_cache_size;
|
|
|
|
int x86_cache_alignment; /* In bytes */
|
2015-01-23 18:45:43 +00:00
|
|
|
/* Cache QoS architectural values: */
|
|
|
|
int x86_cache_max_rmid; /* max index */
|
|
|
|
int x86_cache_occ_scale; /* scale to bytes */
|
2008-02-21 03:24:40 +00:00
|
|
|
int x86_power;
|
|
|
|
unsigned long loops_per_jiffy;
|
|
|
|
/* cpuid returned max cores value: */
|
|
|
|
u16 x86_max_cores;
|
|
|
|
u16 apicid;
|
2008-03-06 21:46:39 +00:00
|
|
|
u16 initial_apicid;
|
2008-02-21 03:24:40 +00:00
|
|
|
u16 x86_clflush_size;
|
|
|
|
/* number of cores as seen by the OS: */
|
|
|
|
u16 booted_cores;
|
|
|
|
/* Physical processor id: */
|
|
|
|
u16 phys_proc_id;
|
2016-02-22 22:19:15 +00:00
|
|
|
/* Logical processor id: */
|
|
|
|
u16 logical_proc_id;
|
2008-02-21 03:24:40 +00:00
|
|
|
/* Core id: */
|
|
|
|
u16 cpu_core_id;
|
|
|
|
/* Index into per_cpu list: */
|
|
|
|
u16 cpu_index;
|
2011-10-13 00:46:33 +00:00
|
|
|
u32 microcode;
|
2014-11-04 08:26:42 +00:00
|
|
|
};
|
2008-01-30 12:31:33 +00:00
|
|
|
|
2016-11-11 09:25:34 +00:00
|
|
|
struct cpuid_regs {
|
|
|
|
u32 eax, ebx, ecx, edx;
|
|
|
|
};
|
|
|
|
|
|
|
|
enum cpuid_regs_idx {
|
|
|
|
CPUID_EAX = 0,
|
|
|
|
CPUID_EBX,
|
|
|
|
CPUID_ECX,
|
|
|
|
CPUID_EDX,
|
|
|
|
};
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
#define X86_VENDOR_INTEL 0
|
|
|
|
#define X86_VENDOR_CYRIX 1
|
|
|
|
#define X86_VENDOR_AMD 2
|
|
|
|
#define X86_VENDOR_UMC 3
|
|
|
|
#define X86_VENDOR_CENTAUR 5
|
|
|
|
#define X86_VENDOR_TRANSMETA 7
|
|
|
|
#define X86_VENDOR_NSC 8
|
|
|
|
#define X86_VENDOR_NUM 9
|
|
|
|
|
|
|
|
#define X86_VENDOR_UNKNOWN 0xff
|
2008-01-30 12:31:33 +00:00
|
|
|
|
2008-01-30 12:31:39 +00:00
|
|
|
/*
|
|
|
|
* capabilities of CPUs
|
|
|
|
*/
|
2008-02-21 03:24:40 +00:00
|
|
|
extern struct cpuinfo_x86 boot_cpu_data;
|
|
|
|
extern struct cpuinfo_x86 new_cpu_data;
|
|
|
|
|
|
|
|
extern struct tss_struct doublefault_tss;
|
2009-05-10 06:47:42 +00:00
|
|
|
extern __u32 cpu_caps_cleared[NCAPINTS];
|
|
|
|
extern __u32 cpu_caps_set[NCAPINTS];
|
2008-01-30 12:31:33 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
2014-11-04 08:26:42 +00:00
|
|
|
DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
|
2008-01-30 12:31:33 +00:00
|
|
|
#define cpu_data(cpu) per_cpu(cpu_info, cpu)
|
|
|
|
#else
|
2010-12-18 15:30:05 +00:00
|
|
|
#define cpu_info boot_cpu_data
|
2008-01-30 12:31:33 +00:00
|
|
|
#define cpu_data(cpu) boot_cpu_data
|
|
|
|
#endif
|
|
|
|
|
2008-07-21 17:10:37 +00:00
|
|
|
extern const struct seq_operations cpuinfo_op;
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
#define cache_line_size() (boot_cpu_data.x86_cache_alignment)
|
|
|
|
|
|
|
|
extern void cpu_detect(struct cpuinfo_x86 *c);
|
2008-01-30 12:31:39 +00:00
|
|
|
|
2008-06-21 10:24:19 +00:00
|
|
|
extern void early_cpu_init(void);
|
2008-01-30 12:31:39 +00:00
|
|
|
extern void identify_boot_cpu(void);
|
|
|
|
extern void identify_secondary_cpu(struct cpuinfo_x86 *);
|
2008-01-30 12:31:33 +00:00
|
|
|
extern void print_cpu_info(struct cpuinfo_x86 *);
|
2012-02-12 17:53:57 +00:00
|
|
|
void print_cpu_msr(struct cpuinfo_x86 *);
|
2008-01-30 12:31:33 +00:00
|
|
|
extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
|
2016-11-11 09:25:35 +00:00
|
|
|
extern u32 get_scattered_cpuid_leaf(unsigned int level,
|
|
|
|
unsigned int sub_leaf,
|
|
|
|
enum cpuid_regs_idx reg);
|
2008-01-30 12:31:33 +00:00
|
|
|
extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
|
2012-10-19 08:59:33 +00:00
|
|
|
extern void init_amd_cacheinfo(struct cpuinfo_x86 *c);
|
2008-01-30 12:31:33 +00:00
|
|
|
|
2008-08-23 15:47:10 +00:00
|
|
|
extern void detect_extended_topology(struct cpuinfo_x86 *c);
|
2008-01-30 12:31:39 +00:00
|
|
|
extern void detect_ht(struct cpuinfo_x86 *c);
|
|
|
|
|
2012-12-21 07:44:23 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
extern int have_cpuid_p(void);
|
|
|
|
#else
|
|
|
|
static inline int have_cpuid_p(void)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
#endif
|
2008-01-30 12:31:03 +00:00
|
|
|
static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
|
2008-02-21 03:24:40 +00:00
|
|
|
unsigned int *ecx, unsigned int *edx)
|
2008-01-30 12:31:03 +00:00
|
|
|
{
|
|
|
|
/* ecx is often an input as well as an output. */
|
2009-12-17 00:25:42 +00:00
|
|
|
asm volatile("cpuid"
|
2008-03-23 08:03:15 +00:00
|
|
|
: "=a" (*eax),
|
|
|
|
"=b" (*ebx),
|
|
|
|
"=c" (*ecx),
|
|
|
|
"=d" (*edx)
|
2011-10-13 00:46:33 +00:00
|
|
|
: "0" (*eax), "2" (*ecx)
|
|
|
|
: "memory");
|
2008-01-30 12:31:03 +00:00
|
|
|
}
|
|
|
|
|
2008-01-30 12:31:27 +00:00
|
|
|
static inline void load_cr3(pgd_t *pgdir)
|
|
|
|
{
|
|
|
|
write_cr3(__pa(pgdir));
|
|
|
|
}
|
2008-01-30 12:31:03 +00:00
|
|
|
|
2008-01-30 12:31:31 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
/* This is the TSS defined by the hardware. */
|
|
|
|
struct x86_hw_tss {
|
2008-02-21 03:24:40 +00:00
|
|
|
unsigned short back_link, __blh;
|
|
|
|
unsigned long sp0;
|
|
|
|
unsigned short ss0, __ss0h;
|
2015-04-02 19:41:45 +00:00
|
|
|
unsigned long sp1;
|
2015-03-10 18:06:00 +00:00
|
|
|
|
|
|
|
/*
|
2015-04-02 19:41:45 +00:00
|
|
|
* We don't use ring 1, so ss1 is a convenient scratch space in
|
|
|
|
* the same cacheline as sp0. We use ss1 to cache the value in
|
|
|
|
* MSR_IA32_SYSENTER_CS. When we context switch
|
|
|
|
* MSR_IA32_SYSENTER_CS, we first check if the new value being
|
|
|
|
* written matches ss1, and, if it's not, then we wrmsr the new
|
|
|
|
* value and update ss1.
|
2015-03-10 18:06:00 +00:00
|
|
|
*
|
2015-04-02 19:41:45 +00:00
|
|
|
* The only reason we context switch MSR_IA32_SYSENTER_CS is
|
|
|
|
* that we set it to zero in vm86 tasks to avoid corrupting the
|
|
|
|
* stack if we were to go through the sysenter path from vm86
|
|
|
|
* mode.
|
2015-03-10 18:06:00 +00:00
|
|
|
*/
|
|
|
|
unsigned short ss1; /* MSR_IA32_SYSENTER_CS */
|
|
|
|
|
|
|
|
unsigned short __ss1h;
|
2008-02-21 03:24:40 +00:00
|
|
|
unsigned long sp2;
|
|
|
|
unsigned short ss2, __ss2h;
|
|
|
|
unsigned long __cr3;
|
|
|
|
unsigned long ip;
|
|
|
|
unsigned long flags;
|
|
|
|
unsigned long ax;
|
|
|
|
unsigned long cx;
|
|
|
|
unsigned long dx;
|
|
|
|
unsigned long bx;
|
|
|
|
unsigned long sp;
|
|
|
|
unsigned long bp;
|
|
|
|
unsigned long si;
|
|
|
|
unsigned long di;
|
|
|
|
unsigned short es, __esh;
|
|
|
|
unsigned short cs, __csh;
|
|
|
|
unsigned short ss, __ssh;
|
|
|
|
unsigned short ds, __dsh;
|
|
|
|
unsigned short fs, __fsh;
|
|
|
|
unsigned short gs, __gsh;
|
|
|
|
unsigned short ldt, __ldth;
|
|
|
|
unsigned short trace;
|
|
|
|
unsigned short io_bitmap_base;
|
|
|
|
|
2008-01-30 12:31:31 +00:00
|
|
|
} __attribute__((packed));
|
|
|
|
#else
|
|
|
|
struct x86_hw_tss {
|
2008-02-21 03:24:40 +00:00
|
|
|
u32 reserved1;
|
|
|
|
u64 sp0;
|
|
|
|
u64 sp1;
|
|
|
|
u64 sp2;
|
|
|
|
u64 reserved2;
|
|
|
|
u64 ist[7];
|
|
|
|
u32 reserved3;
|
|
|
|
u32 reserved4;
|
|
|
|
u16 reserved5;
|
|
|
|
u16 io_bitmap_base;
|
|
|
|
|
2008-01-30 12:31:31 +00:00
|
|
|
} __attribute__((packed)) ____cacheline_aligned;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
2008-02-21 03:24:40 +00:00
|
|
|
* IO-bitmap sizes:
|
2008-01-30 12:31:31 +00:00
|
|
|
*/
|
2008-02-21 03:24:40 +00:00
|
|
|
#define IO_BITMAP_BITS 65536
|
|
|
|
#define IO_BITMAP_BYTES (IO_BITMAP_BITS/8)
|
|
|
|
#define IO_BITMAP_LONGS (IO_BITMAP_BYTES/sizeof(long))
|
|
|
|
#define IO_BITMAP_OFFSET offsetof(struct tss_struct, io_bitmap)
|
|
|
|
#define INVALID_IO_BITMAP_OFFSET 0x8000
|
2008-01-30 12:31:31 +00:00
|
|
|
|
|
|
|
struct tss_struct {
|
2008-02-21 03:24:40 +00:00
|
|
|
/*
|
|
|
|
* The hardware state:
|
|
|
|
*/
|
|
|
|
struct x86_hw_tss x86_tss;
|
2008-01-30 12:31:31 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The extra 1 is there because the CPU will access an
|
|
|
|
* additional byte beyond the end of the IO permission
|
|
|
|
* bitmap. The extra byte must be all 1 bits, and must
|
|
|
|
* be within the limit.
|
|
|
|
*/
|
2008-02-21 03:24:40 +00:00
|
|
|
unsigned long io_bitmap[IO_BITMAP_LONGS + 1];
|
|
|
|
|
2016-03-10 03:00:31 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-01-30 12:31:31 +00:00
|
|
|
/*
|
2016-03-10 03:00:33 +00:00
|
|
|
* Space for the temporary SYSENTER stack.
|
2008-01-30 12:31:31 +00:00
|
|
|
*/
|
2016-03-10 03:00:33 +00:00
|
|
|
unsigned long SYSENTER_stack_canary;
|
2015-03-09 14:52:18 +00:00
|
|
|
unsigned long SYSENTER_stack[64];
|
2016-03-10 03:00:31 +00:00
|
|
|
#endif
|
2008-02-21 03:24:40 +00:00
|
|
|
|
2008-07-04 12:56:16 +00:00
|
|
|
} ____cacheline_aligned;
|
2008-01-30 12:31:31 +00:00
|
|
|
|
2015-03-06 03:19:05 +00:00
|
|
|
DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss);
|
2008-01-30 12:31:31 +00:00
|
|
|
|
2015-03-07 01:50:19 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
|
|
|
|
#endif
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
/*
|
|
|
|
* Save the original ist values for checking stack pointers during debugging
|
|
|
|
*/
|
2008-01-30 12:31:39 +00:00
|
|
|
struct orig_ist {
|
2008-02-21 03:24:40 +00:00
|
|
|
unsigned long ist[7];
|
2008-01-30 12:31:39 +00:00
|
|
|
};
|
|
|
|
|
2008-03-03 17:12:56 +00:00
|
|
|
#ifdef CONFIG_X86_64
|
2008-01-30 12:31:57 +00:00
|
|
|
DECLARE_PER_CPU(struct orig_ist, orig_ist);
|
2009-01-18 15:38:58 +00:00
|
|
|
|
2009-01-19 03:21:28 +00:00
|
|
|
union irq_stack_union {
|
|
|
|
char irq_stack[IRQ_STACK_SIZE];
|
|
|
|
/*
|
|
|
|
* GCC hardcodes the stack canary as %gs:40. Since the
|
|
|
|
* irq_stack is the object at %gs:0, we reserve the bottom
|
|
|
|
* 48 bytes of the irq stack for the canary.
|
|
|
|
*/
|
|
|
|
struct {
|
|
|
|
char gs_base[40];
|
|
|
|
unsigned long stack_canary;
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
2013-08-05 22:02:43 +00:00
|
|
|
DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible;
|
2009-02-08 14:58:39 +00:00
|
|
|
DECLARE_INIT_PER_CPU(irq_stack_union);
|
|
|
|
|
2009-01-18 15:38:58 +00:00
|
|
|
DECLARE_PER_CPU(char *, irq_stack_ptr);
|
2009-03-14 05:49:49 +00:00
|
|
|
DECLARE_PER_CPU(unsigned int, irq_count);
|
|
|
|
extern asmlinkage void ignore_sysret(void);
|
2009-02-09 13:17:40 +00:00
|
|
|
#else /* X86_64 */
|
|
|
|
#ifdef CONFIG_CC_STACKPROTECTOR
|
2009-09-03 19:27:15 +00:00
|
|
|
/*
|
|
|
|
* Make sure stack canary segment base is cached-aligned:
|
|
|
|
* "For Intel Atom processors, avoid non zero segment base address
|
|
|
|
* that is not aligned to cache line boundary at all cost."
|
|
|
|
* (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
|
|
|
|
*/
|
|
|
|
struct stack_canary {
|
|
|
|
char __pad[20]; /* canary at %gs:20 */
|
|
|
|
unsigned long canary;
|
|
|
|
};
|
2009-09-03 21:31:44 +00:00
|
|
|
DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
|
2007-10-11 09:20:03 +00:00
|
|
|
#endif
|
2014-02-06 14:41:31 +00:00
|
|
|
/*
|
|
|
|
* per-CPU IRQ handling stacks
|
|
|
|
*/
|
|
|
|
struct irq_stack {
|
|
|
|
u32 stack[THREAD_SIZE/sizeof(u32)];
|
|
|
|
} __aligned(THREAD_SIZE);
|
|
|
|
|
|
|
|
DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
|
|
|
|
DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
|
2009-02-09 13:17:40 +00:00
|
|
|
#endif /* X86_64 */
|
2008-01-30 12:31:03 +00:00
|
|
|
|
2016-05-20 17:47:06 +00:00
|
|
|
extern unsigned int fpu_kernel_xstate_size;
|
2016-05-20 17:47:05 +00:00
|
|
|
extern unsigned int fpu_user_xstate_size;
|
2008-01-30 12:31:27 +00:00
|
|
|
|
2009-09-09 17:22:48 +00:00
|
|
|
struct perf_event;
|
|
|
|
|
2016-07-14 20:22:57 +00:00
|
|
|
typedef struct {
|
|
|
|
unsigned long seg;
|
|
|
|
} mm_segment_t;
|
|
|
|
|
2008-01-30 12:31:31 +00:00
|
|
|
struct thread_struct {
|
2008-02-21 03:24:40 +00:00
|
|
|
/* Cached TLS descriptors: */
|
|
|
|
struct desc_struct tls_array[GDT_ENTRY_TLS_ENTRIES];
|
|
|
|
unsigned long sp0;
|
|
|
|
unsigned long sp;
|
2008-01-30 12:31:31 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-02-21 03:24:40 +00:00
|
|
|
unsigned long sysenter_cs;
|
2008-01-30 12:31:31 +00:00
|
|
|
#else
|
2008-02-21 03:24:40 +00:00
|
|
|
unsigned short es;
|
|
|
|
unsigned short ds;
|
|
|
|
unsigned short fsindex;
|
|
|
|
unsigned short gsindex;
|
2008-01-30 12:31:31 +00:00
|
|
|
#endif
|
2016-09-13 21:29:21 +00:00
|
|
|
|
|
|
|
u32 status; /* thread synchronous flags */
|
|
|
|
|
2009-05-03 23:29:52 +00:00
|
|
|
#ifdef CONFIG_X86_64
|
2016-04-26 19:23:29 +00:00
|
|
|
unsigned long fsbase;
|
|
|
|
unsigned long gsbase;
|
|
|
|
#else
|
|
|
|
/*
|
|
|
|
* XXX: this could presumably be unsigned short. Alternatively,
|
|
|
|
* 32-bit kernels could be taught to use fsindex instead.
|
|
|
|
*/
|
|
|
|
unsigned long fs;
|
|
|
|
unsigned long gs;
|
2009-05-03 23:29:52 +00:00
|
|
|
#endif
|
2015-04-23 10:49:20 +00:00
|
|
|
|
2009-09-09 17:22:48 +00:00
|
|
|
/* Save middle states of ptrace breakpoints */
|
|
|
|
struct perf_event *ptrace_bps[HBP_NUM];
|
|
|
|
/* Debug status used for traps, single steps, etc... */
|
|
|
|
unsigned long debugreg6;
|
2010-02-18 17:24:18 +00:00
|
|
|
/* Keep track of the exact dr7 value set by the user */
|
|
|
|
unsigned long ptrace_dr7;
|
2008-02-21 03:24:40 +00:00
|
|
|
/* Fault info: */
|
|
|
|
unsigned long cr2;
|
2012-03-12 09:25:55 +00:00
|
|
|
unsigned long trap_nr;
|
2008-02-21 03:24:40 +00:00
|
|
|
unsigned long error_code;
|
2015-07-29 05:41:16 +00:00
|
|
|
#ifdef CONFIG_VM86
|
2008-02-21 03:24:40 +00:00
|
|
|
/* Virtual 86 mode info */
|
2015-07-29 05:41:16 +00:00
|
|
|
struct vm86 *vm86;
|
2008-01-30 12:31:31 +00:00
|
|
|
#endif
|
2008-02-21 03:24:40 +00:00
|
|
|
/* IO permissions: */
|
|
|
|
unsigned long *io_bitmap_ptr;
|
|
|
|
unsigned long iopl;
|
|
|
|
/* Max allowed port in the bitmap, in bytes: */
|
|
|
|
unsigned io_bitmap_max;
|
2015-07-17 10:28:11 +00:00
|
|
|
|
2016-07-14 20:22:57 +00:00
|
|
|
mm_segment_t addr_limit;
|
|
|
|
|
2016-07-15 08:21:11 +00:00
|
|
|
unsigned int sig_on_uaccess_err:1;
|
2016-07-14 20:22:56 +00:00
|
|
|
unsigned int uaccess_err:1; /* uaccess failed */
|
|
|
|
|
2015-07-17 10:28:11 +00:00
|
|
|
/* Floating point and extended processor state */
|
|
|
|
struct fpu fpu;
|
|
|
|
/*
|
|
|
|
* WARNING: 'fpu' is dynamically-sized. It *MUST* be at
|
|
|
|
* the end.
|
|
|
|
*/
|
2008-01-30 12:31:31 +00:00
|
|
|
};
|
|
|
|
|
2016-09-13 21:29:21 +00:00
|
|
|
/*
|
|
|
|
* Thread-synchronous status.
|
|
|
|
*
|
|
|
|
* This is different from the flags in that nobody else
|
|
|
|
* ever touches our thread-synchronous status, so we don't
|
|
|
|
* have to worry about atomic accesses.
|
|
|
|
*/
|
|
|
|
#define TS_COMPAT 0x0002 /* 32bit syscall active (64BIT)*/
|
|
|
|
|
2008-01-30 12:31:27 +00:00
|
|
|
/*
|
|
|
|
* Set IOPL bits in EFLAGS from given mask
|
|
|
|
*/
|
|
|
|
static inline void native_set_iopl_mask(unsigned mask)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
unsigned int reg;
|
2008-02-21 03:24:40 +00:00
|
|
|
|
2008-03-23 08:03:15 +00:00
|
|
|
asm volatile ("pushfl;"
|
|
|
|
"popl %0;"
|
|
|
|
"andl %1, %0;"
|
|
|
|
"orl %2, %0;"
|
|
|
|
"pushl %0;"
|
|
|
|
"popfl"
|
|
|
|
: "=&r" (reg)
|
|
|
|
: "i" (~X86_EFLAGS_IOPL), "r" (mask));
|
2008-01-30 12:31:27 +00:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
static inline void
|
|
|
|
native_load_sp0(struct tss_struct *tss, struct thread_struct *thread)
|
2008-01-30 12:31:31 +00:00
|
|
|
{
|
|
|
|
tss->x86_tss.sp0 = thread->sp0;
|
|
|
|
#ifdef CONFIG_X86_32
|
2008-02-21 03:24:40 +00:00
|
|
|
/* Only happens when SEP is enabled, no need to test "SEP"arately: */
|
2008-01-30 12:31:31 +00:00
|
|
|
if (unlikely(tss->x86_tss.ss1 != thread->sysenter_cs)) {
|
|
|
|
tss->x86_tss.ss1 = thread->sysenter_cs;
|
|
|
|
wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
2008-01-30 12:31:27 +00:00
|
|
|
|
2008-01-30 12:32:08 +00:00
|
|
|
static inline void native_swapgs(void)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
asm volatile("swapgs" ::: "memory");
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2015-03-07 01:50:19 +00:00
|
|
|
static inline unsigned long current_top_of_stack(void)
|
2015-03-06 03:19:02 +00:00
|
|
|
{
|
2015-03-07 01:50:19 +00:00
|
|
|
#ifdef CONFIG_X86_64
|
2015-03-06 03:19:05 +00:00
|
|
|
return this_cpu_read_stable(cpu_tss.x86_tss.sp0);
|
2015-03-07 01:50:19 +00:00
|
|
|
#else
|
|
|
|
/* sp0 on x86_32 is special in and around vm86 mode. */
|
|
|
|
return this_cpu_read_stable(cpu_current_top_of_stack);
|
|
|
|
#endif
|
2015-03-06 03:19:02 +00:00
|
|
|
}
|
|
|
|
|
2008-01-30 12:31:31 +00:00
|
|
|
#ifdef CONFIG_PARAVIRT
|
|
|
|
#include <asm/paravirt.h>
|
|
|
|
#else
|
2008-02-21 03:24:40 +00:00
|
|
|
#define __cpuid native_cpuid
|
2008-01-30 12:31:27 +00:00
|
|
|
|
2008-03-23 08:03:15 +00:00
|
|
|
static inline void load_sp0(struct tss_struct *tss,
|
|
|
|
struct thread_struct *thread)
|
2008-01-30 12:31:31 +00:00
|
|
|
{
|
|
|
|
native_load_sp0(tss, thread);
|
|
|
|
}
|
|
|
|
|
2008-01-30 12:31:27 +00:00
|
|
|
#define set_iopl_mask native_set_iopl_mask
|
2008-01-30 12:31:27 +00:00
|
|
|
#endif /* CONFIG_PARAVIRT */
|
|
|
|
|
2008-01-30 12:31:27 +00:00
|
|
|
/* Free all resources held by a thread. */
|
|
|
|
extern void release_thread(struct task_struct *);
|
|
|
|
|
|
|
|
unsigned long get_wchan(struct task_struct *p);
|
2008-01-30 12:31:03 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Generic CPUID function
|
|
|
|
* clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
|
|
|
|
* resulting in stale register contents being returned.
|
|
|
|
*/
|
|
|
|
static inline void cpuid(unsigned int op,
|
|
|
|
unsigned int *eax, unsigned int *ebx,
|
|
|
|
unsigned int *ecx, unsigned int *edx)
|
|
|
|
{
|
|
|
|
*eax = op;
|
|
|
|
*ecx = 0;
|
|
|
|
__cpuid(eax, ebx, ecx, edx);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Some CPUID calls want 'count' to be placed in ecx */
|
|
|
|
static inline void cpuid_count(unsigned int op, int count,
|
|
|
|
unsigned int *eax, unsigned int *ebx,
|
|
|
|
unsigned int *ecx, unsigned int *edx)
|
|
|
|
{
|
|
|
|
*eax = op;
|
|
|
|
*ecx = count;
|
|
|
|
__cpuid(eax, ebx, ecx, edx);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* CPUID functions returning a single datum
|
|
|
|
*/
|
|
|
|
static inline unsigned int cpuid_eax(unsigned int op)
|
|
|
|
{
|
|
|
|
unsigned int eax, ebx, ecx, edx;
|
|
|
|
|
|
|
|
cpuid(op, &eax, &ebx, &ecx, &edx);
|
2008-02-21 03:24:40 +00:00
|
|
|
|
2008-01-30 12:31:03 +00:00
|
|
|
return eax;
|
|
|
|
}
|
2008-02-21 03:24:40 +00:00
|
|
|
|
2008-01-30 12:31:03 +00:00
|
|
|
static inline unsigned int cpuid_ebx(unsigned int op)
|
|
|
|
{
|
|
|
|
unsigned int eax, ebx, ecx, edx;
|
|
|
|
|
|
|
|
cpuid(op, &eax, &ebx, &ecx, &edx);
|
2008-02-21 03:24:40 +00:00
|
|
|
|
2008-01-30 12:31:03 +00:00
|
|
|
return ebx;
|
|
|
|
}
|
2008-02-21 03:24:40 +00:00
|
|
|
|
2008-01-30 12:31:03 +00:00
|
|
|
static inline unsigned int cpuid_ecx(unsigned int op)
|
|
|
|
{
|
|
|
|
unsigned int eax, ebx, ecx, edx;
|
|
|
|
|
|
|
|
cpuid(op, &eax, &ebx, &ecx, &edx);
|
2008-02-21 03:24:40 +00:00
|
|
|
|
2008-01-30 12:31:03 +00:00
|
|
|
return ecx;
|
|
|
|
}
|
2008-02-21 03:24:40 +00:00
|
|
|
|
2008-01-30 12:31:03 +00:00
|
|
|
static inline unsigned int cpuid_edx(unsigned int op)
|
|
|
|
{
|
|
|
|
unsigned int eax, ebx, ecx, edx;
|
|
|
|
|
|
|
|
cpuid(op, &eax, &ebx, &ecx, &edx);
|
2008-02-21 03:24:40 +00:00
|
|
|
|
2008-01-30 12:31:03 +00:00
|
|
|
return edx;
|
|
|
|
}
|
|
|
|
|
2008-01-30 12:31:27 +00:00
|
|
|
/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
|
2015-09-24 12:02:29 +00:00
|
|
|
static __always_inline void rep_nop(void)
|
2008-01-30 12:31:27 +00:00
|
|
|
{
|
2008-03-23 08:03:15 +00:00
|
|
|
asm volatile("rep; nop" ::: "memory");
|
2008-01-30 12:31:27 +00:00
|
|
|
}
|
|
|
|
|
2015-09-24 12:02:29 +00:00
|
|
|
static __always_inline void cpu_relax(void)
|
2008-02-21 03:24:40 +00:00
|
|
|
{
|
|
|
|
rep_nop();
|
|
|
|
}
|
|
|
|
|
2016-12-09 18:24:08 +00:00
|
|
|
/*
|
|
|
|
* This function forces the icache and prefetched instruction stream to
|
|
|
|
* catch up with reality in two very specific cases:
|
|
|
|
*
|
|
|
|
* a) Text was modified using one virtual address and is about to be executed
|
|
|
|
* from the same physical page at a different virtual address.
|
|
|
|
*
|
|
|
|
* b) Text was modified on a different CPU, may subsequently be
|
|
|
|
* executed on this CPU, and you want to make sure the new version
|
|
|
|
* gets executed. This generally means you're calling this in a IPI.
|
|
|
|
*
|
|
|
|
* If you're calling this for a different reason, you're probably doing
|
|
|
|
* it wrong.
|
|
|
|
*/
|
2008-01-30 12:31:27 +00:00
|
|
|
static inline void sync_core(void)
|
|
|
|
{
|
2012-11-28 19:50:30 +00:00
|
|
|
/*
|
2016-12-09 18:24:08 +00:00
|
|
|
* There are quite a few ways to do this. IRET-to-self is nice
|
|
|
|
* because it works on every CPU, at any CPL (so it's compatible
|
|
|
|
* with paravirtualization), and it never exits to a hypervisor.
|
|
|
|
* The only down sides are that it's a bit slow (it seems to be
|
|
|
|
* a bit more than 2x slower than the fastest options) and that
|
|
|
|
* it unmasks NMIs. The "push %cs" is needed because, in
|
|
|
|
* paravirtual environments, __KERNEL_CS may not be a valid CS
|
|
|
|
* value when we do IRET directly.
|
|
|
|
*
|
|
|
|
* In case NMI unmasking or performance ever becomes a problem,
|
|
|
|
* the next best option appears to be MOV-to-CR2 and an
|
|
|
|
* unconditional jump. That sequence also works on all CPUs,
|
|
|
|
* but it will fault at CPL3 (i.e. Xen PV and lguest).
|
|
|
|
*
|
|
|
|
* CPUID is the conventional way, but it's nasty: it doesn't
|
|
|
|
* exist on some 486-like CPUs, and it usually exits to a
|
|
|
|
* hypervisor.
|
|
|
|
*
|
|
|
|
* Like all of Linux's memory ordering operations, this is a
|
|
|
|
* compiler barrier as well.
|
2012-11-28 19:50:30 +00:00
|
|
|
*/
|
2016-12-09 18:24:08 +00:00
|
|
|
register void *__sp asm(_ASM_SP);
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
asm volatile (
|
|
|
|
"pushfl\n\t"
|
|
|
|
"pushl %%cs\n\t"
|
|
|
|
"pushl $1f\n\t"
|
|
|
|
"iret\n\t"
|
|
|
|
"1:"
|
|
|
|
: "+r" (__sp) : : "memory");
|
2012-11-28 19:50:30 +00:00
|
|
|
#else
|
2016-12-09 18:24:08 +00:00
|
|
|
unsigned int tmp;
|
|
|
|
|
|
|
|
asm volatile (
|
|
|
|
"mov %%ss, %0\n\t"
|
|
|
|
"pushq %q0\n\t"
|
|
|
|
"pushq %%rsp\n\t"
|
|
|
|
"addq $8, (%%rsp)\n\t"
|
|
|
|
"pushfq\n\t"
|
|
|
|
"mov %%cs, %0\n\t"
|
|
|
|
"pushq %q0\n\t"
|
|
|
|
"pushq $1f\n\t"
|
|
|
|
"iretq\n\t"
|
|
|
|
"1:"
|
|
|
|
: "=&r" (tmp), "+r" (__sp) : : "cc", "memory");
|
2009-09-10 01:53:50 +00:00
|
|
|
#endif
|
2008-01-30 12:31:27 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
extern void select_idle_routine(const struct cpuinfo_x86 *c);
|
2016-12-09 18:29:11 +00:00
|
|
|
extern void amd_e400_c1e_apic_setup(void);
|
2008-01-30 12:31:27 +00:00
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
extern unsigned long boot_option_idle_override;
|
2008-01-30 12:31:27 +00:00
|
|
|
|
2010-11-03 16:06:14 +00:00
|
|
|
enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
|
2013-02-10 06:38:39 +00:00
|
|
|
IDLE_POLL};
|
2010-11-03 16:06:14 +00:00
|
|
|
|
2008-01-30 12:31:39 +00:00
|
|
|
extern void enable_sep_cpu(void);
|
|
|
|
extern int sysenter_setup(void);
|
|
|
|
|
2010-05-21 02:04:29 +00:00
|
|
|
extern void early_trap_init(void);
|
x86, 64bit: Use a #PF handler to materialize early mappings on demand
Linear mode (CR0.PG = 0) is mutually exclusive with 64-bit mode; all
64-bit code has to use page tables. This makes it awkward before we
have first set up properly all-covering page tables to access objects
that are outside the static kernel range.
So far we have dealt with that simply by mapping a fixed amount of
low memory, but that fails in at least two upcoming use cases:
1. We will support load and run kernel, struct boot_params, ramdisk,
command line, etc. above the 4 GiB mark.
2. need to access ramdisk early to get microcode to update that as
early possible.
We could use early_iomap to access them too, but it will make code to
messy and hard to be unified with 32 bit.
Hence, set up a #PF table and use a fixed number of buffers to set up
page tables on demand. If the buffers fill up then we simply flush
them and start over. These buffers are all in __initdata, so it does
not increase RAM usage at runtime.
Thus, with the help of the #PF handler, we can set the final kernel
mapping from blank, and switch to init_level4_pgt later.
During the switchover in head_64.S, before #PF handler is available,
we use three pages to handle kernel crossing 1G, 512G boundaries with
sharing page by playing games with page aliasing: the same page is
mapped twice in the higher-level tables with appropriate wraparound.
The kernel region itself will be properly mapped; other mappings may
be spurious.
early_make_pgtable is using kernel high mapping address to access pages
to set page table.
-v4: Add phys_base offset to make kexec happy, and add
init_mapping_kernel() - Yinghai
-v5: fix compiling with xen, and add back ident level3 and level2 for xen
also move back init_level4_pgt from BSS to DATA again.
because we have to clear it anyway. - Yinghai
-v6: switch to init_level4_pgt in init_mem_mapping. - Yinghai
-v7: remove not needed clear_page for init_level4_page
it is with fill 512,8,0 already in head_64.S - Yinghai
-v8: we need to keep that handler alive until init_mem_mapping and don't
let early_trap_init to trash that early #PF handler.
So split early_trap_pf_init out and move it down. - Yinghai
-v9: switchover only cover kernel space instead of 1G so could avoid
touch possible mem holes. - Yinghai
-v11: change far jmp back to far return to initial_code, that is needed
to fix failure that is reported by Konrad on AMD systems. - Yinghai
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Link: http://lkml.kernel.org/r/1359058816-7615-12-git-send-email-yinghai@kernel.org
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2013-01-24 20:19:52 +00:00
|
|
|
void early_trap_pf_init(void);
|
2010-05-21 02:04:29 +00:00
|
|
|
|
2008-01-30 12:31:39 +00:00
|
|
|
/* Defined in head.S */
|
2008-02-21 03:24:40 +00:00
|
|
|
extern struct desc_ptr early_gdt_descr;
|
2008-01-30 12:31:39 +00:00
|
|
|
|
|
|
|
extern void cpu_set_gdt(int);
|
2009-01-30 08:47:53 +00:00
|
|
|
extern void switch_to_new_gdt(int);
|
2009-01-30 08:47:54 +00:00
|
|
|
extern void load_percpu_segment(int);
|
2008-01-30 12:31:39 +00:00
|
|
|
extern void cpu_init(void);
|
|
|
|
|
2008-12-11 12:49:59 +00:00
|
|
|
static inline unsigned long get_debugctlmsr(void)
|
|
|
|
{
|
2010-03-25 13:51:51 +00:00
|
|
|
unsigned long debugctlmsr = 0;
|
2008-12-11 12:49:59 +00:00
|
|
|
|
|
|
|
#ifndef CONFIG_X86_DEBUGCTLMSR
|
|
|
|
if (boot_cpu_data.x86 < 6)
|
|
|
|
return 0;
|
|
|
|
#endif
|
|
|
|
rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
|
|
|
|
|
2010-03-25 13:51:51 +00:00
|
|
|
return debugctlmsr;
|
2008-12-11 12:49:59 +00:00
|
|
|
}
|
|
|
|
|
2008-03-10 13:11:17 +00:00
|
|
|
static inline void update_debugctlmsr(unsigned long debugctlmsr)
|
|
|
|
{
|
|
|
|
#ifndef CONFIG_X86_DEBUGCTLMSR
|
|
|
|
if (boot_cpu_data.x86 < 6)
|
|
|
|
return;
|
|
|
|
#endif
|
|
|
|
wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
|
|
|
|
}
|
|
|
|
|
2012-09-03 13:24:17 +00:00
|
|
|
extern void set_task_blockstep(struct task_struct *task, bool on);
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
/* Boot loader type from the setup header: */
|
|
|
|
extern int bootloader_type;
|
2009-05-07 23:54:11 +00:00
|
|
|
extern int bootloader_version;
|
2008-01-30 12:31:39 +00:00
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
extern char ignore_fpu_irq;
|
2008-01-30 12:31:27 +00:00
|
|
|
|
|
|
|
#define HAVE_ARCH_PICK_MMAP_LAYOUT 1
|
|
|
|
#define ARCH_HAS_PREFETCHW
|
|
|
|
#define ARCH_HAS_SPINLOCK_PREFETCH
|
|
|
|
|
2008-01-30 12:31:40 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
x86/asm: Cleanup prefetch primitives
This is based on a patch originally by hpa.
With the current improvements to the alternatives, we can simply use %P1
as a mem8 operand constraint and rely on the toolchain to generate the
proper instruction sizes. For example, on 32-bit, where we use an empty
old instruction we get:
apply_alternatives: feat: 6*32+8, old: (c104648b, len: 4), repl: (c195566c, len: 4)
c104648b: alt_insn: 90 90 90 90
c195566c: rpl_insn: 0f 0d 4b 5c
...
apply_alternatives: feat: 6*32+8, old: (c18e09b4, len: 3), repl: (c1955948, len: 3)
c18e09b4: alt_insn: 90 90 90
c1955948: rpl_insn: 0f 0d 08
...
apply_alternatives: feat: 6*32+8, old: (c1190cf9, len: 7), repl: (c1955a79, len: 7)
c1190cf9: alt_insn: 90 90 90 90 90 90 90
c1955a79: rpl_insn: 0f 0d 0d a0 d4 85 c1
all with the proper padding done depending on the size of the
replacement instruction the compiler generates.
Signed-off-by: Borislav Petkov <bp@suse.de>
Cc: H. Peter Anvin <hpa@linux.intel.com>
2015-01-18 16:48:18 +00:00
|
|
|
# define BASE_PREFETCH ""
|
2008-02-21 03:24:40 +00:00
|
|
|
# define ARCH_HAS_PREFETCH
|
2008-01-30 12:31:40 +00:00
|
|
|
#else
|
x86/asm: Cleanup prefetch primitives
This is based on a patch originally by hpa.
With the current improvements to the alternatives, we can simply use %P1
as a mem8 operand constraint and rely on the toolchain to generate the
proper instruction sizes. For example, on 32-bit, where we use an empty
old instruction we get:
apply_alternatives: feat: 6*32+8, old: (c104648b, len: 4), repl: (c195566c, len: 4)
c104648b: alt_insn: 90 90 90 90
c195566c: rpl_insn: 0f 0d 4b 5c
...
apply_alternatives: feat: 6*32+8, old: (c18e09b4, len: 3), repl: (c1955948, len: 3)
c18e09b4: alt_insn: 90 90 90
c1955948: rpl_insn: 0f 0d 08
...
apply_alternatives: feat: 6*32+8, old: (c1190cf9, len: 7), repl: (c1955a79, len: 7)
c1190cf9: alt_insn: 90 90 90 90 90 90 90
c1955a79: rpl_insn: 0f 0d 0d a0 d4 85 c1
all with the proper padding done depending on the size of the
replacement instruction the compiler generates.
Signed-off-by: Borislav Petkov <bp@suse.de>
Cc: H. Peter Anvin <hpa@linux.intel.com>
2015-01-18 16:48:18 +00:00
|
|
|
# define BASE_PREFETCH "prefetcht0 %P1"
|
2008-01-30 12:31:40 +00:00
|
|
|
#endif
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
/*
|
|
|
|
* Prefetch instructions for Pentium III (+) and AMD Athlon (+)
|
|
|
|
*
|
|
|
|
* It's not worth to care about 3dnow prefetches for the K6
|
|
|
|
* because they are microcoded there and very slow.
|
|
|
|
*/
|
2008-01-30 12:31:40 +00:00
|
|
|
static inline void prefetch(const void *x)
|
|
|
|
{
|
x86/asm: Cleanup prefetch primitives
This is based on a patch originally by hpa.
With the current improvements to the alternatives, we can simply use %P1
as a mem8 operand constraint and rely on the toolchain to generate the
proper instruction sizes. For example, on 32-bit, where we use an empty
old instruction we get:
apply_alternatives: feat: 6*32+8, old: (c104648b, len: 4), repl: (c195566c, len: 4)
c104648b: alt_insn: 90 90 90 90
c195566c: rpl_insn: 0f 0d 4b 5c
...
apply_alternatives: feat: 6*32+8, old: (c18e09b4, len: 3), repl: (c1955948, len: 3)
c18e09b4: alt_insn: 90 90 90
c1955948: rpl_insn: 0f 0d 08
...
apply_alternatives: feat: 6*32+8, old: (c1190cf9, len: 7), repl: (c1955a79, len: 7)
c1190cf9: alt_insn: 90 90 90 90 90 90 90
c1955a79: rpl_insn: 0f 0d 0d a0 d4 85 c1
all with the proper padding done depending on the size of the
replacement instruction the compiler generates.
Signed-off-by: Borislav Petkov <bp@suse.de>
Cc: H. Peter Anvin <hpa@linux.intel.com>
2015-01-18 16:48:18 +00:00
|
|
|
alternative_input(BASE_PREFETCH, "prefetchnta %P1",
|
2008-01-30 12:31:40 +00:00
|
|
|
X86_FEATURE_XMM,
|
x86/asm: Cleanup prefetch primitives
This is based on a patch originally by hpa.
With the current improvements to the alternatives, we can simply use %P1
as a mem8 operand constraint and rely on the toolchain to generate the
proper instruction sizes. For example, on 32-bit, where we use an empty
old instruction we get:
apply_alternatives: feat: 6*32+8, old: (c104648b, len: 4), repl: (c195566c, len: 4)
c104648b: alt_insn: 90 90 90 90
c195566c: rpl_insn: 0f 0d 4b 5c
...
apply_alternatives: feat: 6*32+8, old: (c18e09b4, len: 3), repl: (c1955948, len: 3)
c18e09b4: alt_insn: 90 90 90
c1955948: rpl_insn: 0f 0d 08
...
apply_alternatives: feat: 6*32+8, old: (c1190cf9, len: 7), repl: (c1955a79, len: 7)
c1190cf9: alt_insn: 90 90 90 90 90 90 90
c1955a79: rpl_insn: 0f 0d 0d a0 d4 85 c1
all with the proper padding done depending on the size of the
replacement instruction the compiler generates.
Signed-off-by: Borislav Petkov <bp@suse.de>
Cc: H. Peter Anvin <hpa@linux.intel.com>
2015-01-18 16:48:18 +00:00
|
|
|
"m" (*(const char *)x));
|
2008-01-30 12:31:40 +00:00
|
|
|
}
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
/*
|
|
|
|
* 3dnow prefetch to get an exclusive cache line.
|
|
|
|
* Useful for spinlocks to avoid one state transition in the
|
|
|
|
* cache coherency protocol:
|
|
|
|
*/
|
2008-01-30 12:31:40 +00:00
|
|
|
static inline void prefetchw(const void *x)
|
|
|
|
{
|
x86/asm: Cleanup prefetch primitives
This is based on a patch originally by hpa.
With the current improvements to the alternatives, we can simply use %P1
as a mem8 operand constraint and rely on the toolchain to generate the
proper instruction sizes. For example, on 32-bit, where we use an empty
old instruction we get:
apply_alternatives: feat: 6*32+8, old: (c104648b, len: 4), repl: (c195566c, len: 4)
c104648b: alt_insn: 90 90 90 90
c195566c: rpl_insn: 0f 0d 4b 5c
...
apply_alternatives: feat: 6*32+8, old: (c18e09b4, len: 3), repl: (c1955948, len: 3)
c18e09b4: alt_insn: 90 90 90
c1955948: rpl_insn: 0f 0d 08
...
apply_alternatives: feat: 6*32+8, old: (c1190cf9, len: 7), repl: (c1955a79, len: 7)
c1190cf9: alt_insn: 90 90 90 90 90 90 90
c1955a79: rpl_insn: 0f 0d 0d a0 d4 85 c1
all with the proper padding done depending on the size of the
replacement instruction the compiler generates.
Signed-off-by: Borislav Petkov <bp@suse.de>
Cc: H. Peter Anvin <hpa@linux.intel.com>
2015-01-18 16:48:18 +00:00
|
|
|
alternative_input(BASE_PREFETCH, "prefetchw %P1",
|
|
|
|
X86_FEATURE_3DNOWPREFETCH,
|
|
|
|
"m" (*(const char *)x));
|
2008-01-30 12:31:40 +00:00
|
|
|
}
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
static inline void spin_lock_prefetch(const void *x)
|
|
|
|
{
|
|
|
|
prefetchw(x);
|
|
|
|
}
|
|
|
|
|
2015-03-10 18:05:59 +00:00
|
|
|
#define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
|
|
|
|
TOP_OF_KERNEL_STACK_PADDING)
|
|
|
|
|
2008-01-30 12:31:57 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
/*
|
|
|
|
* User space process size: 3GB (default).
|
|
|
|
*/
|
2008-02-21 03:24:40 +00:00
|
|
|
#define TASK_SIZE PAGE_OFFSET
|
2009-02-20 22:32:28 +00:00
|
|
|
#define TASK_SIZE_MAX TASK_SIZE
|
2008-02-21 03:24:40 +00:00
|
|
|
#define STACK_TOP TASK_SIZE
|
|
|
|
#define STACK_TOP_MAX STACK_TOP
|
|
|
|
|
|
|
|
#define INIT_THREAD { \
|
2015-03-10 18:05:59 +00:00
|
|
|
.sp0 = TOP_OF_INIT_STACK, \
|
2008-02-21 03:24:40 +00:00
|
|
|
.sysenter_cs = __KERNEL_CS, \
|
|
|
|
.io_bitmap_ptr = NULL, \
|
2016-07-14 20:22:57 +00:00
|
|
|
.addr_limit = KERNEL_DS, \
|
2008-01-30 12:31:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2015-03-13 14:09:03 +00:00
|
|
|
* TOP_OF_KERNEL_STACK_PADDING reserves 8 bytes on top of the ring0 stack.
|
2008-01-30 12:31:57 +00:00
|
|
|
* This is necessary to guarantee that the entire "struct pt_regs"
|
tree-wide: fix comment/printk typos
"gadget", "through", "command", "maintain", "maintain", "controller", "address",
"between", "initiali[zs]e", "instead", "function", "select", "already",
"equal", "access", "management", "hierarchy", "registration", "interest",
"relative", "memory", "offset", "already",
Signed-off-by: Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
Signed-off-by: Jiri Kosina <jkosina@suse.cz>
2010-11-01 19:38:34 +00:00
|
|
|
* is accessible even if the CPU haven't stored the SS/ESP registers
|
2008-01-30 12:31:57 +00:00
|
|
|
* on the stack (interrupt gate does not save these registers
|
|
|
|
* when switching to the same priv ring).
|
|
|
|
* Therefore beware: accessing the ss/esp fields of the
|
|
|
|
* "struct pt_regs" is possible, but they may contain the
|
|
|
|
* completely wrong values.
|
|
|
|
*/
|
2015-03-13 14:09:03 +00:00
|
|
|
#define task_pt_regs(task) \
|
|
|
|
({ \
|
|
|
|
unsigned long __ptr = (unsigned long)task_stack_page(task); \
|
|
|
|
__ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING; \
|
|
|
|
((struct pt_regs *)__ptr) - 1; \
|
2008-01-30 12:31:57 +00:00
|
|
|
})
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
#define KSTK_ESP(task) (task_pt_regs(task)->sp)
|
2008-01-30 12:31:57 +00:00
|
|
|
|
|
|
|
#else
|
|
|
|
/*
|
2014-11-04 23:46:21 +00:00
|
|
|
* User space process size. 47bits minus one guard page. The guard
|
|
|
|
* page is necessary on Intel CPUs: if a SYSCALL instruction is at
|
|
|
|
* the highest possible canonical userspace address, then that
|
|
|
|
* syscall will enter the kernel with a non-canonical return
|
|
|
|
* address, and SYSRET will explode dangerously. We avoid this
|
|
|
|
* particular problem by preventing anything from being mapped
|
|
|
|
* at the maximum canonical address.
|
2008-01-30 12:31:57 +00:00
|
|
|
*/
|
2009-02-20 22:32:28 +00:00
|
|
|
#define TASK_SIZE_MAX ((1UL << 47) - PAGE_SIZE)
|
2008-01-30 12:31:57 +00:00
|
|
|
|
|
|
|
/* This decides where the kernel will search for a free chunk of vm
|
|
|
|
* space during mmap's.
|
|
|
|
*/
|
2008-02-21 03:24:40 +00:00
|
|
|
#define IA32_PAGE_OFFSET ((current->personality & ADDR_LIMIT_3GB) ? \
|
|
|
|
0xc0000000 : 0xFFFFe000)
|
2008-01-30 12:31:57 +00:00
|
|
|
|
2012-02-06 21:03:09 +00:00
|
|
|
#define TASK_SIZE (test_thread_flag(TIF_ADDR32) ? \
|
2009-02-20 22:32:28 +00:00
|
|
|
IA32_PAGE_OFFSET : TASK_SIZE_MAX)
|
2012-02-06 21:03:09 +00:00
|
|
|
#define TASK_SIZE_OF(child) ((test_tsk_thread_flag(child, TIF_ADDR32)) ? \
|
2009-02-20 22:32:28 +00:00
|
|
|
IA32_PAGE_OFFSET : TASK_SIZE_MAX)
|
2008-01-30 12:31:57 +00:00
|
|
|
|
2008-02-08 12:19:26 +00:00
|
|
|
#define STACK_TOP TASK_SIZE
|
2009-02-20 22:32:28 +00:00
|
|
|
#define STACK_TOP_MAX TASK_SIZE_MAX
|
2008-02-08 12:19:26 +00:00
|
|
|
|
2016-07-14 20:22:57 +00:00
|
|
|
#define INIT_THREAD { \
|
|
|
|
.sp0 = TOP_OF_INIT_STACK, \
|
|
|
|
.addr_limit = KERNEL_DS, \
|
2008-01-30 12:31:57 +00:00
|
|
|
}
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
#define task_pt_regs(tsk) ((struct pt_regs *)(tsk)->thread.sp0 - 1)
|
2009-11-03 09:22:40 +00:00
|
|
|
extern unsigned long KSTK_ESP(struct task_struct *task);
|
2012-02-14 21:49:48 +00:00
|
|
|
|
2008-01-30 12:31:57 +00:00
|
|
|
#endif /* CONFIG_X86_64 */
|
|
|
|
|
2016-08-13 16:38:21 +00:00
|
|
|
extern unsigned long thread_saved_pc(struct task_struct *tsk);
|
|
|
|
|
2008-02-21 04:18:40 +00:00
|
|
|
extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
|
|
|
|
unsigned long new_sp);
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
/*
|
|
|
|
* This decides where the kernel will search for a free chunk of vm
|
2008-01-30 12:31:27 +00:00
|
|
|
* space during mmap's.
|
|
|
|
*/
|
|
|
|
#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 3))
|
|
|
|
|
2008-02-21 03:24:40 +00:00
|
|
|
#define KSTK_EIP(task) (task_pt_regs(task)->ip)
|
2008-01-30 12:31:27 +00:00
|
|
|
|
2008-04-13 22:24:18 +00:00
|
|
|
/* Get/set a process' ability to use the timestamp counter instruction */
|
|
|
|
#define GET_TSC_CTL(adr) get_tsc_mode((adr))
|
|
|
|
#define SET_TSC_CTL(val) set_tsc_mode((val))
|
|
|
|
|
|
|
|
extern int get_tsc_mode(unsigned long adr);
|
|
|
|
extern int set_tsc_mode(unsigned int val);
|
|
|
|
|
x86, mpx: On-demand kernel allocation of bounds tables
This is really the meat of the MPX patch set. If there is one patch to
review in the entire series, this is the one. There is a new ABI here
and this kernel code also interacts with userspace memory in a
relatively unusual manner. (small FAQ below).
Long Description:
This patch adds two prctl() commands to provide enable or disable the
management of bounds tables in kernel, including on-demand kernel
allocation (See the patch "on-demand kernel allocation of bounds tables")
and cleanup (See the patch "cleanup unused bound tables"). Applications
do not strictly need the kernel to manage bounds tables and we expect
some applications to use MPX without taking advantage of this kernel
support. This means the kernel can not simply infer whether an application
needs bounds table management from the MPX registers. The prctl() is an
explicit signal from userspace.
PR_MPX_ENABLE_MANAGEMENT is meant to be a signal from userspace to
require kernel's help in managing bounds tables.
PR_MPX_DISABLE_MANAGEMENT is the opposite, meaning that userspace don't
want kernel's help any more. With PR_MPX_DISABLE_MANAGEMENT, the kernel
won't allocate and free bounds tables even if the CPU supports MPX.
PR_MPX_ENABLE_MANAGEMENT will fetch the base address of the bounds
directory out of a userspace register (bndcfgu) and then cache it into
a new field (->bd_addr) in the 'mm_struct'. PR_MPX_DISABLE_MANAGEMENT
will set "bd_addr" to an invalid address. Using this scheme, we can
use "bd_addr" to determine whether the management of bounds tables in
kernel is enabled.
Also, the only way to access that bndcfgu register is via an xsaves,
which can be expensive. Caching "bd_addr" like this also helps reduce
the cost of those xsaves when doing table cleanup at munmap() time.
Unfortunately, we can not apply this optimization to #BR fault time
because we need an xsave to get the value of BNDSTATUS.
==== Why does the hardware even have these Bounds Tables? ====
MPX only has 4 hardware registers for storing bounds information.
If MPX-enabled code needs more than these 4 registers, it needs to
spill them somewhere. It has two special instructions for this
which allow the bounds to be moved between the bounds registers
and some new "bounds tables".
They are similar conceptually to a page fault and will be raised by
the MPX hardware during both bounds violations or when the tables
are not present. This patch handles those #BR exceptions for
not-present tables by carving the space out of the normal processes
address space (essentially calling the new mmap() interface indroduced
earlier in this patch set.) and then pointing the bounds-directory
over to it.
The tables *need* to be accessed and controlled by userspace because
the instructions for moving bounds in and out of them are extremely
frequent. They potentially happen every time a register pointing to
memory is dereferenced. Any direct kernel involvement (like a syscall)
to access the tables would obviously destroy performance.
==== Why not do this in userspace? ====
This patch is obviously doing this allocation in the kernel.
However, MPX does not strictly *require* anything in the kernel.
It can theoretically be done completely from userspace. Here are
a few ways this *could* be done. I don't think any of them are
practical in the real-world, but here they are.
Q: Can virtual space simply be reserved for the bounds tables so
that we never have to allocate them?
A: As noted earlier, these tables are *HUGE*. An X-GB virtual
area needs 4*X GB of virtual space, plus 2GB for the bounds
directory. If we were to preallocate them for the 128TB of
user virtual address space, we would need to reserve 512TB+2GB,
which is larger than the entire virtual address space today.
This means they can not be reserved ahead of time. Also, a
single process's pre-popualated bounds directory consumes 2GB
of virtual *AND* physical memory. IOW, it's completely
infeasible to prepopulate bounds directories.
Q: Can we preallocate bounds table space at the same time memory
is allocated which might contain pointers that might eventually
need bounds tables?
A: This would work if we could hook the site of each and every
memory allocation syscall. This can be done for small,
constrained applications. But, it isn't practical at a larger
scale since a given app has no way of controlling how all the
parts of the app might allocate memory (think libraries). The
kernel is really the only place to intercept these calls.
Q: Could a bounds fault be handed to userspace and the tables
allocated there in a signal handler instead of in the kernel?
A: (thanks to tglx) mmap() is not on the list of safe async
handler functions and even if mmap() would work it still
requires locking or nasty tricks to keep track of the
allocation state there.
Having ruled out all of the userspace-only approaches for managing
bounds tables that we could think of, we create them on demand in
the kernel.
Based-on-patch-by: Qiaowei Ren <qiaowei.ren@intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Cc: linux-mm@kvack.org
Cc: linux-mips@linux-mips.org
Cc: Dave Hansen <dave@sr71.net>
Link: http://lkml.kernel.org/r/20141114151829.AD4310DE@viggo.jf.intel.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2014-11-14 15:18:29 +00:00
|
|
|
/* Register/unregister a process' MPX related resource */
|
2015-06-07 18:37:02 +00:00
|
|
|
#define MPX_ENABLE_MANAGEMENT() mpx_enable_management()
|
|
|
|
#define MPX_DISABLE_MANAGEMENT() mpx_disable_management()
|
x86, mpx: On-demand kernel allocation of bounds tables
This is really the meat of the MPX patch set. If there is one patch to
review in the entire series, this is the one. There is a new ABI here
and this kernel code also interacts with userspace memory in a
relatively unusual manner. (small FAQ below).
Long Description:
This patch adds two prctl() commands to provide enable or disable the
management of bounds tables in kernel, including on-demand kernel
allocation (See the patch "on-demand kernel allocation of bounds tables")
and cleanup (See the patch "cleanup unused bound tables"). Applications
do not strictly need the kernel to manage bounds tables and we expect
some applications to use MPX without taking advantage of this kernel
support. This means the kernel can not simply infer whether an application
needs bounds table management from the MPX registers. The prctl() is an
explicit signal from userspace.
PR_MPX_ENABLE_MANAGEMENT is meant to be a signal from userspace to
require kernel's help in managing bounds tables.
PR_MPX_DISABLE_MANAGEMENT is the opposite, meaning that userspace don't
want kernel's help any more. With PR_MPX_DISABLE_MANAGEMENT, the kernel
won't allocate and free bounds tables even if the CPU supports MPX.
PR_MPX_ENABLE_MANAGEMENT will fetch the base address of the bounds
directory out of a userspace register (bndcfgu) and then cache it into
a new field (->bd_addr) in the 'mm_struct'. PR_MPX_DISABLE_MANAGEMENT
will set "bd_addr" to an invalid address. Using this scheme, we can
use "bd_addr" to determine whether the management of bounds tables in
kernel is enabled.
Also, the only way to access that bndcfgu register is via an xsaves,
which can be expensive. Caching "bd_addr" like this also helps reduce
the cost of those xsaves when doing table cleanup at munmap() time.
Unfortunately, we can not apply this optimization to #BR fault time
because we need an xsave to get the value of BNDSTATUS.
==== Why does the hardware even have these Bounds Tables? ====
MPX only has 4 hardware registers for storing bounds information.
If MPX-enabled code needs more than these 4 registers, it needs to
spill them somewhere. It has two special instructions for this
which allow the bounds to be moved between the bounds registers
and some new "bounds tables".
They are similar conceptually to a page fault and will be raised by
the MPX hardware during both bounds violations or when the tables
are not present. This patch handles those #BR exceptions for
not-present tables by carving the space out of the normal processes
address space (essentially calling the new mmap() interface indroduced
earlier in this patch set.) and then pointing the bounds-directory
over to it.
The tables *need* to be accessed and controlled by userspace because
the instructions for moving bounds in and out of them are extremely
frequent. They potentially happen every time a register pointing to
memory is dereferenced. Any direct kernel involvement (like a syscall)
to access the tables would obviously destroy performance.
==== Why not do this in userspace? ====
This patch is obviously doing this allocation in the kernel.
However, MPX does not strictly *require* anything in the kernel.
It can theoretically be done completely from userspace. Here are
a few ways this *could* be done. I don't think any of them are
practical in the real-world, but here they are.
Q: Can virtual space simply be reserved for the bounds tables so
that we never have to allocate them?
A: As noted earlier, these tables are *HUGE*. An X-GB virtual
area needs 4*X GB of virtual space, plus 2GB for the bounds
directory. If we were to preallocate them for the 128TB of
user virtual address space, we would need to reserve 512TB+2GB,
which is larger than the entire virtual address space today.
This means they can not be reserved ahead of time. Also, a
single process's pre-popualated bounds directory consumes 2GB
of virtual *AND* physical memory. IOW, it's completely
infeasible to prepopulate bounds directories.
Q: Can we preallocate bounds table space at the same time memory
is allocated which might contain pointers that might eventually
need bounds tables?
A: This would work if we could hook the site of each and every
memory allocation syscall. This can be done for small,
constrained applications. But, it isn't practical at a larger
scale since a given app has no way of controlling how all the
parts of the app might allocate memory (think libraries). The
kernel is really the only place to intercept these calls.
Q: Could a bounds fault be handed to userspace and the tables
allocated there in a signal handler instead of in the kernel?
A: (thanks to tglx) mmap() is not on the list of safe async
handler functions and even if mmap() would work it still
requires locking or nasty tricks to keep track of the
allocation state there.
Having ruled out all of the userspace-only approaches for managing
bounds tables that we could think of, we create them on demand in
the kernel.
Based-on-patch-by: Qiaowei Ren <qiaowei.ren@intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Cc: linux-mm@kvack.org
Cc: linux-mips@linux-mips.org
Cc: Dave Hansen <dave@sr71.net>
Link: http://lkml.kernel.org/r/20141114151829.AD4310DE@viggo.jf.intel.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2014-11-14 15:18:29 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_X86_INTEL_MPX
|
2015-06-07 18:37:02 +00:00
|
|
|
extern int mpx_enable_management(void);
|
|
|
|
extern int mpx_disable_management(void);
|
x86, mpx: On-demand kernel allocation of bounds tables
This is really the meat of the MPX patch set. If there is one patch to
review in the entire series, this is the one. There is a new ABI here
and this kernel code also interacts with userspace memory in a
relatively unusual manner. (small FAQ below).
Long Description:
This patch adds two prctl() commands to provide enable or disable the
management of bounds tables in kernel, including on-demand kernel
allocation (See the patch "on-demand kernel allocation of bounds tables")
and cleanup (See the patch "cleanup unused bound tables"). Applications
do not strictly need the kernel to manage bounds tables and we expect
some applications to use MPX without taking advantage of this kernel
support. This means the kernel can not simply infer whether an application
needs bounds table management from the MPX registers. The prctl() is an
explicit signal from userspace.
PR_MPX_ENABLE_MANAGEMENT is meant to be a signal from userspace to
require kernel's help in managing bounds tables.
PR_MPX_DISABLE_MANAGEMENT is the opposite, meaning that userspace don't
want kernel's help any more. With PR_MPX_DISABLE_MANAGEMENT, the kernel
won't allocate and free bounds tables even if the CPU supports MPX.
PR_MPX_ENABLE_MANAGEMENT will fetch the base address of the bounds
directory out of a userspace register (bndcfgu) and then cache it into
a new field (->bd_addr) in the 'mm_struct'. PR_MPX_DISABLE_MANAGEMENT
will set "bd_addr" to an invalid address. Using this scheme, we can
use "bd_addr" to determine whether the management of bounds tables in
kernel is enabled.
Also, the only way to access that bndcfgu register is via an xsaves,
which can be expensive. Caching "bd_addr" like this also helps reduce
the cost of those xsaves when doing table cleanup at munmap() time.
Unfortunately, we can not apply this optimization to #BR fault time
because we need an xsave to get the value of BNDSTATUS.
==== Why does the hardware even have these Bounds Tables? ====
MPX only has 4 hardware registers for storing bounds information.
If MPX-enabled code needs more than these 4 registers, it needs to
spill them somewhere. It has two special instructions for this
which allow the bounds to be moved between the bounds registers
and some new "bounds tables".
They are similar conceptually to a page fault and will be raised by
the MPX hardware during both bounds violations or when the tables
are not present. This patch handles those #BR exceptions for
not-present tables by carving the space out of the normal processes
address space (essentially calling the new mmap() interface indroduced
earlier in this patch set.) and then pointing the bounds-directory
over to it.
The tables *need* to be accessed and controlled by userspace because
the instructions for moving bounds in and out of them are extremely
frequent. They potentially happen every time a register pointing to
memory is dereferenced. Any direct kernel involvement (like a syscall)
to access the tables would obviously destroy performance.
==== Why not do this in userspace? ====
This patch is obviously doing this allocation in the kernel.
However, MPX does not strictly *require* anything in the kernel.
It can theoretically be done completely from userspace. Here are
a few ways this *could* be done. I don't think any of them are
practical in the real-world, but here they are.
Q: Can virtual space simply be reserved for the bounds tables so
that we never have to allocate them?
A: As noted earlier, these tables are *HUGE*. An X-GB virtual
area needs 4*X GB of virtual space, plus 2GB for the bounds
directory. If we were to preallocate them for the 128TB of
user virtual address space, we would need to reserve 512TB+2GB,
which is larger than the entire virtual address space today.
This means they can not be reserved ahead of time. Also, a
single process's pre-popualated bounds directory consumes 2GB
of virtual *AND* physical memory. IOW, it's completely
infeasible to prepopulate bounds directories.
Q: Can we preallocate bounds table space at the same time memory
is allocated which might contain pointers that might eventually
need bounds tables?
A: This would work if we could hook the site of each and every
memory allocation syscall. This can be done for small,
constrained applications. But, it isn't practical at a larger
scale since a given app has no way of controlling how all the
parts of the app might allocate memory (think libraries). The
kernel is really the only place to intercept these calls.
Q: Could a bounds fault be handed to userspace and the tables
allocated there in a signal handler instead of in the kernel?
A: (thanks to tglx) mmap() is not on the list of safe async
handler functions and even if mmap() would work it still
requires locking or nasty tricks to keep track of the
allocation state there.
Having ruled out all of the userspace-only approaches for managing
bounds tables that we could think of, we create them on demand in
the kernel.
Based-on-patch-by: Qiaowei Ren <qiaowei.ren@intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Cc: linux-mm@kvack.org
Cc: linux-mips@linux-mips.org
Cc: Dave Hansen <dave@sr71.net>
Link: http://lkml.kernel.org/r/20141114151829.AD4310DE@viggo.jf.intel.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2014-11-14 15:18:29 +00:00
|
|
|
#else
|
2015-06-07 18:37:02 +00:00
|
|
|
static inline int mpx_enable_management(void)
|
x86, mpx: On-demand kernel allocation of bounds tables
This is really the meat of the MPX patch set. If there is one patch to
review in the entire series, this is the one. There is a new ABI here
and this kernel code also interacts with userspace memory in a
relatively unusual manner. (small FAQ below).
Long Description:
This patch adds two prctl() commands to provide enable or disable the
management of bounds tables in kernel, including on-demand kernel
allocation (See the patch "on-demand kernel allocation of bounds tables")
and cleanup (See the patch "cleanup unused bound tables"). Applications
do not strictly need the kernel to manage bounds tables and we expect
some applications to use MPX without taking advantage of this kernel
support. This means the kernel can not simply infer whether an application
needs bounds table management from the MPX registers. The prctl() is an
explicit signal from userspace.
PR_MPX_ENABLE_MANAGEMENT is meant to be a signal from userspace to
require kernel's help in managing bounds tables.
PR_MPX_DISABLE_MANAGEMENT is the opposite, meaning that userspace don't
want kernel's help any more. With PR_MPX_DISABLE_MANAGEMENT, the kernel
won't allocate and free bounds tables even if the CPU supports MPX.
PR_MPX_ENABLE_MANAGEMENT will fetch the base address of the bounds
directory out of a userspace register (bndcfgu) and then cache it into
a new field (->bd_addr) in the 'mm_struct'. PR_MPX_DISABLE_MANAGEMENT
will set "bd_addr" to an invalid address. Using this scheme, we can
use "bd_addr" to determine whether the management of bounds tables in
kernel is enabled.
Also, the only way to access that bndcfgu register is via an xsaves,
which can be expensive. Caching "bd_addr" like this also helps reduce
the cost of those xsaves when doing table cleanup at munmap() time.
Unfortunately, we can not apply this optimization to #BR fault time
because we need an xsave to get the value of BNDSTATUS.
==== Why does the hardware even have these Bounds Tables? ====
MPX only has 4 hardware registers for storing bounds information.
If MPX-enabled code needs more than these 4 registers, it needs to
spill them somewhere. It has two special instructions for this
which allow the bounds to be moved between the bounds registers
and some new "bounds tables".
They are similar conceptually to a page fault and will be raised by
the MPX hardware during both bounds violations or when the tables
are not present. This patch handles those #BR exceptions for
not-present tables by carving the space out of the normal processes
address space (essentially calling the new mmap() interface indroduced
earlier in this patch set.) and then pointing the bounds-directory
over to it.
The tables *need* to be accessed and controlled by userspace because
the instructions for moving bounds in and out of them are extremely
frequent. They potentially happen every time a register pointing to
memory is dereferenced. Any direct kernel involvement (like a syscall)
to access the tables would obviously destroy performance.
==== Why not do this in userspace? ====
This patch is obviously doing this allocation in the kernel.
However, MPX does not strictly *require* anything in the kernel.
It can theoretically be done completely from userspace. Here are
a few ways this *could* be done. I don't think any of them are
practical in the real-world, but here they are.
Q: Can virtual space simply be reserved for the bounds tables so
that we never have to allocate them?
A: As noted earlier, these tables are *HUGE*. An X-GB virtual
area needs 4*X GB of virtual space, plus 2GB for the bounds
directory. If we were to preallocate them for the 128TB of
user virtual address space, we would need to reserve 512TB+2GB,
which is larger than the entire virtual address space today.
This means they can not be reserved ahead of time. Also, a
single process's pre-popualated bounds directory consumes 2GB
of virtual *AND* physical memory. IOW, it's completely
infeasible to prepopulate bounds directories.
Q: Can we preallocate bounds table space at the same time memory
is allocated which might contain pointers that might eventually
need bounds tables?
A: This would work if we could hook the site of each and every
memory allocation syscall. This can be done for small,
constrained applications. But, it isn't practical at a larger
scale since a given app has no way of controlling how all the
parts of the app might allocate memory (think libraries). The
kernel is really the only place to intercept these calls.
Q: Could a bounds fault be handed to userspace and the tables
allocated there in a signal handler instead of in the kernel?
A: (thanks to tglx) mmap() is not on the list of safe async
handler functions and even if mmap() would work it still
requires locking or nasty tricks to keep track of the
allocation state there.
Having ruled out all of the userspace-only approaches for managing
bounds tables that we could think of, we create them on demand in
the kernel.
Based-on-patch-by: Qiaowei Ren <qiaowei.ren@intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Cc: linux-mm@kvack.org
Cc: linux-mips@linux-mips.org
Cc: Dave Hansen <dave@sr71.net>
Link: http://lkml.kernel.org/r/20141114151829.AD4310DE@viggo.jf.intel.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2014-11-14 15:18:29 +00:00
|
|
|
{
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
2015-06-07 18:37:02 +00:00
|
|
|
static inline int mpx_disable_management(void)
|
x86, mpx: On-demand kernel allocation of bounds tables
This is really the meat of the MPX patch set. If there is one patch to
review in the entire series, this is the one. There is a new ABI here
and this kernel code also interacts with userspace memory in a
relatively unusual manner. (small FAQ below).
Long Description:
This patch adds two prctl() commands to provide enable or disable the
management of bounds tables in kernel, including on-demand kernel
allocation (See the patch "on-demand kernel allocation of bounds tables")
and cleanup (See the patch "cleanup unused bound tables"). Applications
do not strictly need the kernel to manage bounds tables and we expect
some applications to use MPX without taking advantage of this kernel
support. This means the kernel can not simply infer whether an application
needs bounds table management from the MPX registers. The prctl() is an
explicit signal from userspace.
PR_MPX_ENABLE_MANAGEMENT is meant to be a signal from userspace to
require kernel's help in managing bounds tables.
PR_MPX_DISABLE_MANAGEMENT is the opposite, meaning that userspace don't
want kernel's help any more. With PR_MPX_DISABLE_MANAGEMENT, the kernel
won't allocate and free bounds tables even if the CPU supports MPX.
PR_MPX_ENABLE_MANAGEMENT will fetch the base address of the bounds
directory out of a userspace register (bndcfgu) and then cache it into
a new field (->bd_addr) in the 'mm_struct'. PR_MPX_DISABLE_MANAGEMENT
will set "bd_addr" to an invalid address. Using this scheme, we can
use "bd_addr" to determine whether the management of bounds tables in
kernel is enabled.
Also, the only way to access that bndcfgu register is via an xsaves,
which can be expensive. Caching "bd_addr" like this also helps reduce
the cost of those xsaves when doing table cleanup at munmap() time.
Unfortunately, we can not apply this optimization to #BR fault time
because we need an xsave to get the value of BNDSTATUS.
==== Why does the hardware even have these Bounds Tables? ====
MPX only has 4 hardware registers for storing bounds information.
If MPX-enabled code needs more than these 4 registers, it needs to
spill them somewhere. It has two special instructions for this
which allow the bounds to be moved between the bounds registers
and some new "bounds tables".
They are similar conceptually to a page fault and will be raised by
the MPX hardware during both bounds violations or when the tables
are not present. This patch handles those #BR exceptions for
not-present tables by carving the space out of the normal processes
address space (essentially calling the new mmap() interface indroduced
earlier in this patch set.) and then pointing the bounds-directory
over to it.
The tables *need* to be accessed and controlled by userspace because
the instructions for moving bounds in and out of them are extremely
frequent. They potentially happen every time a register pointing to
memory is dereferenced. Any direct kernel involvement (like a syscall)
to access the tables would obviously destroy performance.
==== Why not do this in userspace? ====
This patch is obviously doing this allocation in the kernel.
However, MPX does not strictly *require* anything in the kernel.
It can theoretically be done completely from userspace. Here are
a few ways this *could* be done. I don't think any of them are
practical in the real-world, but here they are.
Q: Can virtual space simply be reserved for the bounds tables so
that we never have to allocate them?
A: As noted earlier, these tables are *HUGE*. An X-GB virtual
area needs 4*X GB of virtual space, plus 2GB for the bounds
directory. If we were to preallocate them for the 128TB of
user virtual address space, we would need to reserve 512TB+2GB,
which is larger than the entire virtual address space today.
This means they can not be reserved ahead of time. Also, a
single process's pre-popualated bounds directory consumes 2GB
of virtual *AND* physical memory. IOW, it's completely
infeasible to prepopulate bounds directories.
Q: Can we preallocate bounds table space at the same time memory
is allocated which might contain pointers that might eventually
need bounds tables?
A: This would work if we could hook the site of each and every
memory allocation syscall. This can be done for small,
constrained applications. But, it isn't practical at a larger
scale since a given app has no way of controlling how all the
parts of the app might allocate memory (think libraries). The
kernel is really the only place to intercept these calls.
Q: Could a bounds fault be handed to userspace and the tables
allocated there in a signal handler instead of in the kernel?
A: (thanks to tglx) mmap() is not on the list of safe async
handler functions and even if mmap() would work it still
requires locking or nasty tricks to keep track of the
allocation state there.
Having ruled out all of the userspace-only approaches for managing
bounds tables that we could think of, we create them on demand in
the kernel.
Based-on-patch-by: Qiaowei Ren <qiaowei.ren@intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Cc: linux-mm@kvack.org
Cc: linux-mips@linux-mips.org
Cc: Dave Hansen <dave@sr71.net>
Link: http://lkml.kernel.org/r/20141114151829.AD4310DE@viggo.jf.intel.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2014-11-14 15:18:29 +00:00
|
|
|
{
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_X86_INTEL_MPX */
|
|
|
|
|
2012-11-27 06:32:10 +00:00
|
|
|
extern u16 amd_get_nb_id(int cpu);
|
2015-06-15 08:28:15 +00:00
|
|
|
extern u32 amd_get_nodes_per_socket(void);
|
2009-09-16 09:33:40 +00:00
|
|
|
|
2013-07-25 08:54:32 +00:00
|
|
|
static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)
|
|
|
|
{
|
|
|
|
uint32_t base, eax, signature[3];
|
|
|
|
|
|
|
|
for (base = 0x40000000; base < 0x40010000; base += 0x100) {
|
|
|
|
cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);
|
|
|
|
|
|
|
|
if (!memcmp(sig, signature, 12) &&
|
|
|
|
(leaves == 0 || ((eax - base) >= leaves)))
|
|
|
|
return base;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-03-28 17:11:12 +00:00
|
|
|
extern unsigned long arch_align_stack(unsigned long sp);
|
|
|
|
extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
|
|
|
|
|
|
|
|
void default_idle(void);
|
2013-02-10 04:08:07 +00:00
|
|
|
#ifdef CONFIG_XEN
|
|
|
|
bool xen_set_default_idle(void);
|
|
|
|
#else
|
|
|
|
#define xen_set_default_idle 0
|
|
|
|
#endif
|
2012-03-28 17:11:12 +00:00
|
|
|
|
|
|
|
void stop_this_cpu(void *dummy);
|
2013-05-09 10:02:29 +00:00
|
|
|
void df_debug(struct pt_regs *regs, long error_code);
|
2008-10-23 05:26:29 +00:00
|
|
|
#endif /* _ASM_X86_PROCESSOR_H */
|