2019-05-19 12:08:55 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2008-07-29 17:29:19 +00:00
|
|
|
/*
|
|
|
|
* xsave/xrstor support.
|
|
|
|
*
|
|
|
|
* Author: Suresh Siddha <suresh.b.siddha@intel.com>
|
|
|
|
*/
|
2021-10-15 01:16:09 +00:00
|
|
|
#include <linux/bitops.h>
|
2008-07-29 17:29:19 +00:00
|
|
|
#include <linux/compat.h>
|
2014-05-29 18:12:43 +00:00
|
|
|
#include <linux/cpu.h>
|
x86/pkeys: Allocation/free syscalls
This patch adds two new system calls:
int pkey_alloc(unsigned long flags, unsigned long init_access_rights)
int pkey_free(int pkey);
These implement an "allocator" for the protection keys
themselves, which can be thought of as analogous to the allocator
that the kernel has for file descriptors. The kernel tracks
which numbers are in use, and only allows operations on keys that
are valid. A key which was not obtained by pkey_alloc() may not,
for instance, be passed to pkey_mprotect().
These system calls are also very important given the kernel's use
of pkeys to implement execute-only support. These help ensure
that userspace can never assume that it has control of a key
unless it first asks the kernel. The kernel does not promise to
preserve PKRU (right register) contents except for allocated
pkeys.
The 'init_access_rights' argument to pkey_alloc() specifies the
rights that will be established for the returned pkey. For
instance:
pkey = pkey_alloc(flags, PKEY_DENY_WRITE);
will allocate 'pkey', but also sets the bits in PKRU[1] such that
writing to 'pkey' is already denied.
The kernel does not prevent pkey_free() from successfully freeing
in-use pkeys (those still assigned to a memory range by
pkey_mprotect()). It would be expensive to implement the checks
for this, so we instead say, "Just don't do it" since sane
software will never do it anyway.
Any piece of userspace calling pkey_alloc() needs to be prepared
for it to fail. Why? pkey_alloc() returns the same error code
(ENOSPC) when there are no pkeys and when pkeys are unsupported.
They can be unsupported for a whole host of reasons, so apps must
be prepared for this. Also, libraries or LD_PRELOADs might steal
keys before an application gets access to them.
This allocation mechanism could be implemented in userspace.
Even if we did it in userspace, we would still need additional
user/kernel interfaces to tell userspace which keys are being
used by the kernel internally (such as for execute-only
mappings). Having the kernel provide this facility completely
removes the need for these additional interfaces, or having an
implementation of this in userspace at all.
Note that we have to make changes to all of the architectures
that do not use mman-common.h because we use the new
PKEY_DENY_ACCESS/WRITE macros in arch-independent code.
1. PKRU is the Protection Key Rights User register. It is a
usermode-accessible register that controls whether writes
and/or access to each individual pkey is allowed or denied.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Cc: linux-arch@vger.kernel.org
Cc: Dave Hansen <dave@sr71.net>
Cc: arnd@arndb.de
Cc: linux-api@vger.kernel.org
Cc: linux-mm@kvack.org
Cc: luto@kernel.org
Cc: akpm@linux-foundation.org
Cc: torvalds@linux-foundation.org
Link: http://lkml.kernel.org/r/20160729163015.444FE75F@viggo.jf.intel.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2016-07-29 16:30:15 +00:00
|
|
|
#include <linux/mman.h>
|
2016-02-12 21:02:36 +00:00
|
|
|
#include <linux/pkeys.h>
|
2019-06-06 01:22:35 +00:00
|
|
|
#include <linux/seq_file.h>
|
|
|
|
#include <linux/proc_fs.h>
|
2015-04-30 06:53:18 +00:00
|
|
|
|
2015-04-24 00:46:00 +00:00
|
|
|
#include <asm/fpu/api.h>
|
2015-04-30 06:53:18 +00:00
|
|
|
#include <asm/fpu/regset.h>
|
2021-10-15 01:16:31 +00:00
|
|
|
#include <asm/fpu/signal.h>
|
|
|
|
#include <asm/fpu/xcr.h>
|
2015-04-30 10:45:38 +00:00
|
|
|
|
2014-10-24 22:58:07 +00:00
|
|
|
#include <asm/tlbflush.h>
|
2021-10-15 01:16:07 +00:00
|
|
|
|
2021-10-15 01:16:18 +00:00
|
|
|
#include "internal.h"
|
2021-10-15 01:16:26 +00:00
|
|
|
#include "legacy.h"
|
2021-10-15 01:16:07 +00:00
|
|
|
#include "xstate.h"
|
2008-07-29 17:29:19 +00:00
|
|
|
|
2021-10-15 01:16:09 +00:00
|
|
|
#define for_each_extended_xfeature(bit, mask) \
|
|
|
|
(bit) = FIRST_EXTENDED_XFEATURE; \
|
|
|
|
for_each_set_bit_from(bit, (unsigned long *)&(mask), 8 * sizeof(mask))
|
|
|
|
|
2016-02-12 21:01:58 +00:00
|
|
|
/*
|
|
|
|
* Although we spell it out in here, the Processor Trace
|
|
|
|
* xfeature is completely unused. We use other mechanisms
|
|
|
|
* to save/restore PT state in Linux.
|
|
|
|
*/
|
2015-04-28 06:51:17 +00:00
|
|
|
static const char *xfeature_names[] =
|
|
|
|
{
|
|
|
|
"x87 floating point registers" ,
|
|
|
|
"SSE registers" ,
|
|
|
|
"AVX registers" ,
|
|
|
|
"MPX bounds registers" ,
|
|
|
|
"MPX CSR" ,
|
|
|
|
"AVX-512 opmask" ,
|
|
|
|
"AVX-512 Hi256" ,
|
|
|
|
"AVX-512 ZMM_Hi256" ,
|
2016-02-12 21:01:58 +00:00
|
|
|
"Processor Trace (unused)" ,
|
2016-02-12 21:02:04 +00:00
|
|
|
"Protection Keys User registers",
|
2020-09-15 16:30:09 +00:00
|
|
|
"PASID state",
|
2015-04-28 06:51:17 +00:00
|
|
|
"unknown xstate feature" ,
|
|
|
|
};
|
|
|
|
|
2017-10-13 21:56:44 +00:00
|
|
|
static short xsave_cpuid_features[] __initdata = {
|
|
|
|
X86_FEATURE_FPU,
|
|
|
|
X86_FEATURE_XMM,
|
|
|
|
X86_FEATURE_AVX,
|
|
|
|
X86_FEATURE_MPX,
|
|
|
|
X86_FEATURE_MPX,
|
|
|
|
X86_FEATURE_AVX512F,
|
|
|
|
X86_FEATURE_AVX512F,
|
|
|
|
X86_FEATURE_AVX512F,
|
|
|
|
X86_FEATURE_INTEL_PT,
|
|
|
|
X86_FEATURE_PKU,
|
2020-09-15 16:30:09 +00:00
|
|
|
X86_FEATURE_ENQCMD,
|
2017-10-13 21:56:44 +00:00
|
|
|
};
|
|
|
|
|
2008-07-29 17:29:19 +00:00
|
|
|
/*
|
2020-05-12 14:54:37 +00:00
|
|
|
* This represents the full set of bits that should ever be set in a kernel
|
|
|
|
* XSAVE buffer, both supervisor and user xstates.
|
2008-07-29 17:29:19 +00:00
|
|
|
*/
|
2021-06-23 12:01:31 +00:00
|
|
|
u64 xfeatures_mask_all __ro_after_init;
|
2008-07-29 17:29:19 +00:00
|
|
|
|
2021-06-23 12:01:30 +00:00
|
|
|
static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init =
|
|
|
|
{ [ 0 ... XFEATURE_MAX - 1] = -1};
|
|
|
|
static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init =
|
|
|
|
{ [ 0 ... XFEATURE_MAX - 1] = -1};
|
|
|
|
static unsigned int xstate_comp_offsets[XFEATURE_MAX] __ro_after_init =
|
|
|
|
{ [ 0 ... XFEATURE_MAX - 1] = -1};
|
|
|
|
static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] __ro_after_init =
|
|
|
|
{ [ 0 ... XFEATURE_MAX - 1] = -1};
|
2015-04-24 07:23:59 +00:00
|
|
|
|
2016-05-20 17:47:05 +00:00
|
|
|
/*
|
|
|
|
* The XSAVE area of kernel can be in standard or compacted format;
|
|
|
|
* it is always in standard format for user mode. This is the user
|
|
|
|
* mode standard format size used for signal and ptrace frames.
|
|
|
|
*/
|
2021-06-23 12:01:30 +00:00
|
|
|
unsigned int fpu_user_xstate_size __ro_after_init;
|
2016-05-20 17:47:05 +00:00
|
|
|
|
2015-04-28 06:51:17 +00:00
|
|
|
/*
|
|
|
|
* Return whether the system supports a given xfeature.
|
|
|
|
*
|
|
|
|
* Also return the name of the (most advanced) feature that the caller requested:
|
|
|
|
*/
|
|
|
|
int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
|
|
|
|
{
|
2020-05-12 14:54:37 +00:00
|
|
|
u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask_all;
|
2015-04-28 06:51:17 +00:00
|
|
|
|
|
|
|
if (unlikely(feature_name)) {
|
|
|
|
long xfeature_idx, max_idx;
|
|
|
|
u64 xfeatures_print;
|
|
|
|
/*
|
|
|
|
* So we use FLS here to be able to print the most advanced
|
|
|
|
* feature that was requested but is missing. So if a driver
|
2015-09-02 23:31:26 +00:00
|
|
|
* asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the
|
2015-04-28 06:51:17 +00:00
|
|
|
* missing AVX feature - this is the most informative message
|
|
|
|
* to users:
|
|
|
|
*/
|
|
|
|
if (xfeatures_missing)
|
|
|
|
xfeatures_print = xfeatures_missing;
|
|
|
|
else
|
|
|
|
xfeatures_print = xfeatures_needed;
|
|
|
|
|
|
|
|
xfeature_idx = fls64(xfeatures_print)-1;
|
|
|
|
max_idx = ARRAY_SIZE(xfeature_names)-1;
|
|
|
|
xfeature_idx = min(xfeature_idx, max_idx);
|
|
|
|
|
|
|
|
*feature_name = xfeature_names[xfeature_idx];
|
|
|
|
}
|
|
|
|
|
|
|
|
if (xfeatures_missing)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
|
|
|
|
|
2019-12-12 21:08:54 +00:00
|
|
|
static bool xfeature_is_supervisor(int xfeature_nr)
|
2016-06-17 20:07:16 +00:00
|
|
|
{
|
|
|
|
/*
|
2019-12-12 21:08:53 +00:00
|
|
|
* Extended State Enumeration Sub-leaves (EAX = 0DH, ECX = n, n > 1)
|
|
|
|
* returns ECX[0] set to (1) for a supervisor state, and cleared (0)
|
|
|
|
* for a user state.
|
2016-06-17 20:07:16 +00:00
|
|
|
*/
|
|
|
|
u32 eax, ebx, ecx, edx;
|
|
|
|
|
|
|
|
cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
|
2019-12-12 21:08:54 +00:00
|
|
|
return ecx & 1;
|
2016-06-17 20:07:16 +00:00
|
|
|
}
|
|
|
|
|
2008-07-29 17:29:19 +00:00
|
|
|
/*
|
2015-04-25 04:26:36 +00:00
|
|
|
* Enable the extended processor state save/restore feature.
|
|
|
|
* Called once per CPU onlining.
|
2008-07-29 17:29:19 +00:00
|
|
|
*/
|
2015-04-25 04:26:36 +00:00
|
|
|
void fpu__init_cpu_xstate(void)
|
2008-07-29 17:29:19 +00:00
|
|
|
{
|
2020-05-12 14:54:37 +00:00
|
|
|
if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask_all)
|
2015-04-25 04:26:36 +00:00
|
|
|
return;
|
|
|
|
|
2014-10-24 22:58:07 +00:00
|
|
|
cr4_set_bits(X86_CR4_OSXSAVE);
|
2020-05-12 14:54:37 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features
|
|
|
|
* managed by XSAVE{C, OPT, S} and XRSTOR{S}. Only XSAVE user
|
|
|
|
* states can be set here.
|
|
|
|
*/
|
2021-06-23 12:02:16 +00:00
|
|
|
xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_uabi());
|
2020-05-12 14:54:38 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* MSR_IA32_XSS sets supervisor states managed by XSAVES.
|
|
|
|
*/
|
x86/fpu/xstate: Support dynamic supervisor feature for LBR
Last Branch Records (LBR) registers are used to log taken branches and
other control flows. In perf with call stack mode, LBR information is
used to reconstruct a call stack. To get the complete call stack, perf
has to save/restore all LBR registers during a context switch. Due to
the large number of the LBR registers, e.g., the current platform has
96 LBR registers, this process causes a high CPU overhead. To reduce
the CPU overhead during a context switch, an LBR state component that
contains all the LBR related registers is introduced in hardware. All
LBR registers can be saved/restored together using one XSAVES/XRSTORS
instruction.
However, the kernel should not save/restore the LBR state component at
each context switch, like other state components, because of the
following unique features of LBR:
- The LBR state component only contains valuable information when LBR
is enabled in the perf subsystem, but for most of the time, LBR is
disabled.
- The size of the LBR state component is huge. For the current
platform, it's 808 bytes.
If the kernel saves/restores the LBR state at each context switch, for
most of the time, it is just a waste of space and cycles.
To efficiently support the LBR state component, it is desired to have:
- only context-switch the LBR when the LBR feature is enabled in perf.
- only allocate an LBR-specific XSAVE buffer on demand.
(Besides the LBR state, a legacy region and an XSAVE header have to be
included in the buffer as well. There is a total of (808+576) byte
overhead for the LBR-specific XSAVE buffer. The overhead only happens
when the perf is actively using LBRs. There is still a space-saving,
on average, when it replaces the constant 808 bytes of overhead for
every task, all the time on the systems that support architectural
LBR.)
- be able to use XSAVES/XRSTORS for accessing LBR at run time.
However, the IA32_XSS should not be adjusted at run time.
(The XCR0 | IA32_XSS are used to determine the requested-feature
bitmap (RFBM) of XSAVES.)
A solution, called dynamic supervisor feature, is introduced to address
this issue, which
- does not allocate a buffer in each task->fpu;
- does not save/restore a state component at each context switch;
- sets the bit corresponding to the dynamic supervisor feature in
IA32_XSS at boot time, and avoids setting it at run time.
- dynamically allocates a specific buffer for a state component
on demand, e.g. only allocates LBR-specific XSAVE buffer when LBR is
enabled in perf. (Note: The buffer has to include the LBR state
component, a legacy region and a XSAVE header space.)
(Implemented in a later patch)
- saves/restores a state component on demand, e.g. manually invokes
the XSAVES/XRSTORS instruction to save/restore the LBR state
to/from the buffer when perf is active and a call stack is required.
(Implemented in a later patch)
A new mask XFEATURE_MASK_DYNAMIC and a helper xfeatures_mask_dynamic()
are introduced to indicate the dynamic supervisor feature. For the
systems which support the Architecture LBR, LBR is the only dynamic
supervisor feature for now. For the previous systems, there is no
dynamic supervisor feature available.
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lkml.kernel.org/r/1593780569-62993-21-git-send-email-kan.liang@linux.intel.com
2020-07-03 12:49:26 +00:00
|
|
|
if (boot_cpu_has(X86_FEATURE_XSAVES)) {
|
|
|
|
wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() |
|
2021-06-23 12:02:03 +00:00
|
|
|
xfeatures_mask_independent());
|
x86/fpu/xstate: Support dynamic supervisor feature for LBR
Last Branch Records (LBR) registers are used to log taken branches and
other control flows. In perf with call stack mode, LBR information is
used to reconstruct a call stack. To get the complete call stack, perf
has to save/restore all LBR registers during a context switch. Due to
the large number of the LBR registers, e.g., the current platform has
96 LBR registers, this process causes a high CPU overhead. To reduce
the CPU overhead during a context switch, an LBR state component that
contains all the LBR related registers is introduced in hardware. All
LBR registers can be saved/restored together using one XSAVES/XRSTORS
instruction.
However, the kernel should not save/restore the LBR state component at
each context switch, like other state components, because of the
following unique features of LBR:
- The LBR state component only contains valuable information when LBR
is enabled in the perf subsystem, but for most of the time, LBR is
disabled.
- The size of the LBR state component is huge. For the current
platform, it's 808 bytes.
If the kernel saves/restores the LBR state at each context switch, for
most of the time, it is just a waste of space and cycles.
To efficiently support the LBR state component, it is desired to have:
- only context-switch the LBR when the LBR feature is enabled in perf.
- only allocate an LBR-specific XSAVE buffer on demand.
(Besides the LBR state, a legacy region and an XSAVE header have to be
included in the buffer as well. There is a total of (808+576) byte
overhead for the LBR-specific XSAVE buffer. The overhead only happens
when the perf is actively using LBRs. There is still a space-saving,
on average, when it replaces the constant 808 bytes of overhead for
every task, all the time on the systems that support architectural
LBR.)
- be able to use XSAVES/XRSTORS for accessing LBR at run time.
However, the IA32_XSS should not be adjusted at run time.
(The XCR0 | IA32_XSS are used to determine the requested-feature
bitmap (RFBM) of XSAVES.)
A solution, called dynamic supervisor feature, is introduced to address
this issue, which
- does not allocate a buffer in each task->fpu;
- does not save/restore a state component at each context switch;
- sets the bit corresponding to the dynamic supervisor feature in
IA32_XSS at boot time, and avoids setting it at run time.
- dynamically allocates a specific buffer for a state component
on demand, e.g. only allocates LBR-specific XSAVE buffer when LBR is
enabled in perf. (Note: The buffer has to include the LBR state
component, a legacy region and a XSAVE header space.)
(Implemented in a later patch)
- saves/restores a state component on demand, e.g. manually invokes
the XSAVES/XRSTORS instruction to save/restore the LBR state
to/from the buffer when perf is active and a call stack is required.
(Implemented in a later patch)
A new mask XFEATURE_MASK_DYNAMIC and a helper xfeatures_mask_dynamic()
are introduced to indicate the dynamic supervisor feature. For the
systems which support the Architecture LBR, LBR is the only dynamic
supervisor feature for now. For the previous systems, there is no
dynamic supervisor feature available.
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lkml.kernel.org/r/1593780569-62993-21-git-send-email-kan.liang@linux.intel.com
2020-07-03 12:49:26 +00:00
|
|
|
}
|
2008-07-29 17:29:19 +00:00
|
|
|
}
|
|
|
|
|
2020-05-12 14:54:37 +00:00
|
|
|
static bool xfeature_enabled(enum xfeature xfeature)
|
2015-09-02 23:31:30 +00:00
|
|
|
{
|
2020-05-12 14:54:37 +00:00
|
|
|
return xfeatures_mask_all & BIT_ULL(xfeature);
|
2015-09-02 23:31:30 +00:00
|
|
|
}
|
|
|
|
|
2010-07-19 23:05:48 +00:00
|
|
|
/*
|
x86/fpu/xstate: Don't assume the first zero xfeatures zero bit means the end
The current xstate code in setup_xstate_features() assumes that
the first zero bit means the end of xfeatures - but that is not
so, the SDM clearly states that an arbitrary set of xfeatures
might be enabled - and it is also clear from the description
of the compaction feature that holes are possible:
"13-6 Vol. 1MANAGING STATE USING THE XSAVE FEATURE SET
[...]
Compacted format. Each state component i (i ≥ 2) is located at a byte
offset from the base address of the XSAVE area based on the XCOMP_BV
field in the XSAVE header:
— If XCOMP_BV[i] = 0, state component i is not in the XSAVE area.
— If XCOMP_BV[i] = 1, the following items apply:
• If XCOMP_BV[j] = 0 for every j, 2 ≤ j < i, state component i is
located at a byte offset 576 from the base address of the XSAVE
area. (This item applies if i is the first bit set in bits 62:2 of
the XCOMP_BV; it implies that state component i is located at the
beginning of the extended region.)
• Otherwise, let j, 2 ≤ j < i, be the greatest value such that
XCOMP_BV[j] = 1. Then state component i is located at a byte offset
X from the location of state component j, where X is the number of
bytes required for state component j as enumerated in
CPUID.(EAX=0DH,ECX=j):EAX. (This item implies that state component i
immediately follows the preceding state component whose bit is set
in XCOMP_BV.)"
So don't assume that the first zero xfeatures bit means the end of
all xfeatures - iterate through all of them.
I'm not aware of hardware that triggers this currently.
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-05-04 05:37:47 +00:00
|
|
|
* Record the offsets and sizes of various xstates contained
|
|
|
|
* in the XSAVE state memory layout.
|
2010-07-19 23:05:48 +00:00
|
|
|
*/
|
2010-07-21 17:03:56 +00:00
|
|
|
static void __init setup_xstate_features(void)
|
2010-07-19 23:05:48 +00:00
|
|
|
{
|
2015-09-02 23:31:28 +00:00
|
|
|
u32 eax, ebx, ecx, edx, i;
|
2021-03-18 14:28:01 +00:00
|
|
|
/* start at the beginning of the "extended state" */
|
2015-09-02 23:31:30 +00:00
|
|
|
unsigned int last_good_offset = offsetof(struct xregs_state,
|
|
|
|
extended_state_area);
|
2016-06-17 20:07:19 +00:00
|
|
|
/*
|
|
|
|
* The FP xstates and SSE xstates are legacy states. They are always
|
|
|
|
* in the fixed offsets in the xsave area in either compacted form
|
|
|
|
* or standard form.
|
|
|
|
*/
|
2019-11-01 13:01:53 +00:00
|
|
|
xstate_offsets[XFEATURE_FP] = 0;
|
|
|
|
xstate_sizes[XFEATURE_FP] = offsetof(struct fxregs_state,
|
|
|
|
xmm_space);
|
|
|
|
|
|
|
|
xstate_offsets[XFEATURE_SSE] = xstate_sizes[XFEATURE_FP];
|
2019-12-09 18:31:43 +00:00
|
|
|
xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state,
|
2019-11-01 13:01:53 +00:00
|
|
|
xmm_space);
|
2010-07-19 23:05:48 +00:00
|
|
|
|
2021-10-15 01:16:09 +00:00
|
|
|
for_each_extended_xfeature(i, xfeatures_mask_all) {
|
2015-09-02 23:31:30 +00:00
|
|
|
cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
|
2016-06-17 20:07:16 +00:00
|
|
|
|
2020-01-09 21:14:50 +00:00
|
|
|
xstate_sizes[i] = eax;
|
|
|
|
|
2016-06-17 20:07:16 +00:00
|
|
|
/*
|
2020-01-09 21:14:50 +00:00
|
|
|
* If an xfeature is supervisor state, the offset in EBX is
|
|
|
|
* invalid, leave it to -1.
|
2016-06-17 20:07:16 +00:00
|
|
|
*/
|
2020-01-09 21:14:50 +00:00
|
|
|
if (xfeature_is_supervisor(i))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
xstate_offsets[i] = ebx;
|
2016-06-17 20:07:16 +00:00
|
|
|
|
2015-09-02 23:31:30 +00:00
|
|
|
/*
|
2020-01-09 21:14:50 +00:00
|
|
|
* In our xstate size checks, we assume that the highest-numbered
|
|
|
|
* xstate feature has the highest offset in the buffer. Ensure
|
|
|
|
* it does.
|
2015-09-02 23:31:30 +00:00
|
|
|
*/
|
|
|
|
WARN_ONCE(last_good_offset > xstate_offsets[i],
|
2020-01-09 21:14:50 +00:00
|
|
|
"x86/fpu: misordered xstate at %d\n", last_good_offset);
|
|
|
|
|
2015-09-02 23:31:30 +00:00
|
|
|
last_good_offset = xstate_offsets[i];
|
x86/fpu/xstate: Don't assume the first zero xfeatures zero bit means the end
The current xstate code in setup_xstate_features() assumes that
the first zero bit means the end of xfeatures - but that is not
so, the SDM clearly states that an arbitrary set of xfeatures
might be enabled - and it is also clear from the description
of the compaction feature that holes are possible:
"13-6 Vol. 1MANAGING STATE USING THE XSAVE FEATURE SET
[...]
Compacted format. Each state component i (i ≥ 2) is located at a byte
offset from the base address of the XSAVE area based on the XCOMP_BV
field in the XSAVE header:
— If XCOMP_BV[i] = 0, state component i is not in the XSAVE area.
— If XCOMP_BV[i] = 1, the following items apply:
• If XCOMP_BV[j] = 0 for every j, 2 ≤ j < i, state component i is
located at a byte offset 576 from the base address of the XSAVE
area. (This item applies if i is the first bit set in bits 62:2 of
the XCOMP_BV; it implies that state component i is located at the
beginning of the extended region.)
• Otherwise, let j, 2 ≤ j < i, be the greatest value such that
XCOMP_BV[j] = 1. Then state component i is located at a byte offset
X from the location of state component j, where X is the number of
bytes required for state component j as enumerated in
CPUID.(EAX=0DH,ECX=j):EAX. (This item implies that state component i
immediately follows the preceding state component whose bit is set
in XCOMP_BV.)"
So don't assume that the first zero xfeatures bit means the end of
all xfeatures - iterate through all of them.
I'm not aware of hardware that triggers this currently.
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-05-04 05:37:47 +00:00
|
|
|
}
|
2010-07-19 23:05:48 +00:00
|
|
|
}
|
|
|
|
|
2015-05-04 07:52:42 +00:00
|
|
|
static void __init print_xstate_feature(u64 xstate_mask)
|
2015-04-24 06:48:01 +00:00
|
|
|
{
|
2015-04-28 07:17:26 +00:00
|
|
|
const char *feature_name;
|
2015-04-24 06:48:01 +00:00
|
|
|
|
2015-04-28 07:17:26 +00:00
|
|
|
if (cpu_has_xfeatures(xstate_mask, &feature_name))
|
2016-02-12 21:02:04 +00:00
|
|
|
pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name);
|
2015-04-24 06:48:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Print out all the supported xstate features:
|
|
|
|
*/
|
2015-05-04 07:52:42 +00:00
|
|
|
static void __init print_xstate_features(void)
|
2015-04-24 06:48:01 +00:00
|
|
|
{
|
2015-09-02 23:31:26 +00:00
|
|
|
print_xstate_feature(XFEATURE_MASK_FP);
|
|
|
|
print_xstate_feature(XFEATURE_MASK_SSE);
|
|
|
|
print_xstate_feature(XFEATURE_MASK_YMM);
|
|
|
|
print_xstate_feature(XFEATURE_MASK_BNDREGS);
|
|
|
|
print_xstate_feature(XFEATURE_MASK_BNDCSR);
|
|
|
|
print_xstate_feature(XFEATURE_MASK_OPMASK);
|
|
|
|
print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
|
|
|
|
print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
|
2016-02-12 21:02:04 +00:00
|
|
|
print_xstate_feature(XFEATURE_MASK_PKRU);
|
2020-09-15 16:30:09 +00:00
|
|
|
print_xstate_feature(XFEATURE_MASK_PASID);
|
2015-04-24 06:48:01 +00:00
|
|
|
}
|
|
|
|
|
2016-06-17 20:07:15 +00:00
|
|
|
/*
|
|
|
|
* This check is important because it is easy to get XSTATE_*
|
|
|
|
* confused with XSTATE_BIT_*.
|
|
|
|
*/
|
|
|
|
#define CHECK_XFEATURE(nr) do { \
|
|
|
|
WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \
|
|
|
|
WARN_ON(nr >= XFEATURE_MAX); \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We could cache this like xstate_size[], but we only use
|
|
|
|
* it here, so it would be a waste of space.
|
|
|
|
*/
|
|
|
|
static int xfeature_is_aligned(int xfeature_nr)
|
|
|
|
{
|
|
|
|
u32 eax, ebx, ecx, edx;
|
|
|
|
|
|
|
|
CHECK_XFEATURE(xfeature_nr);
|
2020-01-09 21:14:52 +00:00
|
|
|
|
|
|
|
if (!xfeature_enabled(xfeature_nr)) {
|
|
|
|
WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n",
|
|
|
|
xfeature_nr);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-06-17 20:07:15 +00:00
|
|
|
cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
|
|
|
|
/*
|
|
|
|
* The value returned by ECX[1] indicates the alignment
|
|
|
|
* of state component 'i' when the compacted format
|
|
|
|
* of the extended region of an XSAVE area is used:
|
|
|
|
*/
|
|
|
|
return !!(ecx & 2);
|
|
|
|
}
|
|
|
|
|
2014-05-29 18:12:44 +00:00
|
|
|
/*
|
|
|
|
* This function sets up offsets and sizes of all extended states in
|
|
|
|
* xsave area. This supports both standard format and compacted format
|
2020-01-09 21:14:51 +00:00
|
|
|
* of the xsave area.
|
2014-05-29 18:12:44 +00:00
|
|
|
*/
|
2020-01-09 21:14:51 +00:00
|
|
|
static void __init setup_xstate_comp_offsets(void)
|
2014-05-29 18:12:44 +00:00
|
|
|
{
|
2020-01-09 21:14:51 +00:00
|
|
|
unsigned int next_offset;
|
2014-05-29 18:12:44 +00:00
|
|
|
int i;
|
|
|
|
|
2014-05-30 21:59:24 +00:00
|
|
|
/*
|
|
|
|
* The FP xstates and SSE xstates are legacy states. They are always
|
|
|
|
* in the fixed offsets in the xsave area in either compacted form
|
|
|
|
* or standard form.
|
|
|
|
*/
|
2019-11-01 13:01:53 +00:00
|
|
|
xstate_comp_offsets[XFEATURE_FP] = 0;
|
|
|
|
xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state,
|
|
|
|
xmm_space);
|
2014-05-29 18:12:44 +00:00
|
|
|
|
2021-10-15 01:16:09 +00:00
|
|
|
if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) {
|
|
|
|
for_each_extended_xfeature(i, xfeatures_mask_all)
|
|
|
|
xstate_comp_offsets[i] = xstate_offsets[i];
|
2014-05-29 18:12:44 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-01-09 21:14:51 +00:00
|
|
|
next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;
|
2014-05-29 18:12:44 +00:00
|
|
|
|
2021-10-15 01:16:09 +00:00
|
|
|
for_each_extended_xfeature(i, xfeatures_mask_all) {
|
2020-01-09 21:14:51 +00:00
|
|
|
if (xfeature_is_aligned(i))
|
|
|
|
next_offset = ALIGN(next_offset, 64);
|
2014-05-29 18:12:44 +00:00
|
|
|
|
2020-01-09 21:14:51 +00:00
|
|
|
xstate_comp_offsets[i] = next_offset;
|
|
|
|
next_offset += xstate_sizes[i];
|
2014-05-29 18:12:44 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-05-12 14:54:42 +00:00
|
|
|
/*
|
|
|
|
* Setup offsets of a supervisor-state-only XSAVES buffer:
|
|
|
|
*
|
|
|
|
* The offsets stored in xstate_comp_offsets[] only work for one specific
|
|
|
|
* value of the Requested Feature BitMap (RFBM). In cases where a different
|
|
|
|
* RFBM value is used, a different set of offsets is required. This set of
|
|
|
|
* offsets is for when RFBM=xfeatures_mask_supervisor().
|
|
|
|
*/
|
|
|
|
static void __init setup_supervisor_only_offsets(void)
|
|
|
|
{
|
|
|
|
unsigned int next_offset;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;
|
|
|
|
|
2021-10-15 01:16:09 +00:00
|
|
|
for_each_extended_xfeature(i, xfeatures_mask_all) {
|
|
|
|
if (!xfeature_is_supervisor(i))
|
2020-05-12 14:54:42 +00:00
|
|
|
continue;
|
|
|
|
|
|
|
|
if (xfeature_is_aligned(i))
|
|
|
|
next_offset = ALIGN(next_offset, 64);
|
|
|
|
|
|
|
|
xstate_supervisor_only_offsets[i] = next_offset;
|
|
|
|
next_offset += xstate_sizes[i];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-17 20:07:18 +00:00
|
|
|
/*
|
|
|
|
* Print out xstate component offsets and sizes
|
|
|
|
*/
|
|
|
|
static void __init print_xstate_offset_size(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2021-10-15 01:16:09 +00:00
|
|
|
for_each_extended_xfeature(i, xfeatures_mask_all) {
|
2016-06-17 20:07:18 +00:00
|
|
|
pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n",
|
|
|
|
i, xstate_comp_offsets[i], i, xstate_sizes[i]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-10-15 01:16:23 +00:00
|
|
|
/*
|
|
|
|
* This function is called only during boot time when x86 caps are not set
|
|
|
|
* up and alternative can not be used yet.
|
|
|
|
*/
|
|
|
|
static __init void os_xrstor_booting(struct xregs_state *xstate)
|
|
|
|
{
|
|
|
|
u64 mask = xfeatures_mask_fpstate();
|
|
|
|
u32 lmask = mask;
|
|
|
|
u32 hmask = mask >> 32;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (cpu_feature_enabled(X86_FEATURE_XSAVES))
|
|
|
|
XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
|
|
|
|
else
|
|
|
|
XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We should never fault when copying from a kernel buffer, and the FPU
|
|
|
|
* state we set at boot time should be valid.
|
|
|
|
*/
|
|
|
|
WARN_ON_FPU(err);
|
|
|
|
}
|
|
|
|
|
2021-06-18 14:18:25 +00:00
|
|
|
/*
|
|
|
|
* All supported features have either init state all zeros or are
|
|
|
|
* handled in setup_init_fpu() individually. This is an explicit
|
|
|
|
* feature list and does not use XFEATURE_MASK*SUPPORTED to catch
|
|
|
|
* newly added supported features at build time and make people
|
|
|
|
* actually look at the init state for the new feature.
|
|
|
|
*/
|
|
|
|
#define XFEATURES_INIT_FPSTATE_HANDLED \
|
|
|
|
(XFEATURE_MASK_FP | \
|
|
|
|
XFEATURE_MASK_SSE | \
|
|
|
|
XFEATURE_MASK_YMM | \
|
|
|
|
XFEATURE_MASK_OPMASK | \
|
|
|
|
XFEATURE_MASK_ZMM_Hi256 | \
|
|
|
|
XFEATURE_MASK_Hi16_ZMM | \
|
|
|
|
XFEATURE_MASK_PKRU | \
|
|
|
|
XFEATURE_MASK_BNDREGS | \
|
|
|
|
XFEATURE_MASK_BNDCSR | \
|
|
|
|
XFEATURE_MASK_PASID)
|
|
|
|
|
2008-07-29 17:29:19 +00:00
|
|
|
/*
|
|
|
|
* setup the xstate image representing the init state
|
|
|
|
*/
|
2015-05-04 07:52:42 +00:00
|
|
|
static void __init setup_init_fpu_buf(void)
|
2008-07-29 17:29:19 +00:00
|
|
|
{
|
2021-06-18 14:18:25 +00:00
|
|
|
BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED |
|
|
|
|
XFEATURE_MASK_SUPERVISOR_SUPPORTED) !=
|
|
|
|
XFEATURES_INIT_FPSTATE_HANDLED);
|
|
|
|
|
2016-04-04 20:25:02 +00:00
|
|
|
if (!boot_cpu_has(X86_FEATURE_XSAVE))
|
2012-09-06 21:58:52 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
setup_xstate_features();
|
2015-04-24 06:48:01 +00:00
|
|
|
print_xstate_features();
|
2010-07-19 23:05:48 +00:00
|
|
|
|
2021-10-13 14:55:28 +00:00
|
|
|
xstate_init_xcomp_bv(&init_fpstate.regs.xsave, xfeatures_mask_all);
|
2014-05-29 18:12:42 +00:00
|
|
|
|
2010-07-19 23:05:49 +00:00
|
|
|
/*
|
2016-05-20 17:47:07 +00:00
|
|
|
* Init all the features state with header.xfeatures being 0x0
|
2010-07-19 23:05:49 +00:00
|
|
|
*/
|
2021-10-13 14:55:28 +00:00
|
|
|
os_xrstor_booting(&init_fpstate.regs.xsave);
|
2015-04-22 13:08:34 +00:00
|
|
|
|
2010-07-19 23:05:49 +00:00
|
|
|
/*
|
2021-06-18 14:18:25 +00:00
|
|
|
* All components are now in init state. Read the state back so
|
|
|
|
* that init_fpstate contains all non-zero init state. This only
|
|
|
|
* works with XSAVE, but not with XSAVEOPT and XSAVES because
|
|
|
|
* those use the init optimization which skips writing data for
|
|
|
|
* components in init state.
|
|
|
|
*
|
|
|
|
* XSAVE could be used, but that would require to reshuffle the
|
|
|
|
* data when XSAVES is available because XSAVES uses xstate
|
|
|
|
* compaction. But doing so is a pointless exercise because most
|
|
|
|
* components have an all zeros init state except for the legacy
|
|
|
|
* ones (FP and SSE). Those can be saved with FXSAVE into the
|
|
|
|
* legacy area. Adding new features requires to ensure that init
|
|
|
|
* state is all zeroes or if not to add the necessary handling
|
|
|
|
* here.
|
2010-07-19 23:05:49 +00:00
|
|
|
*/
|
2021-10-13 14:55:28 +00:00
|
|
|
fxsave(&init_fpstate.regs.fxsave);
|
2008-07-29 17:29:19 +00:00
|
|
|
}
|
|
|
|
|
2015-09-02 23:31:30 +00:00
|
|
|
static int xfeature_uncompacted_offset(int xfeature_nr)
|
|
|
|
{
|
|
|
|
u32 eax, ebx, ecx, edx;
|
|
|
|
|
2016-06-17 20:07:16 +00:00
|
|
|
/*
|
|
|
|
* Only XSAVES supports supervisor states and it uses compacted
|
|
|
|
* format. Checking a supervisor state's uncompacted offset is
|
|
|
|
* an error.
|
|
|
|
*/
|
2020-05-12 14:54:36 +00:00
|
|
|
if (XFEATURE_MASK_SUPERVISOR_ALL & BIT_ULL(xfeature_nr)) {
|
2016-06-17 20:07:16 +00:00
|
|
|
WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2015-09-02 23:31:30 +00:00
|
|
|
CHECK_XFEATURE(xfeature_nr);
|
|
|
|
cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
|
|
|
|
return ebx;
|
|
|
|
}
|
|
|
|
|
perf/x86/intel/lbr: Support XSAVES/XRSTORS for LBR context switch
In the LBR call stack mode, LBR information is used to reconstruct a
call stack. To get the complete call stack, perf has to save/restore
all LBR registers during a context switch. Due to a large number of the
LBR registers, this process causes a high CPU overhead. To reduce the
CPU overhead during a context switch, use the XSAVES/XRSTORS
instructions.
Every XSAVE area must follow a canonical format: the legacy region, an
XSAVE header and the extended region. Although the LBR information is
only kept in the extended region, a space for the legacy region and
XSAVE header is still required. Add a new dedicated structure for LBR
XSAVES support.
Before enabling XSAVES support, the size of the LBR state has to be
sanity checked, because:
- the size of the software structure is calculated from the max number
of the LBR depth, which is enumerated by the CPUID leaf for Arch LBR.
The size of the LBR state is enumerated by the CPUID leaf for XSAVE
support of Arch LBR. If the values from the two CPUID leaves are not
consistent, it may trigger a buffer overflow. For example, a hypervisor
may unconsciously set inconsistent values for the two emulated CPUID.
- unlike other state components, the size of an LBR state depends on the
max number of LBRs, which may vary from generation to generation.
Expose the function xfeature_size() for the sanity check.
The LBR XSAVES support will be disabled if the size of the LBR state
enumerated by CPUID doesn't match with the size of the software
structure.
The XSAVE instruction requires 64-byte alignment for state buffers. A
new macro is added to reflect the alignment requirement. A 64-byte
aligned kmem_cache is created for architecture LBR.
Currently, the structure for each state component is maintained in
fpu/types.h. The structure for the new LBR state component should be
maintained in the same place. Move structure lbr_entry to fpu/types.h as
well for broader sharing.
Add dedicated lbr_save/lbr_restore functions for LBR XSAVES support,
which invokes the corresponding xstate helpers to XSAVES/XRSTORS LBR
information at the context switch when the call stack mode is enabled.
Since the XSAVES/XRSTORS instructions will be eventually invoked, the
dedicated functions is named with '_xsaves'/'_xrstors' postfix.
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lkml.kernel.org/r/1593780569-62993-23-git-send-email-kan.liang@linux.intel.com
2020-07-03 12:49:28 +00:00
|
|
|
int xfeature_size(int xfeature_nr)
|
2015-09-02 23:31:30 +00:00
|
|
|
{
|
|
|
|
u32 eax, ebx, ecx, edx;
|
|
|
|
|
|
|
|
CHECK_XFEATURE(xfeature_nr);
|
|
|
|
cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
|
|
|
|
return eax;
|
|
|
|
}
|
|
|
|
|
2017-09-24 10:59:04 +00:00
|
|
|
/* Validate an xstate header supplied by userspace (ptrace or sigreturn) */
|
2021-06-23 12:01:36 +00:00
|
|
|
static int validate_user_xstate_header(const struct xstate_header *hdr)
|
2017-09-24 10:59:04 +00:00
|
|
|
{
|
|
|
|
/* No unknown or supervisor features may be set */
|
2021-06-23 12:02:16 +00:00
|
|
|
if (hdr->xfeatures & ~xfeatures_mask_uabi())
|
2017-09-24 10:59:04 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* Userspace must use the uncompacted format */
|
|
|
|
if (hdr->xcomp_bv)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If 'reserved' is shrunken to add a new field, make sure to validate
|
|
|
|
* that new field here!
|
|
|
|
*/
|
|
|
|
BUILD_BUG_ON(sizeof(hdr->reserved) != 48);
|
|
|
|
|
|
|
|
/* No reserved bits may be set */
|
|
|
|
if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved)))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-10-15 01:16:10 +00:00
|
|
|
static void __init __xstate_dump_leaves(void)
|
2015-09-02 23:31:30 +00:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
u32 eax, ebx, ecx, edx;
|
|
|
|
static int should_dump = 1;
|
|
|
|
|
|
|
|
if (!should_dump)
|
|
|
|
return;
|
|
|
|
should_dump = 0;
|
|
|
|
/*
|
|
|
|
* Dump out a few leaves past the ones that we support
|
|
|
|
* just in case there are some goodies up there
|
|
|
|
*/
|
|
|
|
for (i = 0; i < XFEATURE_MAX + 10; i++) {
|
|
|
|
cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
|
|
|
|
pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n",
|
|
|
|
XSTATE_CPUID, i, eax, ebx, ecx, edx);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#define XSTATE_WARN_ON(x) do { \
|
|
|
|
if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) { \
|
|
|
|
__xstate_dump_leaves(); \
|
|
|
|
} \
|
|
|
|
} while (0)
|
|
|
|
|
2015-09-02 23:31:31 +00:00
|
|
|
#define XCHECK_SZ(sz, nr, nr_macro, __struct) do { \
|
|
|
|
if ((nr == nr_macro) && \
|
|
|
|
WARN_ONCE(sz != sizeof(__struct), \
|
|
|
|
"%s: struct is %zu bytes, cpu state %d bytes\n", \
|
|
|
|
__stringify(nr_macro), sizeof(__struct), sz)) { \
|
|
|
|
__xstate_dump_leaves(); \
|
|
|
|
} \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We have a C struct for each 'xstate'. We need to ensure
|
|
|
|
* that our software representation matches what the CPU
|
|
|
|
* tells us about the state's size.
|
|
|
|
*/
|
2021-10-15 01:16:10 +00:00
|
|
|
static void __init check_xstate_against_struct(int nr)
|
2015-09-02 23:31:31 +00:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Ask the CPU for the size of the state.
|
|
|
|
*/
|
|
|
|
int sz = xfeature_size(nr);
|
|
|
|
/*
|
|
|
|
* Match each CPU state with the corresponding software
|
|
|
|
* structure.
|
|
|
|
*/
|
|
|
|
XCHECK_SZ(sz, nr, XFEATURE_YMM, struct ymmh_struct);
|
|
|
|
XCHECK_SZ(sz, nr, XFEATURE_BNDREGS, struct mpx_bndreg_state);
|
|
|
|
XCHECK_SZ(sz, nr, XFEATURE_BNDCSR, struct mpx_bndcsr_state);
|
|
|
|
XCHECK_SZ(sz, nr, XFEATURE_OPMASK, struct avx_512_opmask_state);
|
|
|
|
XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state);
|
|
|
|
XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state);
|
2016-02-12 21:02:04 +00:00
|
|
|
XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state);
|
2020-09-15 16:30:09 +00:00
|
|
|
XCHECK_SZ(sz, nr, XFEATURE_PASID, struct ia32_pasid_state);
|
2015-09-02 23:31:31 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Make *SURE* to add any feature numbers in below if
|
|
|
|
* there are "holes" in the xsave state component
|
|
|
|
* numbers.
|
|
|
|
*/
|
|
|
|
if ((nr < XFEATURE_YMM) ||
|
2016-02-12 21:01:58 +00:00
|
|
|
(nr >= XFEATURE_MAX) ||
|
x86/fpu/xstate: Support dynamic supervisor feature for LBR
Last Branch Records (LBR) registers are used to log taken branches and
other control flows. In perf with call stack mode, LBR information is
used to reconstruct a call stack. To get the complete call stack, perf
has to save/restore all LBR registers during a context switch. Due to
the large number of the LBR registers, e.g., the current platform has
96 LBR registers, this process causes a high CPU overhead. To reduce
the CPU overhead during a context switch, an LBR state component that
contains all the LBR related registers is introduced in hardware. All
LBR registers can be saved/restored together using one XSAVES/XRSTORS
instruction.
However, the kernel should not save/restore the LBR state component at
each context switch, like other state components, because of the
following unique features of LBR:
- The LBR state component only contains valuable information when LBR
is enabled in the perf subsystem, but for most of the time, LBR is
disabled.
- The size of the LBR state component is huge. For the current
platform, it's 808 bytes.
If the kernel saves/restores the LBR state at each context switch, for
most of the time, it is just a waste of space and cycles.
To efficiently support the LBR state component, it is desired to have:
- only context-switch the LBR when the LBR feature is enabled in perf.
- only allocate an LBR-specific XSAVE buffer on demand.
(Besides the LBR state, a legacy region and an XSAVE header have to be
included in the buffer as well. There is a total of (808+576) byte
overhead for the LBR-specific XSAVE buffer. The overhead only happens
when the perf is actively using LBRs. There is still a space-saving,
on average, when it replaces the constant 808 bytes of overhead for
every task, all the time on the systems that support architectural
LBR.)
- be able to use XSAVES/XRSTORS for accessing LBR at run time.
However, the IA32_XSS should not be adjusted at run time.
(The XCR0 | IA32_XSS are used to determine the requested-feature
bitmap (RFBM) of XSAVES.)
A solution, called dynamic supervisor feature, is introduced to address
this issue, which
- does not allocate a buffer in each task->fpu;
- does not save/restore a state component at each context switch;
- sets the bit corresponding to the dynamic supervisor feature in
IA32_XSS at boot time, and avoids setting it at run time.
- dynamically allocates a specific buffer for a state component
on demand, e.g. only allocates LBR-specific XSAVE buffer when LBR is
enabled in perf. (Note: The buffer has to include the LBR state
component, a legacy region and a XSAVE header space.)
(Implemented in a later patch)
- saves/restores a state component on demand, e.g. manually invokes
the XSAVES/XRSTORS instruction to save/restore the LBR state
to/from the buffer when perf is active and a call stack is required.
(Implemented in a later patch)
A new mask XFEATURE_MASK_DYNAMIC and a helper xfeatures_mask_dynamic()
are introduced to indicate the dynamic supervisor feature. For the
systems which support the Architecture LBR, LBR is the only dynamic
supervisor feature for now. For the previous systems, there is no
dynamic supervisor feature available.
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lkml.kernel.org/r/1593780569-62993-21-git-send-email-kan.liang@linux.intel.com
2020-07-03 12:49:26 +00:00
|
|
|
(nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) ||
|
2020-09-15 16:30:09 +00:00
|
|
|
((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_LBR))) {
|
2015-09-02 23:31:31 +00:00
|
|
|
WARN_ONCE(1, "no structure for xstate: %d\n", nr);
|
|
|
|
XSTATE_WARN_ON(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-02 23:31:30 +00:00
|
|
|
/*
|
|
|
|
* This essentially double-checks what the cpu told us about
|
|
|
|
* how large the XSAVE buffer needs to be. We are recalculating
|
|
|
|
* it to be safe.
|
x86/fpu/xstate: Fix an xstate size check warning with architectural LBRs
An xstate size check warning is triggered on machines which support
Architectural LBRs.
XSAVE consistency problem, dumping leaves
WARNING: CPU: 0 PID: 0 at arch/x86/kernel/fpu/xstate.c:649 fpu__init_system_xstate+0x4d4/0xd0e
Modules linked in:
CPU: 0 PID: 0 Comm: swapper Not tainted intel-arch_lbr+
RIP: 0010:fpu__init_system_xstate+0x4d4/0xd0e
The xstate size check routine, init_xstate_size(), compares the size
retrieved from the hardware with the size of task->fpu, which is
calculated by the software.
The size from the hardware is the total size of the enabled xstates in
XCR0 | IA32_XSS. Architectural LBR state is a dynamic supervisor
feature, which sets the corresponding bit in the IA32_XSS at boot time.
The size from the hardware includes the size of the Architectural LBR
state.
However, a dynamic supervisor feature doesn't allocate a buffer in the
task->fpu. The size of task->fpu doesn't include the size of the
Architectural LBR state. The mismatch will trigger the warning.
Three options as below were considered to fix the issue:
- Correct the size from the hardware by subtracting the size of the
dynamic supervisor features.
The purpose of the check is to compare the size CPU told with the size
of the XSAVE buffer, which is calculated by the software. If the
software mucks with the number from hardware, it removes the value of
the check.
This option is not a good option.
- Prevent the hardware from counting the size of the dynamic supervisor
feature by temporarily removing the corresponding bits in IA32_XSS.
Two extra MSR writes are required to flip the IA32_XSS. The option is
not pretty, but it is workable. The check is only called once at early
boot time. The synchronization or context-switching doesn't need to be
worried.
This option is implemented here.
- Remove the check entirely, because the check hasn't found any real
problems. The option may be an alternative as option 2.
This option is not implemented here.
Add a new function, get_xsaves_size_no_dynamic(), which retrieves the
total size without the dynamic supervisor features from the hardware.
The size will be used to compare with the size of task->fpu.
Fixes: f0dccc9da4c0 ("x86/fpu/xstate: Support dynamic supervisor feature for LBR")
Reported-by: Chang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lore.kernel.org/r/1595253051-75374-1-git-send-email-kan.liang@linux.intel.com
2020-07-20 13:50:51 +00:00
|
|
|
*
|
2021-06-23 12:02:03 +00:00
|
|
|
* Independent XSAVE features allocate their own buffers and are not
|
x86/fpu/xstate: Fix an xstate size check warning with architectural LBRs
An xstate size check warning is triggered on machines which support
Architectural LBRs.
XSAVE consistency problem, dumping leaves
WARNING: CPU: 0 PID: 0 at arch/x86/kernel/fpu/xstate.c:649 fpu__init_system_xstate+0x4d4/0xd0e
Modules linked in:
CPU: 0 PID: 0 Comm: swapper Not tainted intel-arch_lbr+
RIP: 0010:fpu__init_system_xstate+0x4d4/0xd0e
The xstate size check routine, init_xstate_size(), compares the size
retrieved from the hardware with the size of task->fpu, which is
calculated by the software.
The size from the hardware is the total size of the enabled xstates in
XCR0 | IA32_XSS. Architectural LBR state is a dynamic supervisor
feature, which sets the corresponding bit in the IA32_XSS at boot time.
The size from the hardware includes the size of the Architectural LBR
state.
However, a dynamic supervisor feature doesn't allocate a buffer in the
task->fpu. The size of task->fpu doesn't include the size of the
Architectural LBR state. The mismatch will trigger the warning.
Three options as below were considered to fix the issue:
- Correct the size from the hardware by subtracting the size of the
dynamic supervisor features.
The purpose of the check is to compare the size CPU told with the size
of the XSAVE buffer, which is calculated by the software. If the
software mucks with the number from hardware, it removes the value of
the check.
This option is not a good option.
- Prevent the hardware from counting the size of the dynamic supervisor
feature by temporarily removing the corresponding bits in IA32_XSS.
Two extra MSR writes are required to flip the IA32_XSS. The option is
not pretty, but it is workable. The check is only called once at early
boot time. The synchronization or context-switching doesn't need to be
worried.
This option is implemented here.
- Remove the check entirely, because the check hasn't found any real
problems. The option may be an alternative as option 2.
This option is not implemented here.
Add a new function, get_xsaves_size_no_dynamic(), which retrieves the
total size without the dynamic supervisor features from the hardware.
The size will be used to compare with the size of task->fpu.
Fixes: f0dccc9da4c0 ("x86/fpu/xstate: Support dynamic supervisor feature for LBR")
Reported-by: Chang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lore.kernel.org/r/1595253051-75374-1-git-send-email-kan.liang@linux.intel.com
2020-07-20 13:50:51 +00:00
|
|
|
* covered by these checks. Only the size of the buffer for task->fpu
|
|
|
|
* is checked here.
|
2015-09-02 23:31:30 +00:00
|
|
|
*/
|
2021-10-15 01:16:10 +00:00
|
|
|
static void __init do_extra_xstate_size_checks(void)
|
2015-09-02 23:31:30 +00:00
|
|
|
{
|
|
|
|
int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE;
|
|
|
|
int i;
|
|
|
|
|
2021-10-15 01:16:09 +00:00
|
|
|
for_each_extended_xfeature(i, xfeatures_mask_all) {
|
2015-09-02 23:31:31 +00:00
|
|
|
check_xstate_against_struct(i);
|
2015-09-02 23:31:30 +00:00
|
|
|
/*
|
|
|
|
* Supervisor state components can be managed only by
|
2021-06-23 12:01:48 +00:00
|
|
|
* XSAVES.
|
2015-09-02 23:31:30 +00:00
|
|
|
*/
|
2021-06-23 12:01:48 +00:00
|
|
|
if (!cpu_feature_enabled(X86_FEATURE_XSAVES))
|
2015-09-02 23:31:30 +00:00
|
|
|
XSTATE_WARN_ON(xfeature_is_supervisor(i));
|
|
|
|
|
|
|
|
/* Align from the end of the previous feature */
|
|
|
|
if (xfeature_is_aligned(i))
|
|
|
|
paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64);
|
|
|
|
/*
|
|
|
|
* The offset of a given state in the non-compacted
|
|
|
|
* format is given to us in a CPUID leaf. We check
|
|
|
|
* them for being ordered (increasing offsets) in
|
2021-06-23 12:01:48 +00:00
|
|
|
* setup_xstate_features(). XSAVES uses compacted format.
|
2015-09-02 23:31:30 +00:00
|
|
|
*/
|
2021-06-23 12:01:48 +00:00
|
|
|
if (!cpu_feature_enabled(X86_FEATURE_XSAVES))
|
2015-09-02 23:31:30 +00:00
|
|
|
paranoid_xstate_size = xfeature_uncompacted_offset(i);
|
|
|
|
/*
|
|
|
|
* The compacted-format offset always depends on where
|
|
|
|
* the previous state ended.
|
|
|
|
*/
|
|
|
|
paranoid_xstate_size += xfeature_size(i);
|
|
|
|
}
|
2016-05-20 17:47:06 +00:00
|
|
|
XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size);
|
2015-09-02 23:31:30 +00:00
|
|
|
}
|
|
|
|
|
2014-05-29 18:12:43 +00:00
|
|
|
/*
|
2020-05-12 14:54:37 +00:00
|
|
|
* Get total size of enabled xstates in XCR0 | IA32_XSS.
|
2015-09-02 23:31:30 +00:00
|
|
|
*
|
|
|
|
* Note the SDM's wording here. "sub-function 0" only enumerates
|
|
|
|
* the size of the *user* states. If we use it to size a buffer
|
|
|
|
* that we use 'XSAVES' on, we could potentially overflow the
|
|
|
|
* buffer because 'XSAVES' saves system states too.
|
2014-05-29 18:12:43 +00:00
|
|
|
*/
|
2016-05-20 17:47:05 +00:00
|
|
|
static unsigned int __init get_xsaves_size(void)
|
2014-05-29 18:12:43 +00:00
|
|
|
{
|
|
|
|
unsigned int eax, ebx, ecx, edx;
|
2016-05-20 17:47:05 +00:00
|
|
|
/*
|
|
|
|
* - CPUID function 0DH, sub-function 1:
|
|
|
|
* EBX enumerates the size (in bytes) required by
|
|
|
|
* the XSAVES instruction for an XSAVE area
|
|
|
|
* containing all the state components
|
|
|
|
* corresponding to bits currently set in
|
|
|
|
* XCR0 | IA32_XSS.
|
|
|
|
*/
|
|
|
|
cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
|
|
|
|
return ebx;
|
|
|
|
}
|
2014-05-29 18:12:43 +00:00
|
|
|
|
x86/fpu/xstate: Fix an xstate size check warning with architectural LBRs
An xstate size check warning is triggered on machines which support
Architectural LBRs.
XSAVE consistency problem, dumping leaves
WARNING: CPU: 0 PID: 0 at arch/x86/kernel/fpu/xstate.c:649 fpu__init_system_xstate+0x4d4/0xd0e
Modules linked in:
CPU: 0 PID: 0 Comm: swapper Not tainted intel-arch_lbr+
RIP: 0010:fpu__init_system_xstate+0x4d4/0xd0e
The xstate size check routine, init_xstate_size(), compares the size
retrieved from the hardware with the size of task->fpu, which is
calculated by the software.
The size from the hardware is the total size of the enabled xstates in
XCR0 | IA32_XSS. Architectural LBR state is a dynamic supervisor
feature, which sets the corresponding bit in the IA32_XSS at boot time.
The size from the hardware includes the size of the Architectural LBR
state.
However, a dynamic supervisor feature doesn't allocate a buffer in the
task->fpu. The size of task->fpu doesn't include the size of the
Architectural LBR state. The mismatch will trigger the warning.
Three options as below were considered to fix the issue:
- Correct the size from the hardware by subtracting the size of the
dynamic supervisor features.
The purpose of the check is to compare the size CPU told with the size
of the XSAVE buffer, which is calculated by the software. If the
software mucks with the number from hardware, it removes the value of
the check.
This option is not a good option.
- Prevent the hardware from counting the size of the dynamic supervisor
feature by temporarily removing the corresponding bits in IA32_XSS.
Two extra MSR writes are required to flip the IA32_XSS. The option is
not pretty, but it is workable. The check is only called once at early
boot time. The synchronization or context-switching doesn't need to be
worried.
This option is implemented here.
- Remove the check entirely, because the check hasn't found any real
problems. The option may be an alternative as option 2.
This option is not implemented here.
Add a new function, get_xsaves_size_no_dynamic(), which retrieves the
total size without the dynamic supervisor features from the hardware.
The size will be used to compare with the size of task->fpu.
Fixes: f0dccc9da4c0 ("x86/fpu/xstate: Support dynamic supervisor feature for LBR")
Reported-by: Chang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lore.kernel.org/r/1595253051-75374-1-git-send-email-kan.liang@linux.intel.com
2020-07-20 13:50:51 +00:00
|
|
|
/*
|
2021-06-23 12:02:03 +00:00
|
|
|
* Get the total size of the enabled xstates without the independent supervisor
|
x86/fpu/xstate: Fix an xstate size check warning with architectural LBRs
An xstate size check warning is triggered on machines which support
Architectural LBRs.
XSAVE consistency problem, dumping leaves
WARNING: CPU: 0 PID: 0 at arch/x86/kernel/fpu/xstate.c:649 fpu__init_system_xstate+0x4d4/0xd0e
Modules linked in:
CPU: 0 PID: 0 Comm: swapper Not tainted intel-arch_lbr+
RIP: 0010:fpu__init_system_xstate+0x4d4/0xd0e
The xstate size check routine, init_xstate_size(), compares the size
retrieved from the hardware with the size of task->fpu, which is
calculated by the software.
The size from the hardware is the total size of the enabled xstates in
XCR0 | IA32_XSS. Architectural LBR state is a dynamic supervisor
feature, which sets the corresponding bit in the IA32_XSS at boot time.
The size from the hardware includes the size of the Architectural LBR
state.
However, a dynamic supervisor feature doesn't allocate a buffer in the
task->fpu. The size of task->fpu doesn't include the size of the
Architectural LBR state. The mismatch will trigger the warning.
Three options as below were considered to fix the issue:
- Correct the size from the hardware by subtracting the size of the
dynamic supervisor features.
The purpose of the check is to compare the size CPU told with the size
of the XSAVE buffer, which is calculated by the software. If the
software mucks with the number from hardware, it removes the value of
the check.
This option is not a good option.
- Prevent the hardware from counting the size of the dynamic supervisor
feature by temporarily removing the corresponding bits in IA32_XSS.
Two extra MSR writes are required to flip the IA32_XSS. The option is
not pretty, but it is workable. The check is only called once at early
boot time. The synchronization or context-switching doesn't need to be
worried.
This option is implemented here.
- Remove the check entirely, because the check hasn't found any real
problems. The option may be an alternative as option 2.
This option is not implemented here.
Add a new function, get_xsaves_size_no_dynamic(), which retrieves the
total size without the dynamic supervisor features from the hardware.
The size will be used to compare with the size of task->fpu.
Fixes: f0dccc9da4c0 ("x86/fpu/xstate: Support dynamic supervisor feature for LBR")
Reported-by: Chang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lore.kernel.org/r/1595253051-75374-1-git-send-email-kan.liang@linux.intel.com
2020-07-20 13:50:51 +00:00
|
|
|
* features.
|
|
|
|
*/
|
2021-06-23 12:02:03 +00:00
|
|
|
static unsigned int __init get_xsaves_size_no_independent(void)
|
x86/fpu/xstate: Fix an xstate size check warning with architectural LBRs
An xstate size check warning is triggered on machines which support
Architectural LBRs.
XSAVE consistency problem, dumping leaves
WARNING: CPU: 0 PID: 0 at arch/x86/kernel/fpu/xstate.c:649 fpu__init_system_xstate+0x4d4/0xd0e
Modules linked in:
CPU: 0 PID: 0 Comm: swapper Not tainted intel-arch_lbr+
RIP: 0010:fpu__init_system_xstate+0x4d4/0xd0e
The xstate size check routine, init_xstate_size(), compares the size
retrieved from the hardware with the size of task->fpu, which is
calculated by the software.
The size from the hardware is the total size of the enabled xstates in
XCR0 | IA32_XSS. Architectural LBR state is a dynamic supervisor
feature, which sets the corresponding bit in the IA32_XSS at boot time.
The size from the hardware includes the size of the Architectural LBR
state.
However, a dynamic supervisor feature doesn't allocate a buffer in the
task->fpu. The size of task->fpu doesn't include the size of the
Architectural LBR state. The mismatch will trigger the warning.
Three options as below were considered to fix the issue:
- Correct the size from the hardware by subtracting the size of the
dynamic supervisor features.
The purpose of the check is to compare the size CPU told with the size
of the XSAVE buffer, which is calculated by the software. If the
software mucks with the number from hardware, it removes the value of
the check.
This option is not a good option.
- Prevent the hardware from counting the size of the dynamic supervisor
feature by temporarily removing the corresponding bits in IA32_XSS.
Two extra MSR writes are required to flip the IA32_XSS. The option is
not pretty, but it is workable. The check is only called once at early
boot time. The synchronization or context-switching doesn't need to be
worried.
This option is implemented here.
- Remove the check entirely, because the check hasn't found any real
problems. The option may be an alternative as option 2.
This option is not implemented here.
Add a new function, get_xsaves_size_no_dynamic(), which retrieves the
total size without the dynamic supervisor features from the hardware.
The size will be used to compare with the size of task->fpu.
Fixes: f0dccc9da4c0 ("x86/fpu/xstate: Support dynamic supervisor feature for LBR")
Reported-by: Chang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lore.kernel.org/r/1595253051-75374-1-git-send-email-kan.liang@linux.intel.com
2020-07-20 13:50:51 +00:00
|
|
|
{
|
2021-06-23 12:02:03 +00:00
|
|
|
u64 mask = xfeatures_mask_independent();
|
x86/fpu/xstate: Fix an xstate size check warning with architectural LBRs
An xstate size check warning is triggered on machines which support
Architectural LBRs.
XSAVE consistency problem, dumping leaves
WARNING: CPU: 0 PID: 0 at arch/x86/kernel/fpu/xstate.c:649 fpu__init_system_xstate+0x4d4/0xd0e
Modules linked in:
CPU: 0 PID: 0 Comm: swapper Not tainted intel-arch_lbr+
RIP: 0010:fpu__init_system_xstate+0x4d4/0xd0e
The xstate size check routine, init_xstate_size(), compares the size
retrieved from the hardware with the size of task->fpu, which is
calculated by the software.
The size from the hardware is the total size of the enabled xstates in
XCR0 | IA32_XSS. Architectural LBR state is a dynamic supervisor
feature, which sets the corresponding bit in the IA32_XSS at boot time.
The size from the hardware includes the size of the Architectural LBR
state.
However, a dynamic supervisor feature doesn't allocate a buffer in the
task->fpu. The size of task->fpu doesn't include the size of the
Architectural LBR state. The mismatch will trigger the warning.
Three options as below were considered to fix the issue:
- Correct the size from the hardware by subtracting the size of the
dynamic supervisor features.
The purpose of the check is to compare the size CPU told with the size
of the XSAVE buffer, which is calculated by the software. If the
software mucks with the number from hardware, it removes the value of
the check.
This option is not a good option.
- Prevent the hardware from counting the size of the dynamic supervisor
feature by temporarily removing the corresponding bits in IA32_XSS.
Two extra MSR writes are required to flip the IA32_XSS. The option is
not pretty, but it is workable. The check is only called once at early
boot time. The synchronization or context-switching doesn't need to be
worried.
This option is implemented here.
- Remove the check entirely, because the check hasn't found any real
problems. The option may be an alternative as option 2.
This option is not implemented here.
Add a new function, get_xsaves_size_no_dynamic(), which retrieves the
total size without the dynamic supervisor features from the hardware.
The size will be used to compare with the size of task->fpu.
Fixes: f0dccc9da4c0 ("x86/fpu/xstate: Support dynamic supervisor feature for LBR")
Reported-by: Chang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lore.kernel.org/r/1595253051-75374-1-git-send-email-kan.liang@linux.intel.com
2020-07-20 13:50:51 +00:00
|
|
|
unsigned int size;
|
|
|
|
|
|
|
|
if (!mask)
|
|
|
|
return get_xsaves_size();
|
|
|
|
|
2021-06-23 12:02:03 +00:00
|
|
|
/* Disable independent features. */
|
x86/fpu/xstate: Fix an xstate size check warning with architectural LBRs
An xstate size check warning is triggered on machines which support
Architectural LBRs.
XSAVE consistency problem, dumping leaves
WARNING: CPU: 0 PID: 0 at arch/x86/kernel/fpu/xstate.c:649 fpu__init_system_xstate+0x4d4/0xd0e
Modules linked in:
CPU: 0 PID: 0 Comm: swapper Not tainted intel-arch_lbr+
RIP: 0010:fpu__init_system_xstate+0x4d4/0xd0e
The xstate size check routine, init_xstate_size(), compares the size
retrieved from the hardware with the size of task->fpu, which is
calculated by the software.
The size from the hardware is the total size of the enabled xstates in
XCR0 | IA32_XSS. Architectural LBR state is a dynamic supervisor
feature, which sets the corresponding bit in the IA32_XSS at boot time.
The size from the hardware includes the size of the Architectural LBR
state.
However, a dynamic supervisor feature doesn't allocate a buffer in the
task->fpu. The size of task->fpu doesn't include the size of the
Architectural LBR state. The mismatch will trigger the warning.
Three options as below were considered to fix the issue:
- Correct the size from the hardware by subtracting the size of the
dynamic supervisor features.
The purpose of the check is to compare the size CPU told with the size
of the XSAVE buffer, which is calculated by the software. If the
software mucks with the number from hardware, it removes the value of
the check.
This option is not a good option.
- Prevent the hardware from counting the size of the dynamic supervisor
feature by temporarily removing the corresponding bits in IA32_XSS.
Two extra MSR writes are required to flip the IA32_XSS. The option is
not pretty, but it is workable. The check is only called once at early
boot time. The synchronization or context-switching doesn't need to be
worried.
This option is implemented here.
- Remove the check entirely, because the check hasn't found any real
problems. The option may be an alternative as option 2.
This option is not implemented here.
Add a new function, get_xsaves_size_no_dynamic(), which retrieves the
total size without the dynamic supervisor features from the hardware.
The size will be used to compare with the size of task->fpu.
Fixes: f0dccc9da4c0 ("x86/fpu/xstate: Support dynamic supervisor feature for LBR")
Reported-by: Chang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lore.kernel.org/r/1595253051-75374-1-git-send-email-kan.liang@linux.intel.com
2020-07-20 13:50:51 +00:00
|
|
|
wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor());
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ask the hardware what size is required of the buffer.
|
|
|
|
* This is the size required for the task->fpu buffer.
|
|
|
|
*/
|
|
|
|
size = get_xsaves_size();
|
|
|
|
|
2021-06-23 12:02:03 +00:00
|
|
|
/* Re-enable independent features so XSAVES will work on them again. */
|
x86/fpu/xstate: Fix an xstate size check warning with architectural LBRs
An xstate size check warning is triggered on machines which support
Architectural LBRs.
XSAVE consistency problem, dumping leaves
WARNING: CPU: 0 PID: 0 at arch/x86/kernel/fpu/xstate.c:649 fpu__init_system_xstate+0x4d4/0xd0e
Modules linked in:
CPU: 0 PID: 0 Comm: swapper Not tainted intel-arch_lbr+
RIP: 0010:fpu__init_system_xstate+0x4d4/0xd0e
The xstate size check routine, init_xstate_size(), compares the size
retrieved from the hardware with the size of task->fpu, which is
calculated by the software.
The size from the hardware is the total size of the enabled xstates in
XCR0 | IA32_XSS. Architectural LBR state is a dynamic supervisor
feature, which sets the corresponding bit in the IA32_XSS at boot time.
The size from the hardware includes the size of the Architectural LBR
state.
However, a dynamic supervisor feature doesn't allocate a buffer in the
task->fpu. The size of task->fpu doesn't include the size of the
Architectural LBR state. The mismatch will trigger the warning.
Three options as below were considered to fix the issue:
- Correct the size from the hardware by subtracting the size of the
dynamic supervisor features.
The purpose of the check is to compare the size CPU told with the size
of the XSAVE buffer, which is calculated by the software. If the
software mucks with the number from hardware, it removes the value of
the check.
This option is not a good option.
- Prevent the hardware from counting the size of the dynamic supervisor
feature by temporarily removing the corresponding bits in IA32_XSS.
Two extra MSR writes are required to flip the IA32_XSS. The option is
not pretty, but it is workable. The check is only called once at early
boot time. The synchronization or context-switching doesn't need to be
worried.
This option is implemented here.
- Remove the check entirely, because the check hasn't found any real
problems. The option may be an alternative as option 2.
This option is not implemented here.
Add a new function, get_xsaves_size_no_dynamic(), which retrieves the
total size without the dynamic supervisor features from the hardware.
The size will be used to compare with the size of task->fpu.
Fixes: f0dccc9da4c0 ("x86/fpu/xstate: Support dynamic supervisor feature for LBR")
Reported-by: Chang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lore.kernel.org/r/1595253051-75374-1-git-send-email-kan.liang@linux.intel.com
2020-07-20 13:50:51 +00:00
|
|
|
wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask);
|
|
|
|
|
|
|
|
return size;
|
|
|
|
}
|
|
|
|
|
2016-05-20 17:47:05 +00:00
|
|
|
static unsigned int __init get_xsave_size(void)
|
|
|
|
{
|
|
|
|
unsigned int eax, ebx, ecx, edx;
|
|
|
|
/*
|
|
|
|
* - CPUID function 0DH, sub-function 0:
|
|
|
|
* EBX enumerates the size (in bytes) required by
|
|
|
|
* the XSAVE instruction for an XSAVE area
|
|
|
|
* containing all the *user* state components
|
|
|
|
* corresponding to bits currently set in XCR0.
|
|
|
|
*/
|
|
|
|
cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
|
|
|
|
return ebx;
|
2015-09-02 23:31:25 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Will the runtime-enumerated 'xstate_size' fit in the init
|
|
|
|
* task's statically-allocated buffer?
|
|
|
|
*/
|
2021-10-15 01:16:10 +00:00
|
|
|
static bool __init is_supported_xstate_size(unsigned int test_xstate_size)
|
2015-09-02 23:31:25 +00:00
|
|
|
{
|
2021-10-13 14:55:28 +00:00
|
|
|
if (test_xstate_size <= sizeof(init_fpstate.regs))
|
2015-09-02 23:31:25 +00:00
|
|
|
return true;
|
|
|
|
|
|
|
|
pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n",
|
2021-10-13 14:55:28 +00:00
|
|
|
sizeof(init_fpstate.regs), test_xstate_size);
|
2015-09-02 23:31:25 +00:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2019-01-08 13:02:25 +00:00
|
|
|
static int __init init_xstate_size(void)
|
2015-09-02 23:31:25 +00:00
|
|
|
{
|
|
|
|
/* Recompute the context size for enabled features: */
|
2016-05-20 17:47:05 +00:00
|
|
|
unsigned int possible_xstate_size;
|
|
|
|
unsigned int xsave_size;
|
|
|
|
|
|
|
|
xsave_size = get_xsave_size();
|
|
|
|
|
|
|
|
if (boot_cpu_has(X86_FEATURE_XSAVES))
|
2021-06-23 12:02:03 +00:00
|
|
|
possible_xstate_size = get_xsaves_size_no_independent();
|
2016-05-20 17:47:05 +00:00
|
|
|
else
|
|
|
|
possible_xstate_size = xsave_size;
|
2015-09-02 23:31:25 +00:00
|
|
|
|
|
|
|
/* Ensure we have the space to store all enabled: */
|
|
|
|
if (!is_supported_xstate_size(possible_xstate_size))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The size is OK, we are definitely going to use xsave,
|
|
|
|
* make it known to the world that we need more space.
|
|
|
|
*/
|
2016-05-20 17:47:06 +00:00
|
|
|
fpu_kernel_xstate_size = possible_xstate_size;
|
2015-09-02 23:31:30 +00:00
|
|
|
do_extra_xstate_size_checks();
|
2016-05-20 17:47:05 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* User space is always in standard format.
|
|
|
|
*/
|
|
|
|
fpu_user_xstate_size = xsave_size;
|
2015-09-02 23:31:25 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-09-02 23:31:26 +00:00
|
|
|
/*
|
|
|
|
* We enabled the XSAVE hardware, but something went wrong and
|
|
|
|
* we can not use it. Disable it.
|
|
|
|
*/
|
2021-10-15 01:16:10 +00:00
|
|
|
static void __init fpu__init_disable_system_xstate(void)
|
2015-09-02 23:31:25 +00:00
|
|
|
{
|
2020-05-12 14:54:37 +00:00
|
|
|
xfeatures_mask_all = 0;
|
2015-09-02 23:31:25 +00:00
|
|
|
cr4_clear_bits(X86_CR4_OSXSAVE);
|
2019-07-04 06:07:43 +00:00
|
|
|
setup_clear_cpu_cap(X86_FEATURE_XSAVE);
|
2014-05-29 18:12:43 +00:00
|
|
|
}
|
|
|
|
|
2008-07-29 17:29:19 +00:00
|
|
|
/*
|
|
|
|
* Enable and initialize the xsave feature.
|
2015-04-25 04:26:36 +00:00
|
|
|
* Called once per system bootup.
|
2008-07-29 17:29:19 +00:00
|
|
|
*/
|
2015-05-04 07:52:42 +00:00
|
|
|
void __init fpu__init_system_xstate(void)
|
2008-07-29 17:29:19 +00:00
|
|
|
{
|
|
|
|
unsigned int eax, ebx, ecx, edx;
|
2021-06-23 12:01:31 +00:00
|
|
|
u64 xfeatures;
|
2015-09-02 23:31:25 +00:00
|
|
|
int err;
|
2017-10-13 21:56:44 +00:00
|
|
|
int i;
|
2015-05-05 09:34:49 +00:00
|
|
|
|
2017-01-18 19:15:40 +00:00
|
|
|
if (!boot_cpu_has(X86_FEATURE_FPU)) {
|
|
|
|
pr_info("x86/fpu: No FPU detected\n");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2016-04-04 20:25:02 +00:00
|
|
|
if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
|
2017-01-18 19:15:40 +00:00
|
|
|
pr_info("x86/fpu: x87 FPU will use %s\n",
|
|
|
|
boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE");
|
2015-04-25 04:47:24 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2010-07-21 17:03:54 +00:00
|
|
|
if (boot_cpu_data.cpuid_level < XSTATE_CPUID) {
|
2015-05-05 09:34:49 +00:00
|
|
|
WARN_ON_FPU(1);
|
2010-07-21 17:03:54 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-05-12 14:54:37 +00:00
|
|
|
/*
|
|
|
|
* Find user xstates supported by the processor.
|
|
|
|
*/
|
2010-07-21 17:03:54 +00:00
|
|
|
cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
|
2020-05-12 14:54:37 +00:00
|
|
|
xfeatures_mask_all = eax + ((u64)edx << 32);
|
2008-07-29 17:29:19 +00:00
|
|
|
|
2020-05-12 14:54:38 +00:00
|
|
|
/*
|
|
|
|
* Find supervisor xstates supported by the processor.
|
|
|
|
*/
|
|
|
|
cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
|
|
|
|
xfeatures_mask_all |= ecx + ((u64)edx << 32);
|
|
|
|
|
2021-06-23 12:02:16 +00:00
|
|
|
if ((xfeatures_mask_uabi() & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
|
2016-07-20 19:45:51 +00:00
|
|
|
/*
|
|
|
|
* This indicates that something really unexpected happened
|
|
|
|
* with the enumeration. Disable XSAVE and try to continue
|
|
|
|
* booting without it. This is too early to BUG().
|
|
|
|
*/
|
2020-05-12 14:54:37 +00:00
|
|
|
pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n",
|
|
|
|
xfeatures_mask_all);
|
2016-07-20 19:45:51 +00:00
|
|
|
goto out_disable;
|
2008-07-29 17:29:19 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 21:56:44 +00:00
|
|
|
/*
|
|
|
|
* Clear XSAVE features that are disabled in the normal CPUID.
|
|
|
|
*/
|
|
|
|
for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) {
|
|
|
|
if (!boot_cpu_has(xsave_cpuid_features[i]))
|
2020-05-12 14:54:37 +00:00
|
|
|
xfeatures_mask_all &= ~BIT_ULL(i);
|
2017-10-13 21:56:44 +00:00
|
|
|
}
|
|
|
|
|
2021-06-23 12:01:32 +00:00
|
|
|
xfeatures_mask_all &= XFEATURE_MASK_USER_SUPPORTED |
|
|
|
|
XFEATURE_MASK_SUPERVISOR_SUPPORTED;
|
|
|
|
|
2021-06-23 12:01:31 +00:00
|
|
|
/* Store it for paranoia check at the end */
|
|
|
|
xfeatures = xfeatures_mask_all;
|
2010-07-21 17:03:53 +00:00
|
|
|
|
2015-04-25 04:26:36 +00:00
|
|
|
/* Enable xstate instructions to be able to continue with initialization: */
|
|
|
|
fpu__init_cpu_xstate();
|
2015-09-02 23:31:25 +00:00
|
|
|
err = init_xstate_size();
|
2016-07-20 19:45:51 +00:00
|
|
|
if (err)
|
|
|
|
goto out_disable;
|
2008-07-29 17:29:19 +00:00
|
|
|
|
2016-06-17 20:07:17 +00:00
|
|
|
/*
|
|
|
|
* Update info used for ptrace frames; use standard-format size and no
|
|
|
|
* supervisor xstates:
|
|
|
|
*/
|
2021-06-23 12:02:16 +00:00
|
|
|
update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask_uabi());
|
2016-06-17 20:07:17 +00:00
|
|
|
|
2015-04-30 10:45:38 +00:00
|
|
|
fpu__init_prepare_fx_sw_frame();
|
2012-09-06 21:58:52 +00:00
|
|
|
setup_init_fpu_buf();
|
2020-01-09 21:14:51 +00:00
|
|
|
setup_xstate_comp_offsets();
|
2020-05-12 14:54:42 +00:00
|
|
|
setup_supervisor_only_offsets();
|
2008-07-29 17:29:19 +00:00
|
|
|
|
2021-06-23 12:01:31 +00:00
|
|
|
/*
|
|
|
|
* Paranoia check whether something in the setup modified the
|
|
|
|
* xfeatures mask.
|
|
|
|
*/
|
|
|
|
if (xfeatures != xfeatures_mask_all) {
|
|
|
|
pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init, disabling XSAVE\n",
|
|
|
|
xfeatures, xfeatures_mask_all);
|
|
|
|
goto out_disable;
|
|
|
|
}
|
|
|
|
|
|
|
|
print_xstate_offset_size();
|
2015-09-02 23:31:24 +00:00
|
|
|
pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
|
2020-05-12 14:54:37 +00:00
|
|
|
xfeatures_mask_all,
|
2016-05-20 17:47:06 +00:00
|
|
|
fpu_kernel_xstate_size,
|
2016-04-04 20:25:03 +00:00
|
|
|
boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard");
|
2016-07-20 19:45:51 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
out_disable:
|
|
|
|
/* something went wrong, try to boot without any XSAVE support */
|
|
|
|
fpu__init_disable_system_xstate();
|
2008-07-29 17:29:19 +00:00
|
|
|
}
|
2010-07-20 18:50:51 +00:00
|
|
|
|
2015-04-24 08:02:32 +00:00
|
|
|
/*
|
|
|
|
* Restore minimal FPU state after suspend:
|
|
|
|
*/
|
|
|
|
void fpu__resume_cpu(void)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Restore XCR0 on xsave capable CPUs:
|
|
|
|
*/
|
2021-06-23 12:02:16 +00:00
|
|
|
if (cpu_feature_enabled(X86_FEATURE_XSAVE))
|
|
|
|
xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_uabi());
|
2020-05-12 14:54:38 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Restore IA32_XSS. The same CPUID bit enumerates support
|
|
|
|
* of XSAVES and MSR_IA32_XSS.
|
|
|
|
*/
|
2021-06-23 12:02:16 +00:00
|
|
|
if (cpu_feature_enabled(X86_FEATURE_XSAVES)) {
|
x86/fpu/xstate: Support dynamic supervisor feature for LBR
Last Branch Records (LBR) registers are used to log taken branches and
other control flows. In perf with call stack mode, LBR information is
used to reconstruct a call stack. To get the complete call stack, perf
has to save/restore all LBR registers during a context switch. Due to
the large number of the LBR registers, e.g., the current platform has
96 LBR registers, this process causes a high CPU overhead. To reduce
the CPU overhead during a context switch, an LBR state component that
contains all the LBR related registers is introduced in hardware. All
LBR registers can be saved/restored together using one XSAVES/XRSTORS
instruction.
However, the kernel should not save/restore the LBR state component at
each context switch, like other state components, because of the
following unique features of LBR:
- The LBR state component only contains valuable information when LBR
is enabled in the perf subsystem, but for most of the time, LBR is
disabled.
- The size of the LBR state component is huge. For the current
platform, it's 808 bytes.
If the kernel saves/restores the LBR state at each context switch, for
most of the time, it is just a waste of space and cycles.
To efficiently support the LBR state component, it is desired to have:
- only context-switch the LBR when the LBR feature is enabled in perf.
- only allocate an LBR-specific XSAVE buffer on demand.
(Besides the LBR state, a legacy region and an XSAVE header have to be
included in the buffer as well. There is a total of (808+576) byte
overhead for the LBR-specific XSAVE buffer. The overhead only happens
when the perf is actively using LBRs. There is still a space-saving,
on average, when it replaces the constant 808 bytes of overhead for
every task, all the time on the systems that support architectural
LBR.)
- be able to use XSAVES/XRSTORS for accessing LBR at run time.
However, the IA32_XSS should not be adjusted at run time.
(The XCR0 | IA32_XSS are used to determine the requested-feature
bitmap (RFBM) of XSAVES.)
A solution, called dynamic supervisor feature, is introduced to address
this issue, which
- does not allocate a buffer in each task->fpu;
- does not save/restore a state component at each context switch;
- sets the bit corresponding to the dynamic supervisor feature in
IA32_XSS at boot time, and avoids setting it at run time.
- dynamically allocates a specific buffer for a state component
on demand, e.g. only allocates LBR-specific XSAVE buffer when LBR is
enabled in perf. (Note: The buffer has to include the LBR state
component, a legacy region and a XSAVE header space.)
(Implemented in a later patch)
- saves/restores a state component on demand, e.g. manually invokes
the XSAVES/XRSTORS instruction to save/restore the LBR state
to/from the buffer when perf is active and a call stack is required.
(Implemented in a later patch)
A new mask XFEATURE_MASK_DYNAMIC and a helper xfeatures_mask_dynamic()
are introduced to indicate the dynamic supervisor feature. For the
systems which support the Architecture LBR, LBR is the only dynamic
supervisor feature for now. For the previous systems, there is no
dynamic supervisor feature available.
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lkml.kernel.org/r/1593780569-62993-21-git-send-email-kan.liang@linux.intel.com
2020-07-03 12:49:26 +00:00
|
|
|
wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() |
|
2021-06-23 12:02:03 +00:00
|
|
|
xfeatures_mask_independent());
|
x86/fpu/xstate: Support dynamic supervisor feature for LBR
Last Branch Records (LBR) registers are used to log taken branches and
other control flows. In perf with call stack mode, LBR information is
used to reconstruct a call stack. To get the complete call stack, perf
has to save/restore all LBR registers during a context switch. Due to
the large number of the LBR registers, e.g., the current platform has
96 LBR registers, this process causes a high CPU overhead. To reduce
the CPU overhead during a context switch, an LBR state component that
contains all the LBR related registers is introduced in hardware. All
LBR registers can be saved/restored together using one XSAVES/XRSTORS
instruction.
However, the kernel should not save/restore the LBR state component at
each context switch, like other state components, because of the
following unique features of LBR:
- The LBR state component only contains valuable information when LBR
is enabled in the perf subsystem, but for most of the time, LBR is
disabled.
- The size of the LBR state component is huge. For the current
platform, it's 808 bytes.
If the kernel saves/restores the LBR state at each context switch, for
most of the time, it is just a waste of space and cycles.
To efficiently support the LBR state component, it is desired to have:
- only context-switch the LBR when the LBR feature is enabled in perf.
- only allocate an LBR-specific XSAVE buffer on demand.
(Besides the LBR state, a legacy region and an XSAVE header have to be
included in the buffer as well. There is a total of (808+576) byte
overhead for the LBR-specific XSAVE buffer. The overhead only happens
when the perf is actively using LBRs. There is still a space-saving,
on average, when it replaces the constant 808 bytes of overhead for
every task, all the time on the systems that support architectural
LBR.)
- be able to use XSAVES/XRSTORS for accessing LBR at run time.
However, the IA32_XSS should not be adjusted at run time.
(The XCR0 | IA32_XSS are used to determine the requested-feature
bitmap (RFBM) of XSAVES.)
A solution, called dynamic supervisor feature, is introduced to address
this issue, which
- does not allocate a buffer in each task->fpu;
- does not save/restore a state component at each context switch;
- sets the bit corresponding to the dynamic supervisor feature in
IA32_XSS at boot time, and avoids setting it at run time.
- dynamically allocates a specific buffer for a state component
on demand, e.g. only allocates LBR-specific XSAVE buffer when LBR is
enabled in perf. (Note: The buffer has to include the LBR state
component, a legacy region and a XSAVE header space.)
(Implemented in a later patch)
- saves/restores a state component on demand, e.g. manually invokes
the XSAVES/XRSTORS instruction to save/restore the LBR state
to/from the buffer when perf is active and a call stack is required.
(Implemented in a later patch)
A new mask XFEATURE_MASK_DYNAMIC and a helper xfeatures_mask_dynamic()
are introduced to indicate the dynamic supervisor feature. For the
systems which support the Architecture LBR, LBR is the only dynamic
supervisor feature for now. For the previous systems, there is no
dynamic supervisor feature available.
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Link: https://lkml.kernel.org/r/1593780569-62993-21-git-send-email-kan.liang@linux.intel.com
2020-07-03 12:49:26 +00:00
|
|
|
}
|
2015-04-24 08:02:32 +00:00
|
|
|
}
|
|
|
|
|
2016-02-12 21:02:35 +00:00
|
|
|
/*
|
2019-04-03 16:41:39 +00:00
|
|
|
* Given an xstate feature nr, calculate where in the xsave
|
2016-02-12 21:02:35 +00:00
|
|
|
* buffer the state is. Callers should ensure that the buffer
|
|
|
|
* is valid.
|
|
|
|
*/
|
2019-04-03 16:41:39 +00:00
|
|
|
static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
|
2016-02-12 21:02:35 +00:00
|
|
|
{
|
2019-04-03 16:41:39 +00:00
|
|
|
if (!xfeature_enabled(xfeature_nr)) {
|
2016-07-11 16:18:55 +00:00
|
|
|
WARN_ON_FPU(1);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2019-04-03 16:41:39 +00:00
|
|
|
return (void *)xsave + xstate_comp_offsets[xfeature_nr];
|
2016-02-12 21:02:35 +00:00
|
|
|
}
|
2014-05-29 18:12:44 +00:00
|
|
|
/*
|
|
|
|
* Given the xsave area and a state inside, this function returns the
|
|
|
|
* address of the state.
|
|
|
|
*
|
|
|
|
* This is the API that is called to get xstate address in either
|
|
|
|
* standard format or compacted format of xsave area.
|
|
|
|
*
|
2015-06-07 18:37:00 +00:00
|
|
|
* Note that if there is no data for the field in the xsave buffer
|
|
|
|
* this will return NULL.
|
|
|
|
*
|
2014-05-29 18:12:44 +00:00
|
|
|
* Inputs:
|
2015-06-07 18:37:00 +00:00
|
|
|
* xstate: the thread's storage area for all FPU data
|
2019-04-03 16:41:40 +00:00
|
|
|
* xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP,
|
|
|
|
* XFEATURE_SSE, etc...)
|
2014-05-29 18:12:44 +00:00
|
|
|
* Output:
|
2015-06-07 18:37:00 +00:00
|
|
|
* address of the state in the xsave area, or NULL if the
|
|
|
|
* field is not present in the xsave buffer.
|
2014-05-29 18:12:44 +00:00
|
|
|
*/
|
2019-04-03 16:41:40 +00:00
|
|
|
void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
|
2014-05-29 18:12:44 +00:00
|
|
|
{
|
2015-06-07 18:37:00 +00:00
|
|
|
/*
|
|
|
|
* Do we even *have* xsave state?
|
|
|
|
*/
|
|
|
|
if (!boot_cpu_has(X86_FEATURE_XSAVE))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We should not ever be requesting features that we
|
2020-05-12 14:54:37 +00:00
|
|
|
* have not enabled.
|
2015-06-07 18:37:00 +00:00
|
|
|
*/
|
2020-05-12 14:54:37 +00:00
|
|
|
WARN_ONCE(!(xfeatures_mask_all & BIT_ULL(xfeature_nr)),
|
2015-06-07 18:37:00 +00:00
|
|
|
"get of unsupported state");
|
|
|
|
/*
|
|
|
|
* This assumes the last 'xsave*' instruction to
|
2019-04-03 16:41:40 +00:00
|
|
|
* have requested that 'xfeature_nr' be saved.
|
2015-06-07 18:37:00 +00:00
|
|
|
* If it did not, we might be seeing and old value
|
|
|
|
* of the field in the buffer.
|
|
|
|
*
|
|
|
|
* This can happen because the last 'xsave' did not
|
|
|
|
* request that this feature be saved (unlikely)
|
|
|
|
* or because the "init optimization" caused it
|
|
|
|
* to not be saved.
|
|
|
|
*/
|
2019-04-03 16:41:40 +00:00
|
|
|
if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr)))
|
2014-05-29 18:12:44 +00:00
|
|
|
return NULL;
|
|
|
|
|
2019-04-03 16:41:39 +00:00
|
|
|
return __raw_xsave_addr(xsave, xfeature_nr);
|
2014-05-29 18:12:44 +00:00
|
|
|
}
|
x86/fpu/xstate: Wrap get_xsave_addr() to make it safer
The MPX code appears is calling a low-level FPU function
(copy_fpregs_to_fpstate()). This function is not able to
be called in all contexts, although it is safe to call
directly in some cases.
Although probably correct, the current code is ugly and
potentially error-prone. So, add a wrapper that calls
the (slightly) higher-level fpu__save() (which is preempt-
safe) and also ensures that we even *have* an FPU context
(in the case that this was called when in lazy FPU mode).
Ingo had this to say about the details about when we need
preemption disabled:
> it's indeed generally unsafe to access/copy FPU registers with preemption enabled,
> for two reasons:
>
> - on older systems that use FSAVE the instruction destroys FPU register
> contents, which has to be handled carefully
>
> - even on newer systems if we copy to FPU registers (which this code doesn't)
> then we don't want a context switch to occur in the middle of it, because a
> context switch will write to the fpstate, potentially overwriting our new data
> with old FPU state.
>
> But it's safe to access FPU registers with preemption enabled in a couple of
> special cases:
>
> - potentially destructively saving FPU registers: the signal handling code does
> this in copy_fpstate_to_sigframe(), because it can rely on the signal restore
> side to restore the original FPU state.
>
> - reading FPU registers on modern systems: we don't do this anywhere at the
> moment, mostly to keep symmetry with older systems where FSAVE is
> destructive.
>
> - initializing FPU registers on modern systems: fpu__clear() does this. Here
> it's safe because we don't copy from the fpstate.
>
> - directly writing FPU registers from user-space memory (!). We do this in
> fpu__restore_sig(), and it's safe because neither context switches nor
> irq-handler FPU use can corrupt the source context of the copy (which is
> user-space memory).
>
> Note that the MPX code's current use of copy_fpregs_to_fpstate() was safe I think,
> because:
>
> - MPX is predicated on eagerfpu, so the destructive F[N]SAVE instruction won't be
> used.
>
> - the code was only reading FPU registers, and was doing it only in places that
> guaranteed that an FPU state was already active (i.e. didn't do it in
> kthreads)
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave@sr71.net>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Suresh Siddha <sbsiddha@gmail.com>
Cc: bp@alien8.de
Link: http://lkml.kernel.org/r/20150607183700.AA881696@viggo.jf.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-06-07 18:37:00 +00:00
|
|
|
|
x86/pkeys: Allocation/free syscalls
This patch adds two new system calls:
int pkey_alloc(unsigned long flags, unsigned long init_access_rights)
int pkey_free(int pkey);
These implement an "allocator" for the protection keys
themselves, which can be thought of as analogous to the allocator
that the kernel has for file descriptors. The kernel tracks
which numbers are in use, and only allows operations on keys that
are valid. A key which was not obtained by pkey_alloc() may not,
for instance, be passed to pkey_mprotect().
These system calls are also very important given the kernel's use
of pkeys to implement execute-only support. These help ensure
that userspace can never assume that it has control of a key
unless it first asks the kernel. The kernel does not promise to
preserve PKRU (right register) contents except for allocated
pkeys.
The 'init_access_rights' argument to pkey_alloc() specifies the
rights that will be established for the returned pkey. For
instance:
pkey = pkey_alloc(flags, PKEY_DENY_WRITE);
will allocate 'pkey', but also sets the bits in PKRU[1] such that
writing to 'pkey' is already denied.
The kernel does not prevent pkey_free() from successfully freeing
in-use pkeys (those still assigned to a memory range by
pkey_mprotect()). It would be expensive to implement the checks
for this, so we instead say, "Just don't do it" since sane
software will never do it anyway.
Any piece of userspace calling pkey_alloc() needs to be prepared
for it to fail. Why? pkey_alloc() returns the same error code
(ENOSPC) when there are no pkeys and when pkeys are unsupported.
They can be unsupported for a whole host of reasons, so apps must
be prepared for this. Also, libraries or LD_PRELOADs might steal
keys before an application gets access to them.
This allocation mechanism could be implemented in userspace.
Even if we did it in userspace, we would still need additional
user/kernel interfaces to tell userspace which keys are being
used by the kernel internally (such as for execute-only
mappings). Having the kernel provide this facility completely
removes the need for these additional interfaces, or having an
implementation of this in userspace at all.
Note that we have to make changes to all of the architectures
that do not use mman-common.h because we use the new
PKEY_DENY_ACCESS/WRITE macros in arch-independent code.
1. PKRU is the Protection Key Rights User register. It is a
usermode-accessible register that controls whether writes
and/or access to each individual pkey is allowed or denied.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Cc: linux-arch@vger.kernel.org
Cc: Dave Hansen <dave@sr71.net>
Cc: arnd@arndb.de
Cc: linux-api@vger.kernel.org
Cc: linux-mm@kvack.org
Cc: luto@kernel.org
Cc: akpm@linux-foundation.org
Cc: torvalds@linux-foundation.org
Link: http://lkml.kernel.org/r/20160729163015.444FE75F@viggo.jf.intel.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2016-07-29 16:30:15 +00:00
|
|
|
#ifdef CONFIG_ARCH_HAS_PKEYS
|
|
|
|
|
2016-02-12 21:02:36 +00:00
|
|
|
/*
|
x86/mm/pkeys: Fix compact mode by removing protection keys' XSAVE buffer manipulation
The Memory Protection Keys "rights register" (PKRU) is
XSAVE-managed, and is saved/restored along with the FPU state.
When kernel code accesses FPU regsisters, it does a delicate
dance with preempt. Otherwise, the context switching code can
get confused as to whether the most up-to-date state is in the
registers themselves or in the XSAVE buffer.
But, PKRU is not a normal FPU register. Using it does not
generate the normal device-not-available (#NM) exceptions which
means we can not manage it lazily, and the kernel completley
disallows using lazy mode when it is enabled.
The dance with preempt *only* occurs when managing the FPU
lazily. Since we never manage PKRU lazily, we do not have to do
the dance with preempt; we can access it directly. Doing it
this way saves a ton of complicated code (and is faster too).
Further, the XSAVES reenabling failed to patch a bit of code
in fpu__xfeature_set_state() the checked for compacted buffers.
That check caused fpu__xfeature_set_state() to silently refuse to
work when the kernel is using compacted XSAVE buffers. This
broke execute-only and future pkey_mprotect() support when using
compact XSAVE buffers.
But, removing fpu__xfeature_set_state() gets rid of this issue,
in addition to the nice cleanup and speedup.
This fixes the same thing as a fix that Sai posted:
https://lkml.org/lkml/2016/7/25/637
The fix that he posted is a much more obviously correct, but I
think we should just do this instead.
Reported-by: Sai Praneeth Prakhya <sai.praneeth.prakhya@intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Dave Hansen <dave@sr71.net>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Quentin Casasnovas <quentin.casasnovas@oracle.com>
Cc: Ravi Shankar <ravi.v.shankar@intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Yu-Cheng Yu <yu-cheng.yu@intel.com>
Link: http://lkml.kernel.org/r/20160727232040.7D060DAD@viggo.jf.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-07-27 23:20:40 +00:00
|
|
|
* This will go out and modify PKRU register to set the access
|
|
|
|
* rights for @pkey to @init_val.
|
2016-02-12 21:02:36 +00:00
|
|
|
*/
|
|
|
|
int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
|
2021-06-23 12:01:50 +00:00
|
|
|
unsigned long init_val)
|
2016-02-12 21:02:36 +00:00
|
|
|
{
|
2021-06-23 12:01:50 +00:00
|
|
|
u32 old_pkru, new_pkru_bits = 0;
|
|
|
|
int pkey_shift;
|
2016-02-12 21:02:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* This check implies XSAVE support. OSPKE only gets
|
|
|
|
* set if we enable XSAVE and we enable PKU in XCR0.
|
|
|
|
*/
|
2021-06-23 12:02:07 +00:00
|
|
|
if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
|
2016-02-12 21:02:36 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
2020-01-22 16:53:46 +00:00
|
|
|
/*
|
|
|
|
* This code should only be called with valid 'pkey'
|
|
|
|
* values originating from in-kernel users. Complain
|
|
|
|
* if a bad value is observed.
|
|
|
|
*/
|
2021-06-23 12:01:50 +00:00
|
|
|
if (WARN_ON_ONCE(pkey >= arch_max_pkey()))
|
|
|
|
return -EINVAL;
|
2020-01-22 16:53:46 +00:00
|
|
|
|
2016-06-17 20:07:17 +00:00
|
|
|
/* Set the bits we need in PKRU: */
|
2016-02-12 21:02:36 +00:00
|
|
|
if (init_val & PKEY_DISABLE_ACCESS)
|
|
|
|
new_pkru_bits |= PKRU_AD_BIT;
|
|
|
|
if (init_val & PKEY_DISABLE_WRITE)
|
|
|
|
new_pkru_bits |= PKRU_WD_BIT;
|
|
|
|
|
2016-06-17 20:07:17 +00:00
|
|
|
/* Shift the bits in to the correct place in PKRU for pkey: */
|
2021-06-23 12:01:50 +00:00
|
|
|
pkey_shift = pkey * PKRU_BITS_PER_PKEY;
|
2016-02-12 21:02:36 +00:00
|
|
|
new_pkru_bits <<= pkey_shift;
|
|
|
|
|
x86/mm/pkeys: Fix compact mode by removing protection keys' XSAVE buffer manipulation
The Memory Protection Keys "rights register" (PKRU) is
XSAVE-managed, and is saved/restored along with the FPU state.
When kernel code accesses FPU regsisters, it does a delicate
dance with preempt. Otherwise, the context switching code can
get confused as to whether the most up-to-date state is in the
registers themselves or in the XSAVE buffer.
But, PKRU is not a normal FPU register. Using it does not
generate the normal device-not-available (#NM) exceptions which
means we can not manage it lazily, and the kernel completley
disallows using lazy mode when it is enabled.
The dance with preempt *only* occurs when managing the FPU
lazily. Since we never manage PKRU lazily, we do not have to do
the dance with preempt; we can access it directly. Doing it
this way saves a ton of complicated code (and is faster too).
Further, the XSAVES reenabling failed to patch a bit of code
in fpu__xfeature_set_state() the checked for compacted buffers.
That check caused fpu__xfeature_set_state() to silently refuse to
work when the kernel is using compacted XSAVE buffers. This
broke execute-only and future pkey_mprotect() support when using
compact XSAVE buffers.
But, removing fpu__xfeature_set_state() gets rid of this issue,
in addition to the nice cleanup and speedup.
This fixes the same thing as a fix that Sai posted:
https://lkml.org/lkml/2016/7/25/637
The fix that he posted is a much more obviously correct, but I
think we should just do this instead.
Reported-by: Sai Praneeth Prakhya <sai.praneeth.prakhya@intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Dave Hansen <dave@sr71.net>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Quentin Casasnovas <quentin.casasnovas@oracle.com>
Cc: Ravi Shankar <ravi.v.shankar@intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Yu-Cheng Yu <yu-cheng.yu@intel.com>
Link: http://lkml.kernel.org/r/20160727232040.7D060DAD@viggo.jf.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-07-27 23:20:40 +00:00
|
|
|
/* Get old PKRU and mask off any old bits in place: */
|
|
|
|
old_pkru = read_pkru();
|
|
|
|
old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift);
|
2016-02-12 21:02:36 +00:00
|
|
|
|
x86/mm/pkeys: Fix compact mode by removing protection keys' XSAVE buffer manipulation
The Memory Protection Keys "rights register" (PKRU) is
XSAVE-managed, and is saved/restored along with the FPU state.
When kernel code accesses FPU regsisters, it does a delicate
dance with preempt. Otherwise, the context switching code can
get confused as to whether the most up-to-date state is in the
registers themselves or in the XSAVE buffer.
But, PKRU is not a normal FPU register. Using it does not
generate the normal device-not-available (#NM) exceptions which
means we can not manage it lazily, and the kernel completley
disallows using lazy mode when it is enabled.
The dance with preempt *only* occurs when managing the FPU
lazily. Since we never manage PKRU lazily, we do not have to do
the dance with preempt; we can access it directly. Doing it
this way saves a ton of complicated code (and is faster too).
Further, the XSAVES reenabling failed to patch a bit of code
in fpu__xfeature_set_state() the checked for compacted buffers.
That check caused fpu__xfeature_set_state() to silently refuse to
work when the kernel is using compacted XSAVE buffers. This
broke execute-only and future pkey_mprotect() support when using
compact XSAVE buffers.
But, removing fpu__xfeature_set_state() gets rid of this issue,
in addition to the nice cleanup and speedup.
This fixes the same thing as a fix that Sai posted:
https://lkml.org/lkml/2016/7/25/637
The fix that he posted is a much more obviously correct, but I
think we should just do this instead.
Reported-by: Sai Praneeth Prakhya <sai.praneeth.prakhya@intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Dave Hansen <dave@sr71.net>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Quentin Casasnovas <quentin.casasnovas@oracle.com>
Cc: Ravi Shankar <ravi.v.shankar@intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Yu-Cheng Yu <yu-cheng.yu@intel.com>
Link: http://lkml.kernel.org/r/20160727232040.7D060DAD@viggo.jf.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-07-27 23:20:40 +00:00
|
|
|
/* Write old part along with new part: */
|
|
|
|
write_pkru(old_pkru | new_pkru_bits);
|
2016-06-17 20:07:17 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
x86/pkeys: Allocation/free syscalls
This patch adds two new system calls:
int pkey_alloc(unsigned long flags, unsigned long init_access_rights)
int pkey_free(int pkey);
These implement an "allocator" for the protection keys
themselves, which can be thought of as analogous to the allocator
that the kernel has for file descriptors. The kernel tracks
which numbers are in use, and only allows operations on keys that
are valid. A key which was not obtained by pkey_alloc() may not,
for instance, be passed to pkey_mprotect().
These system calls are also very important given the kernel's use
of pkeys to implement execute-only support. These help ensure
that userspace can never assume that it has control of a key
unless it first asks the kernel. The kernel does not promise to
preserve PKRU (right register) contents except for allocated
pkeys.
The 'init_access_rights' argument to pkey_alloc() specifies the
rights that will be established for the returned pkey. For
instance:
pkey = pkey_alloc(flags, PKEY_DENY_WRITE);
will allocate 'pkey', but also sets the bits in PKRU[1] such that
writing to 'pkey' is already denied.
The kernel does not prevent pkey_free() from successfully freeing
in-use pkeys (those still assigned to a memory range by
pkey_mprotect()). It would be expensive to implement the checks
for this, so we instead say, "Just don't do it" since sane
software will never do it anyway.
Any piece of userspace calling pkey_alloc() needs to be prepared
for it to fail. Why? pkey_alloc() returns the same error code
(ENOSPC) when there are no pkeys and when pkeys are unsupported.
They can be unsupported for a whole host of reasons, so apps must
be prepared for this. Also, libraries or LD_PRELOADs might steal
keys before an application gets access to them.
This allocation mechanism could be implemented in userspace.
Even if we did it in userspace, we would still need additional
user/kernel interfaces to tell userspace which keys are being
used by the kernel internally (such as for execute-only
mappings). Having the kernel provide this facility completely
removes the need for these additional interfaces, or having an
implementation of this in userspace at all.
Note that we have to make changes to all of the architectures
that do not use mman-common.h because we use the new
PKEY_DENY_ACCESS/WRITE macros in arch-independent code.
1. PKRU is the Protection Key Rights User register. It is a
usermode-accessible register that controls whether writes
and/or access to each individual pkey is allowed or denied.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Cc: linux-arch@vger.kernel.org
Cc: Dave Hansen <dave@sr71.net>
Cc: arnd@arndb.de
Cc: linux-api@vger.kernel.org
Cc: linux-mm@kvack.org
Cc: luto@kernel.org
Cc: akpm@linux-foundation.org
Cc: torvalds@linux-foundation.org
Link: http://lkml.kernel.org/r/20160729163015.444FE75F@viggo.jf.intel.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2016-07-29 16:30:15 +00:00
|
|
|
#endif /* ! CONFIG_ARCH_HAS_PKEYS */
|
2016-06-17 20:07:17 +00:00
|
|
|
|
2021-06-23 12:01:28 +00:00
|
|
|
static void copy_feature(bool from_xstate, struct membuf *to, void *xstate,
|
|
|
|
void *init_xstate, unsigned int size)
|
2017-09-23 12:59:45 +00:00
|
|
|
{
|
2021-06-23 12:01:28 +00:00
|
|
|
membuf_write(to, from_xstate ? xstate : init_xstate, size);
|
2017-09-23 12:59:45 +00:00
|
|
|
}
|
|
|
|
|
2021-06-23 12:01:42 +00:00
|
|
|
/**
|
2021-10-15 01:16:15 +00:00
|
|
|
* __copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer
|
2021-06-23 12:01:42 +00:00
|
|
|
* @to: membuf descriptor
|
2021-10-15 01:16:15 +00:00
|
|
|
* @xsave: The xsave from which to copy
|
|
|
|
* @pkru_val: The PKRU value to store in the PKRU component
|
2021-06-23 12:01:42 +00:00
|
|
|
* @copy_mode: The requested copy mode
|
|
|
|
*
|
|
|
|
* Converts from kernel XSAVE or XSAVES compacted format to UABI conforming
|
|
|
|
* format, i.e. from the kernel internal hardware dependent storage format
|
|
|
|
* to the requested @mode. UABI XSTATE is always uncompacted!
|
2017-09-23 12:59:45 +00:00
|
|
|
*
|
2021-06-23 12:01:42 +00:00
|
|
|
* It supports partial copy but @to.pos always starts from zero.
|
2017-09-23 12:59:45 +00:00
|
|
|
*/
|
2021-10-15 01:16:15 +00:00
|
|
|
void __copy_xstate_to_uabi_buf(struct membuf to, struct xregs_state *xsave,
|
|
|
|
u32 pkru_val, enum xstate_copy_mode copy_mode)
|
2017-09-23 12:59:45 +00:00
|
|
|
{
|
2021-06-23 12:01:28 +00:00
|
|
|
const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr);
|
2021-10-13 14:55:28 +00:00
|
|
|
struct xregs_state *xinit = &init_fpstate.regs.xsave;
|
2017-09-23 12:59:45 +00:00
|
|
|
struct xstate_header header;
|
2021-06-23 12:01:28 +00:00
|
|
|
unsigned int zerofrom;
|
2021-10-15 01:16:09 +00:00
|
|
|
u64 mask;
|
2017-09-23 12:59:53 +00:00
|
|
|
int i;
|
2017-09-23 12:59:45 +00:00
|
|
|
|
2021-06-24 15:09:18 +00:00
|
|
|
memset(&header, 0, sizeof(header));
|
2017-09-23 12:59:45 +00:00
|
|
|
header.xfeatures = xsave->header.xfeatures;
|
2021-06-23 12:01:42 +00:00
|
|
|
|
|
|
|
/* Mask out the feature bits depending on copy mode */
|
|
|
|
switch (copy_mode) {
|
|
|
|
case XSTATE_COPY_FP:
|
|
|
|
header.xfeatures &= XFEATURE_MASK_FP;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case XSTATE_COPY_FX:
|
|
|
|
header.xfeatures &= XFEATURE_MASK_FP | XFEATURE_MASK_SSE;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case XSTATE_COPY_XSAVE:
|
2021-06-23 12:02:16 +00:00
|
|
|
header.xfeatures &= xfeatures_mask_uabi();
|
2021-06-23 12:01:42 +00:00
|
|
|
break;
|
|
|
|
}
|
2017-09-23 12:59:45 +00:00
|
|
|
|
2021-06-23 12:01:28 +00:00
|
|
|
/* Copy FP state up to MXCSR */
|
|
|
|
copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387,
|
|
|
|
&xinit->i387, off_mxcsr);
|
|
|
|
|
|
|
|
/* Copy MXCSR when SSE or YMM are set in the feature mask */
|
|
|
|
copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM),
|
|
|
|
&to, &xsave->i387.mxcsr, &xinit->i387.mxcsr,
|
|
|
|
MXCSR_AND_FLAGS_SIZE);
|
|
|
|
|
|
|
|
/* Copy the remaining FP state */
|
|
|
|
copy_feature(header.xfeatures & XFEATURE_MASK_FP,
|
|
|
|
&to, &xsave->i387.st_space, &xinit->i387.st_space,
|
|
|
|
sizeof(xsave->i387.st_space));
|
|
|
|
|
|
|
|
/* Copy the SSE state - shared with YMM, but independently managed */
|
|
|
|
copy_feature(header.xfeatures & XFEATURE_MASK_SSE,
|
|
|
|
&to, &xsave->i387.xmm_space, &xinit->i387.xmm_space,
|
|
|
|
sizeof(xsave->i387.xmm_space));
|
|
|
|
|
2021-06-23 12:01:42 +00:00
|
|
|
if (copy_mode != XSTATE_COPY_XSAVE)
|
|
|
|
goto out;
|
|
|
|
|
2021-06-23 12:01:28 +00:00
|
|
|
/* Zero the padding area */
|
|
|
|
membuf_zero(&to, sizeof(xsave->i387.padding));
|
|
|
|
|
|
|
|
/* Copy xsave->i387.sw_reserved */
|
|
|
|
membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved));
|
|
|
|
|
|
|
|
/* Copy the user space relevant state of @xsave->header */
|
|
|
|
membuf_write(&to, &header, sizeof(header));
|
|
|
|
|
|
|
|
zerofrom = offsetof(struct xregs_state, extended_state_area);
|
2017-09-23 12:59:45 +00:00
|
|
|
|
2021-10-15 01:16:09 +00:00
|
|
|
/*
|
|
|
|
* The ptrace buffer is in non-compacted XSAVE format. In
|
|
|
|
* non-compacted format disabled features still occupy state space,
|
|
|
|
* but there is no state to copy from in the compacted
|
|
|
|
* init_fpstate. The gap tracking will zero these states.
|
|
|
|
*/
|
|
|
|
mask = xfeatures_mask_uabi();
|
2017-09-23 12:59:45 +00:00
|
|
|
|
2021-10-15 01:16:09 +00:00
|
|
|
for_each_extended_xfeature(i, mask) {
|
2021-06-23 12:01:28 +00:00
|
|
|
/*
|
|
|
|
* If there was a feature or alignment gap, zero the space
|
|
|
|
* in the destination buffer.
|
|
|
|
*/
|
|
|
|
if (zerofrom < xstate_offsets[i])
|
|
|
|
membuf_zero(&to, xstate_offsets[i] - zerofrom);
|
|
|
|
|
2021-06-23 12:02:19 +00:00
|
|
|
if (i == XFEATURE_PKRU) {
|
|
|
|
struct pkru_state pkru = {0};
|
|
|
|
/*
|
|
|
|
* PKRU is not necessarily up to date in the
|
2021-10-15 01:16:15 +00:00
|
|
|
* XSAVE buffer. Use the provided value.
|
2021-06-23 12:02:19 +00:00
|
|
|
*/
|
2021-10-15 01:16:15 +00:00
|
|
|
pkru.pkru = pkru_val;
|
2021-06-23 12:02:19 +00:00
|
|
|
membuf_write(&to, &pkru, sizeof(pkru));
|
|
|
|
} else {
|
|
|
|
copy_feature(header.xfeatures & BIT_ULL(i), &to,
|
|
|
|
__raw_xsave_addr(xsave, i),
|
|
|
|
__raw_xsave_addr(xinit, i),
|
|
|
|
xstate_sizes[i]);
|
|
|
|
}
|
2021-06-23 12:01:28 +00:00
|
|
|
/*
|
|
|
|
* Keep track of the last copied state in the non-compacted
|
|
|
|
* target buffer for gap zeroing.
|
|
|
|
*/
|
|
|
|
zerofrom = xstate_offsets[i] + xstate_sizes[i];
|
2017-09-23 12:59:45 +00:00
|
|
|
}
|
2021-06-23 12:01:28 +00:00
|
|
|
|
2021-06-23 12:01:42 +00:00
|
|
|
out:
|
2021-06-23 12:01:28 +00:00
|
|
|
if (to.left)
|
|
|
|
membuf_zero(&to, to.left);
|
2016-06-17 20:07:17 +00:00
|
|
|
}
|
|
|
|
|
2021-10-15 01:16:15 +00:00
|
|
|
/**
|
|
|
|
* copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer
|
|
|
|
* @to: membuf descriptor
|
|
|
|
* @tsk: The task from which to copy the saved xstate
|
|
|
|
* @copy_mode: The requested copy mode
|
|
|
|
*
|
|
|
|
* Converts from kernel XSAVE or XSAVES compacted format to UABI conforming
|
|
|
|
* format, i.e. from the kernel internal hardware dependent storage format
|
|
|
|
* to the requested @mode. UABI XSTATE is always uncompacted!
|
|
|
|
*
|
|
|
|
* It supports partial copy but @to.pos always starts from zero.
|
|
|
|
*/
|
|
|
|
void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk,
|
|
|
|
enum xstate_copy_mode copy_mode)
|
|
|
|
{
|
2021-10-13 14:55:39 +00:00
|
|
|
__copy_xstate_to_uabi_buf(to, &tsk->thread.fpu.fpstate->regs.xsave,
|
2021-10-15 01:16:15 +00:00
|
|
|
tsk->thread.pkru, copy_mode);
|
|
|
|
}
|
|
|
|
|
2021-06-23 12:01:58 +00:00
|
|
|
static int copy_from_buffer(void *dst, unsigned int offset, unsigned int size,
|
|
|
|
const void *kbuf, const void __user *ubuf)
|
2021-06-23 12:01:37 +00:00
|
|
|
{
|
2021-06-23 12:01:58 +00:00
|
|
|
if (kbuf) {
|
|
|
|
memcpy(dst, kbuf + offset, size);
|
|
|
|
} else {
|
|
|
|
if (copy_from_user(dst, ubuf + offset, size))
|
|
|
|
return -EFAULT;
|
2021-06-23 12:01:37 +00:00
|
|
|
}
|
2021-06-23 12:01:58 +00:00
|
|
|
return 0;
|
2021-06-23 12:01:37 +00:00
|
|
|
}
|
|
|
|
|
2021-06-23 12:01:58 +00:00
|
|
|
|
|
|
|
static int copy_uabi_to_xstate(struct xregs_state *xsave, const void *kbuf,
|
|
|
|
const void __user *ubuf)
|
2017-09-23 12:59:54 +00:00
|
|
|
{
|
|
|
|
unsigned int offset, size;
|
2017-09-24 10:59:07 +00:00
|
|
|
struct xstate_header hdr;
|
2021-06-23 12:01:58 +00:00
|
|
|
u64 mask;
|
|
|
|
int i;
|
2017-09-23 12:59:54 +00:00
|
|
|
|
|
|
|
offset = offsetof(struct xregs_state, header);
|
2021-06-23 12:01:58 +00:00
|
|
|
if (copy_from_buffer(&hdr, offset, sizeof(hdr), kbuf, ubuf))
|
|
|
|
return -EFAULT;
|
2017-09-23 12:59:54 +00:00
|
|
|
|
2020-05-12 14:54:35 +00:00
|
|
|
if (validate_user_xstate_header(&hdr))
|
2017-09-23 12:59:54 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
2021-06-23 12:01:58 +00:00
|
|
|
/* Validate MXCSR when any of the related features is in use */
|
|
|
|
mask = XFEATURE_MASK_FP | XFEATURE_MASK_SSE | XFEATURE_MASK_YMM;
|
|
|
|
if (hdr.xfeatures & mask) {
|
|
|
|
u32 mxcsr[2];
|
|
|
|
|
|
|
|
offset = offsetof(struct fxregs_state, mxcsr);
|
|
|
|
if (copy_from_buffer(mxcsr, offset, sizeof(mxcsr), kbuf, ubuf))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
/* Reserved bits in MXCSR must be zero. */
|
|
|
|
if (mxcsr[0] & ~mxcsr_feature_mask)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* SSE and YMM require MXCSR even when FP is not in use. */
|
|
|
|
if (!(hdr.xfeatures & XFEATURE_MASK_FP)) {
|
|
|
|
xsave->i387.mxcsr = mxcsr[0];
|
|
|
|
xsave->i387.mxcsr_mask = mxcsr[1];
|
|
|
|
}
|
|
|
|
}
|
2021-06-23 12:01:37 +00:00
|
|
|
|
2017-09-23 12:59:54 +00:00
|
|
|
for (i = 0; i < XFEATURE_MAX; i++) {
|
|
|
|
u64 mask = ((u64)1 << i);
|
|
|
|
|
2017-09-24 10:59:08 +00:00
|
|
|
if (hdr.xfeatures & mask) {
|
2019-04-03 16:41:39 +00:00
|
|
|
void *dst = __raw_xsave_addr(xsave, i);
|
2017-09-23 12:59:54 +00:00
|
|
|
|
|
|
|
offset = xstate_offsets[i];
|
|
|
|
size = xstate_sizes[i];
|
|
|
|
|
2021-06-23 12:01:58 +00:00
|
|
|
if (copy_from_buffer(dst, offset, size, kbuf, ubuf))
|
|
|
|
return -EFAULT;
|
2017-09-23 12:59:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The state that came in from userspace was user-state only.
|
|
|
|
* Mask all the user states out of 'xfeatures':
|
|
|
|
*/
|
2020-05-12 14:54:36 +00:00
|
|
|
xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL;
|
2017-09-23 12:59:54 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Add back in the features that came in from userspace:
|
|
|
|
*/
|
2017-09-24 10:59:08 +00:00
|
|
|
xsave->header.xfeatures |= hdr.xfeatures;
|
2017-09-23 12:59:54 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-06-23 12:01:58 +00:00
|
|
|
/*
|
|
|
|
* Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S]
|
2021-10-15 01:16:13 +00:00
|
|
|
* format and copy to the target thread. Used by ptrace and KVM.
|
2021-06-23 12:01:58 +00:00
|
|
|
*/
|
|
|
|
int copy_uabi_from_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf)
|
|
|
|
{
|
|
|
|
return copy_uabi_to_xstate(xsave, kbuf, NULL);
|
|
|
|
}
|
|
|
|
|
2017-09-23 12:59:54 +00:00
|
|
|
/*
|
2021-06-23 12:01:36 +00:00
|
|
|
* Convert from a sigreturn standard-format user-space buffer to kernel
|
|
|
|
* XSAVE[S] format and copy to the target thread. This is called from the
|
|
|
|
* sigreturn() and rt_sigreturn() system calls.
|
2016-06-17 20:07:17 +00:00
|
|
|
*/
|
2021-06-23 12:01:57 +00:00
|
|
|
int copy_sigframe_from_user_to_xstate(struct xregs_state *xsave,
|
|
|
|
const void __user *ubuf)
|
2016-06-17 20:07:17 +00:00
|
|
|
{
|
2021-06-23 12:01:58 +00:00
|
|
|
return copy_uabi_to_xstate(xsave, NULL, ubuf);
|
2016-02-12 21:02:36 +00:00
|
|
|
}
|
2019-06-06 01:22:35 +00:00
|
|
|
|
2021-10-15 01:15:59 +00:00
|
|
|
static bool validate_independent_components(u64 mask)
|
2021-06-23 12:02:04 +00:00
|
|
|
{
|
|
|
|
u64 xchk;
|
|
|
|
|
|
|
|
if (WARN_ON_FPU(!cpu_feature_enabled(X86_FEATURE_XSAVES)))
|
|
|
|
return false;
|
2021-10-15 01:15:59 +00:00
|
|
|
|
|
|
|
xchk = ~xfeatures_mask_independent();
|
2021-06-23 12:02:04 +00:00
|
|
|
|
|
|
|
if (WARN_ON_ONCE(!mask || mask & xchk))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2020-07-03 12:49:27 +00:00
|
|
|
/**
|
2021-06-23 12:02:04 +00:00
|
|
|
* xsaves - Save selected components to a kernel xstate buffer
|
|
|
|
* @xstate: Pointer to the buffer
|
|
|
|
* @mask: Feature mask to select the components to save
|
2020-07-03 12:49:27 +00:00
|
|
|
*
|
2021-06-23 12:02:04 +00:00
|
|
|
* The @xstate buffer must be 64 byte aligned and correctly initialized as
|
|
|
|
* XSAVES does not write the full xstate header. Before first use the
|
|
|
|
* buffer should be zeroed otherwise a consecutive XRSTORS from that buffer
|
|
|
|
* can #GP.
|
2020-07-03 12:49:27 +00:00
|
|
|
*
|
2021-10-15 01:15:59 +00:00
|
|
|
* The feature mask must be a subset of the independent features.
|
2020-07-03 12:49:27 +00:00
|
|
|
*/
|
2021-06-23 12:02:04 +00:00
|
|
|
void xsaves(struct xregs_state *xstate, u64 mask)
|
2020-07-03 12:49:27 +00:00
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
2021-10-15 01:15:59 +00:00
|
|
|
if (!validate_independent_components(mask))
|
2020-07-03 12:49:27 +00:00
|
|
|
return;
|
|
|
|
|
2021-06-23 12:02:04 +00:00
|
|
|
XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err);
|
|
|
|
WARN_ON_ONCE(err);
|
2020-07-03 12:49:27 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2021-06-23 12:02:04 +00:00
|
|
|
* xrstors - Restore selected components from a kernel xstate buffer
|
|
|
|
* @xstate: Pointer to the buffer
|
|
|
|
* @mask: Feature mask to select the components to restore
|
|
|
|
*
|
|
|
|
* The @xstate buffer must be 64 byte aligned and correctly initialized
|
|
|
|
* otherwise XRSTORS from that buffer can #GP.
|
2020-07-03 12:49:27 +00:00
|
|
|
*
|
2021-06-23 12:02:04 +00:00
|
|
|
* Proper usage is to restore the state which was saved with
|
|
|
|
* xsaves() into @xstate.
|
2020-07-03 12:49:27 +00:00
|
|
|
*
|
2021-10-15 01:15:59 +00:00
|
|
|
* The feature mask must be a subset of the independent features.
|
2020-07-03 12:49:27 +00:00
|
|
|
*/
|
2021-06-23 12:02:04 +00:00
|
|
|
void xrstors(struct xregs_state *xstate, u64 mask)
|
2020-07-03 12:49:27 +00:00
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
2021-10-15 01:15:59 +00:00
|
|
|
if (!validate_independent_components(mask))
|
2020-07-03 12:49:27 +00:00
|
|
|
return;
|
|
|
|
|
2021-06-23 12:02:04 +00:00
|
|
|
XSTATE_OP(XRSTORS, xstate, (u32)mask, (u32)(mask >> 32), err);
|
|
|
|
WARN_ON_ONCE(err);
|
2020-07-03 12:49:27 +00:00
|
|
|
}
|
|
|
|
|
2021-10-13 14:55:31 +00:00
|
|
|
#if IS_ENABLED(CONFIG_KVM)
|
|
|
|
void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature)
|
|
|
|
{
|
|
|
|
void *addr = get_xsave_addr(&fps->regs.xsave, xfeature);
|
|
|
|
|
|
|
|
if (addr)
|
|
|
|
memset(addr, 0, xstate_sizes[xfeature]);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(fpstate_clear_xstate_component);
|
|
|
|
#endif
|
|
|
|
|
2019-06-06 01:22:35 +00:00
|
|
|
#ifdef CONFIG_PROC_PID_ARCH_STATUS
|
|
|
|
/*
|
|
|
|
* Report the amount of time elapsed in millisecond since last AVX512
|
|
|
|
* use in the task.
|
|
|
|
*/
|
|
|
|
static void avx512_status(struct seq_file *m, struct task_struct *task)
|
|
|
|
{
|
|
|
|
unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp);
|
|
|
|
long delta;
|
|
|
|
|
|
|
|
if (!timestamp) {
|
|
|
|
/*
|
|
|
|
* Report -1 if no AVX512 usage
|
|
|
|
*/
|
|
|
|
delta = -1;
|
|
|
|
} else {
|
|
|
|
delta = (long)(jiffies - timestamp);
|
|
|
|
/*
|
|
|
|
* Cap to LONG_MAX if time difference > LONG_MAX
|
|
|
|
*/
|
|
|
|
if (delta < 0)
|
|
|
|
delta = LONG_MAX;
|
|
|
|
delta = jiffies_to_msecs(delta);
|
|
|
|
}
|
|
|
|
|
|
|
|
seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta);
|
|
|
|
seq_putc(m, '\n');
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Report architecture specific information
|
|
|
|
*/
|
|
|
|
int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns,
|
|
|
|
struct pid *pid, struct task_struct *task)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Report AVX512 state if the processor and build option supported.
|
|
|
|
*/
|
|
|
|
if (cpu_feature_enabled(X86_FEATURE_AVX512F))
|
|
|
|
avx512_status(m, task);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_PROC_PID_ARCH_STATUS */
|