50596b7559
Now that the user space TLS register is assigned on every return to user
space, we can use it to keep the 'current' pointer while running in the
kernel. This removes the need to access it via thread_info, which is
located at the base of the stack, but will be moved out of there in a
subsequent patch.
Use the __builtin_thread_pointer() helper when available - this will
help GCC understand that reloading the value within the same function is
not necessary, even when using the per-task stack protector (which also
generates accesses via the TLS register). For example, the generated
code below loads TPIDRURO only once, and uses it to access both the
stack canary and the preempt_count fields.
<do_one_initcall>:
e92d 41f0 stmdb sp!, {r4, r5, r6, r7, r8, lr}
ee1d 4f70 mrc 15, 0, r4, cr13, cr0, {3}
4606 mov r6, r0
b094 sub sp, #80 ; 0x50
f8d4 34e8 ldr.w r3, [r4, #1256] ; 0x4e8 <- stack canary
9313 str r3, [sp, #76] ; 0x4c
f8d4 8004 ldr.w r8, [r4, #4] <- preempt count
Co-developed-by: Keith Packard <keithpac@amazon.com>
Signed-off-by: Keith Packard <keithpac@amazon.com>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Linus Walleij <linus.walleij@linaro.org>
Tested-by: Amit Daniel Kachhap <amit.kachhap@arm.com>
51 lines
1.2 KiB
C
51 lines
1.2 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (c) 2021 Keith Packard <keithp@keithp.com>
|
|
* Copyright (c) 2021 Google, LLC <ardb@kernel.org>
|
|
*/
|
|
|
|
#ifndef _ASM_ARM_CURRENT_H
|
|
#define _ASM_ARM_CURRENT_H
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
struct task_struct;
|
|
|
|
static inline void set_current(struct task_struct *cur)
|
|
{
|
|
if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO))
|
|
return;
|
|
|
|
/* Set TPIDRURO */
|
|
asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory");
|
|
}
|
|
|
|
#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
|
|
|
|
static inline struct task_struct *get_current(void)
|
|
{
|
|
struct task_struct *cur;
|
|
|
|
#if __has_builtin(__builtin_thread_pointer)
|
|
/*
|
|
* Use the __builtin helper when available - this results in better
|
|
* code, especially when using GCC in combination with the per-task
|
|
* stack protector, as the compiler will recognize that it needs to
|
|
* load the TLS register only once in every function.
|
|
*/
|
|
cur = __builtin_thread_pointer();
|
|
#else
|
|
asm("mrc p15, 0, %0, c13, c0, 3" : "=r"(cur));
|
|
#endif
|
|
return cur;
|
|
}
|
|
|
|
#define current get_current()
|
|
#else
|
|
#include <asm-generic/current.h>
|
|
#endif /* CONFIG_CURRENT_POINTER_IN_TPIDRURO */
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif /* _ASM_ARM_CURRENT_H */
|