mirror of
https://github.com/torvalds/linux.git
synced 2024-12-25 20:32:22 +00:00
caab277b1d
Based on 1 normalized pattern(s): this program is free software you can redistribute it and or modify it under the terms of the gnu general public license version 2 as published by the free software foundation this program is distributed in the hope that it will be useful but without any warranty without even the implied warranty of merchantability or fitness for a particular purpose see the gnu general public license for more details you should have received a copy of the gnu general public license along with this program if not see http www gnu org licenses extracted by the scancode license scanner the SPDX license identifier GPL-2.0-only has been chosen to replace the boilerplate/reference in 503 file(s). Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Reviewed-by: Alexios Zavras <alexios.zavras@intel.com> Reviewed-by: Allison Randal <allison@lohutok.net> Reviewed-by: Enrico Weigelt <info@metux.net> Cc: linux-spdx@vger.kernel.org Link: https://lkml.kernel.org/r/20190602204653.811534538@linutronix.de Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
420 lines
11 KiB
C
420 lines
11 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Based on arch/arm/include/asm/uaccess.h
|
|
*
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
*/
|
|
#ifndef __ASM_UACCESS_H
|
|
#define __ASM_UACCESS_H
|
|
|
|
#include <asm/alternative.h>
|
|
#include <asm/kernel-pgtable.h>
|
|
#include <asm/sysreg.h>
|
|
|
|
/*
|
|
* User space memory access functions
|
|
*/
|
|
#include <linux/bitops.h>
|
|
#include <linux/kasan-checks.h>
|
|
#include <linux/string.h>
|
|
|
|
#include <asm/cpufeature.h>
|
|
#include <asm/ptrace.h>
|
|
#include <asm/memory.h>
|
|
#include <asm/extable.h>
|
|
|
|
#define get_fs() (current_thread_info()->addr_limit)
|
|
|
|
static inline void set_fs(mm_segment_t fs)
|
|
{
|
|
current_thread_info()->addr_limit = fs;
|
|
|
|
/*
|
|
* Prevent a mispredicted conditional call to set_fs from forwarding
|
|
* the wrong address limit to access_ok under speculation.
|
|
*/
|
|
spec_bar();
|
|
|
|
/* On user-mode return, check fs is correct */
|
|
set_thread_flag(TIF_FSCHECK);
|
|
|
|
/*
|
|
* Enable/disable UAO so that copy_to_user() etc can access
|
|
* kernel memory with the unprivileged instructions.
|
|
*/
|
|
if (IS_ENABLED(CONFIG_ARM64_UAO) && fs == KERNEL_DS)
|
|
asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO));
|
|
else
|
|
asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO,
|
|
CONFIG_ARM64_UAO));
|
|
}
|
|
|
|
#define segment_eq(a, b) ((a) == (b))
|
|
|
|
/*
|
|
* Test whether a block of memory is a valid user space address.
|
|
* Returns 1 if the range is valid, 0 otherwise.
|
|
*
|
|
* This is equivalent to the following test:
|
|
* (u65)addr + (u65)size <= (u65)current->addr_limit + 1
|
|
*/
|
|
static inline unsigned long __range_ok(const void __user *addr, unsigned long size)
|
|
{
|
|
unsigned long ret, limit = current_thread_info()->addr_limit;
|
|
|
|
__chk_user_ptr(addr);
|
|
asm volatile(
|
|
// A + B <= C + 1 for all A,B,C, in four easy steps:
|
|
// 1: X = A + B; X' = X % 2^64
|
|
" adds %0, %3, %2\n"
|
|
// 2: Set C = 0 if X > 2^64, to guarantee X' > C in step 4
|
|
" csel %1, xzr, %1, hi\n"
|
|
// 3: Set X' = ~0 if X >= 2^64. For X == 2^64, this decrements X'
|
|
// to compensate for the carry flag being set in step 4. For
|
|
// X > 2^64, X' merely has to remain nonzero, which it does.
|
|
" csinv %0, %0, xzr, cc\n"
|
|
// 4: For X < 2^64, this gives us X' - C - 1 <= 0, where the -1
|
|
// comes from the carry in being clear. Otherwise, we are
|
|
// testing X' - C == 0, subject to the previous adjustments.
|
|
" sbcs xzr, %0, %1\n"
|
|
" cset %0, ls\n"
|
|
: "=&r" (ret), "+r" (limit) : "Ir" (size), "0" (addr) : "cc");
|
|
|
|
return ret;
|
|
}
|
|
|
|
#define access_ok(addr, size) __range_ok(addr, size)
|
|
#define user_addr_max get_fs
|
|
|
|
#define _ASM_EXTABLE(from, to) \
|
|
" .pushsection __ex_table, \"a\"\n" \
|
|
" .align 3\n" \
|
|
" .long (" #from " - .), (" #to " - .)\n" \
|
|
" .popsection\n"
|
|
|
|
/*
|
|
* User access enabling/disabling.
|
|
*/
|
|
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
|
|
static inline void __uaccess_ttbr0_disable(void)
|
|
{
|
|
unsigned long flags, ttbr;
|
|
|
|
local_irq_save(flags);
|
|
ttbr = read_sysreg(ttbr1_el1);
|
|
ttbr &= ~TTBR_ASID_MASK;
|
|
/* reserved_ttbr0 placed before swapper_pg_dir */
|
|
write_sysreg(ttbr - RESERVED_TTBR0_SIZE, ttbr0_el1);
|
|
isb();
|
|
/* Set reserved ASID */
|
|
write_sysreg(ttbr, ttbr1_el1);
|
|
isb();
|
|
local_irq_restore(flags);
|
|
}
|
|
|
|
static inline void __uaccess_ttbr0_enable(void)
|
|
{
|
|
unsigned long flags, ttbr0, ttbr1;
|
|
|
|
/*
|
|
* Disable interrupts to avoid preemption between reading the 'ttbr0'
|
|
* variable and the MSR. A context switch could trigger an ASID
|
|
* roll-over and an update of 'ttbr0'.
|
|
*/
|
|
local_irq_save(flags);
|
|
ttbr0 = READ_ONCE(current_thread_info()->ttbr0);
|
|
|
|
/* Restore active ASID */
|
|
ttbr1 = read_sysreg(ttbr1_el1);
|
|
ttbr1 &= ~TTBR_ASID_MASK; /* safety measure */
|
|
ttbr1 |= ttbr0 & TTBR_ASID_MASK;
|
|
write_sysreg(ttbr1, ttbr1_el1);
|
|
isb();
|
|
|
|
/* Restore user page table */
|
|
write_sysreg(ttbr0, ttbr0_el1);
|
|
isb();
|
|
local_irq_restore(flags);
|
|
}
|
|
|
|
static inline bool uaccess_ttbr0_disable(void)
|
|
{
|
|
if (!system_uses_ttbr0_pan())
|
|
return false;
|
|
__uaccess_ttbr0_disable();
|
|
return true;
|
|
}
|
|
|
|
static inline bool uaccess_ttbr0_enable(void)
|
|
{
|
|
if (!system_uses_ttbr0_pan())
|
|
return false;
|
|
__uaccess_ttbr0_enable();
|
|
return true;
|
|
}
|
|
#else
|
|
static inline bool uaccess_ttbr0_disable(void)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static inline bool uaccess_ttbr0_enable(void)
|
|
{
|
|
return false;
|
|
}
|
|
#endif
|
|
|
|
static inline void __uaccess_disable_hw_pan(void)
|
|
{
|
|
asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,
|
|
CONFIG_ARM64_PAN));
|
|
}
|
|
|
|
static inline void __uaccess_enable_hw_pan(void)
|
|
{
|
|
asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,
|
|
CONFIG_ARM64_PAN));
|
|
}
|
|
|
|
#define __uaccess_disable(alt) \
|
|
do { \
|
|
if (!uaccess_ttbr0_disable()) \
|
|
asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt, \
|
|
CONFIG_ARM64_PAN)); \
|
|
} while (0)
|
|
|
|
#define __uaccess_enable(alt) \
|
|
do { \
|
|
if (!uaccess_ttbr0_enable()) \
|
|
asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt, \
|
|
CONFIG_ARM64_PAN)); \
|
|
} while (0)
|
|
|
|
static inline void uaccess_disable(void)
|
|
{
|
|
__uaccess_disable(ARM64_HAS_PAN);
|
|
}
|
|
|
|
static inline void uaccess_enable(void)
|
|
{
|
|
__uaccess_enable(ARM64_HAS_PAN);
|
|
}
|
|
|
|
/*
|
|
* These functions are no-ops when UAO is present.
|
|
*/
|
|
static inline void uaccess_disable_not_uao(void)
|
|
{
|
|
__uaccess_disable(ARM64_ALT_PAN_NOT_UAO);
|
|
}
|
|
|
|
static inline void uaccess_enable_not_uao(void)
|
|
{
|
|
__uaccess_enable(ARM64_ALT_PAN_NOT_UAO);
|
|
}
|
|
|
|
/*
|
|
* Sanitise a uaccess pointer such that it becomes NULL if above the
|
|
* current addr_limit.
|
|
*/
|
|
#define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr)
|
|
static inline void __user *__uaccess_mask_ptr(const void __user *ptr)
|
|
{
|
|
void __user *safe_ptr;
|
|
|
|
asm volatile(
|
|
" bics xzr, %1, %2\n"
|
|
" csel %0, %1, xzr, eq\n"
|
|
: "=&r" (safe_ptr)
|
|
: "r" (ptr), "r" (current_thread_info()->addr_limit)
|
|
: "cc");
|
|
|
|
csdb();
|
|
return safe_ptr;
|
|
}
|
|
|
|
/*
|
|
* The "__xxx" versions of the user access functions do not verify the address
|
|
* space - it must have been done previously with a separate "access_ok()"
|
|
* call.
|
|
*
|
|
* The "__xxx_error" versions set the third argument to -EFAULT if an error
|
|
* occurs, and leave it unchanged on success.
|
|
*/
|
|
#define __get_user_asm(instr, alt_instr, reg, x, addr, err, feature) \
|
|
asm volatile( \
|
|
"1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \
|
|
alt_instr " " reg "1, [%2]\n", feature) \
|
|
"2:\n" \
|
|
" .section .fixup, \"ax\"\n" \
|
|
" .align 2\n" \
|
|
"3: mov %w0, %3\n" \
|
|
" mov %1, #0\n" \
|
|
" b 2b\n" \
|
|
" .previous\n" \
|
|
_ASM_EXTABLE(1b, 3b) \
|
|
: "+r" (err), "=&r" (x) \
|
|
: "r" (addr), "i" (-EFAULT))
|
|
|
|
#define __raw_get_user(x, ptr, err) \
|
|
do { \
|
|
unsigned long __gu_val; \
|
|
__chk_user_ptr(ptr); \
|
|
uaccess_enable_not_uao(); \
|
|
switch (sizeof(*(ptr))) { \
|
|
case 1: \
|
|
__get_user_asm("ldrb", "ldtrb", "%w", __gu_val, (ptr), \
|
|
(err), ARM64_HAS_UAO); \
|
|
break; \
|
|
case 2: \
|
|
__get_user_asm("ldrh", "ldtrh", "%w", __gu_val, (ptr), \
|
|
(err), ARM64_HAS_UAO); \
|
|
break; \
|
|
case 4: \
|
|
__get_user_asm("ldr", "ldtr", "%w", __gu_val, (ptr), \
|
|
(err), ARM64_HAS_UAO); \
|
|
break; \
|
|
case 8: \
|
|
__get_user_asm("ldr", "ldtr", "%x", __gu_val, (ptr), \
|
|
(err), ARM64_HAS_UAO); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
uaccess_disable_not_uao(); \
|
|
(x) = (__force __typeof__(*(ptr)))__gu_val; \
|
|
} while (0)
|
|
|
|
#define __get_user_error(x, ptr, err) \
|
|
do { \
|
|
__typeof__(*(ptr)) __user *__p = (ptr); \
|
|
might_fault(); \
|
|
if (access_ok(__p, sizeof(*__p))) { \
|
|
__p = uaccess_mask_ptr(__p); \
|
|
__raw_get_user((x), __p, (err)); \
|
|
} else { \
|
|
(x) = 0; (err) = -EFAULT; \
|
|
} \
|
|
} while (0)
|
|
|
|
#define __get_user(x, ptr) \
|
|
({ \
|
|
int __gu_err = 0; \
|
|
__get_user_error((x), (ptr), __gu_err); \
|
|
__gu_err; \
|
|
})
|
|
|
|
#define get_user __get_user
|
|
|
|
#define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature) \
|
|
asm volatile( \
|
|
"1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \
|
|
alt_instr " " reg "1, [%2]\n", feature) \
|
|
"2:\n" \
|
|
" .section .fixup,\"ax\"\n" \
|
|
" .align 2\n" \
|
|
"3: mov %w0, %3\n" \
|
|
" b 2b\n" \
|
|
" .previous\n" \
|
|
_ASM_EXTABLE(1b, 3b) \
|
|
: "+r" (err) \
|
|
: "r" (x), "r" (addr), "i" (-EFAULT))
|
|
|
|
#define __raw_put_user(x, ptr, err) \
|
|
do { \
|
|
__typeof__(*(ptr)) __pu_val = (x); \
|
|
__chk_user_ptr(ptr); \
|
|
uaccess_enable_not_uao(); \
|
|
switch (sizeof(*(ptr))) { \
|
|
case 1: \
|
|
__put_user_asm("strb", "sttrb", "%w", __pu_val, (ptr), \
|
|
(err), ARM64_HAS_UAO); \
|
|
break; \
|
|
case 2: \
|
|
__put_user_asm("strh", "sttrh", "%w", __pu_val, (ptr), \
|
|
(err), ARM64_HAS_UAO); \
|
|
break; \
|
|
case 4: \
|
|
__put_user_asm("str", "sttr", "%w", __pu_val, (ptr), \
|
|
(err), ARM64_HAS_UAO); \
|
|
break; \
|
|
case 8: \
|
|
__put_user_asm("str", "sttr", "%x", __pu_val, (ptr), \
|
|
(err), ARM64_HAS_UAO); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
uaccess_disable_not_uao(); \
|
|
} while (0)
|
|
|
|
#define __put_user_error(x, ptr, err) \
|
|
do { \
|
|
__typeof__(*(ptr)) __user *__p = (ptr); \
|
|
might_fault(); \
|
|
if (access_ok(__p, sizeof(*__p))) { \
|
|
__p = uaccess_mask_ptr(__p); \
|
|
__raw_put_user((x), __p, (err)); \
|
|
} else { \
|
|
(err) = -EFAULT; \
|
|
} \
|
|
} while (0)
|
|
|
|
#define __put_user(x, ptr) \
|
|
({ \
|
|
int __pu_err = 0; \
|
|
__put_user_error((x), (ptr), __pu_err); \
|
|
__pu_err; \
|
|
})
|
|
|
|
#define put_user __put_user
|
|
|
|
extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
|
|
#define raw_copy_from_user(to, from, n) \
|
|
({ \
|
|
__arch_copy_from_user((to), __uaccess_mask_ptr(from), (n)); \
|
|
})
|
|
|
|
extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
|
|
#define raw_copy_to_user(to, from, n) \
|
|
({ \
|
|
__arch_copy_to_user(__uaccess_mask_ptr(to), (from), (n)); \
|
|
})
|
|
|
|
extern unsigned long __must_check __arch_copy_in_user(void __user *to, const void __user *from, unsigned long n);
|
|
#define raw_copy_in_user(to, from, n) \
|
|
({ \
|
|
__arch_copy_in_user(__uaccess_mask_ptr(to), \
|
|
__uaccess_mask_ptr(from), (n)); \
|
|
})
|
|
|
|
#define INLINE_COPY_TO_USER
|
|
#define INLINE_COPY_FROM_USER
|
|
|
|
extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n);
|
|
static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n)
|
|
{
|
|
if (access_ok(to, n))
|
|
n = __arch_clear_user(__uaccess_mask_ptr(to), n);
|
|
return n;
|
|
}
|
|
#define clear_user __clear_user
|
|
|
|
extern long strncpy_from_user(char *dest, const char __user *src, long count);
|
|
|
|
extern __must_check long strnlen_user(const char __user *str, long n);
|
|
|
|
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
|
|
struct page;
|
|
void memcpy_page_flushcache(char *to, struct page *page, size_t offset, size_t len);
|
|
extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n);
|
|
|
|
static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
|
|
{
|
|
kasan_check_write(dst, size);
|
|
return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size);
|
|
}
|
|
#endif
|
|
|
|
#endif /* __ASM_UACCESS_H */
|