d9b6f90794
Patch series "kasan: integrate with init_on_alloc/free", v3. This patch series integrates HW_TAGS KASAN with init_on_alloc/free by initializing memory via the same arm64 instruction that sets memory tags. This is expected to improve HW_TAGS KASAN performance when init_on_alloc/free is enabled. The exact perfomance numbers are unknown as MTE-enabled hardware doesn't exist yet. This patch (of 5): This change adds an argument to mte_set_mem_tag_range() that allows to enable memory initialization when settinh the allocation tags. The implementation uses stzg instruction instead of stg when this argument indicates to initialize memory. Combining setting allocation tags with memory initialization will improve HW_TAGS KASAN performance when init_on_alloc/free is enabled. This change doesn't integrate memory initialization with KASAN, this is done is subsequent patches in this series. Link: https://lkml.kernel.org/r/cover.1615296150.git.andreyknvl@google.com Link: https://lkml.kernel.org/r/d04ae90cc36be3fe246ea8025e5085495681c3d7.1615296150.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov <andreyknvl@google.com> Acked-by: Marco Elver <elver@google.com> Cc: Christoph Lameter <cl@linux.com> Cc: Pekka Enberg <penberg@kernel.org> Cc: David Rientjes <rientjes@google.com> Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com> Cc: Vlastimil Babka <vbabka@suse.cz> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Will Deacon <will.deacon@arm.com> Cc: Vincenzo Frascino <vincenzo.frascino@arm.com> Cc: Dmitry Vyukov <dvyukov@google.com> Cc: Andrey Ryabinin <aryabinin@virtuozzo.com> Cc: Alexander Potapenko <glider@google.com> Cc: Peter Collingbourne <pcc@google.com> Cc: Evgenii Stepanov <eugenis@google.com> Cc: Branislav Rankov <Branislav.Rankov@arm.com> Cc: Kevin Brodsky <kevin.brodsky@arm.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
145 lines
2.7 KiB
C
145 lines
2.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* Copyright (C) 2020 ARM Ltd.
|
|
*/
|
|
#ifndef __ASM_MTE_KASAN_H
|
|
#define __ASM_MTE_KASAN_H
|
|
|
|
#include <asm/mte-def.h>
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#include <linux/types.h>
|
|
|
|
#ifdef CONFIG_ARM64_MTE
|
|
|
|
/*
|
|
* These functions are meant to be only used from KASAN runtime through
|
|
* the arch_*() interface defined in asm/memory.h.
|
|
* These functions don't include system_supports_mte() checks,
|
|
* as KASAN only calls them when MTE is supported and enabled.
|
|
*/
|
|
|
|
static inline u8 mte_get_ptr_tag(void *ptr)
|
|
{
|
|
/* Note: The format of KASAN tags is 0xF<x> */
|
|
u8 tag = 0xF0 | (u8)(((u64)(ptr)) >> MTE_TAG_SHIFT);
|
|
|
|
return tag;
|
|
}
|
|
|
|
/* Get allocation tag for the address. */
|
|
static inline u8 mte_get_mem_tag(void *addr)
|
|
{
|
|
asm(__MTE_PREAMBLE "ldg %0, [%0]"
|
|
: "+r" (addr));
|
|
|
|
return mte_get_ptr_tag(addr);
|
|
}
|
|
|
|
/* Generate a random tag. */
|
|
static inline u8 mte_get_random_tag(void)
|
|
{
|
|
void *addr;
|
|
|
|
asm(__MTE_PREAMBLE "irg %0, %0"
|
|
: "=r" (addr));
|
|
|
|
return mte_get_ptr_tag(addr);
|
|
}
|
|
|
|
/*
|
|
* Assign allocation tags for a region of memory based on the pointer tag.
|
|
* Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and
|
|
* size must be non-zero and MTE_GRANULE_SIZE aligned.
|
|
*/
|
|
static inline void mte_set_mem_tag_range(void *addr, size_t size,
|
|
u8 tag, bool init)
|
|
{
|
|
u64 curr, end;
|
|
|
|
if (!size)
|
|
return;
|
|
|
|
curr = (u64)__tag_set(addr, tag);
|
|
end = curr + size;
|
|
|
|
/*
|
|
* 'asm volatile' is required to prevent the compiler to move
|
|
* the statement outside of the loop.
|
|
*/
|
|
if (init) {
|
|
do {
|
|
asm volatile(__MTE_PREAMBLE "stzg %0, [%0]"
|
|
:
|
|
: "r" (curr)
|
|
: "memory");
|
|
curr += MTE_GRANULE_SIZE;
|
|
} while (curr != end);
|
|
} else {
|
|
do {
|
|
asm volatile(__MTE_PREAMBLE "stg %0, [%0]"
|
|
:
|
|
: "r" (curr)
|
|
: "memory");
|
|
curr += MTE_GRANULE_SIZE;
|
|
} while (curr != end);
|
|
}
|
|
}
|
|
|
|
void mte_enable_kernel_sync(void);
|
|
void mte_enable_kernel_async(void);
|
|
void mte_init_tags(u64 max_tag);
|
|
|
|
void mte_set_report_once(bool state);
|
|
bool mte_report_once(void);
|
|
|
|
#else /* CONFIG_ARM64_MTE */
|
|
|
|
static inline u8 mte_get_ptr_tag(void *ptr)
|
|
{
|
|
return 0xFF;
|
|
}
|
|
|
|
static inline u8 mte_get_mem_tag(void *addr)
|
|
{
|
|
return 0xFF;
|
|
}
|
|
|
|
static inline u8 mte_get_random_tag(void)
|
|
{
|
|
return 0xFF;
|
|
}
|
|
|
|
static inline void mte_set_mem_tag_range(void *addr, size_t size,
|
|
u8 tag, bool init)
|
|
{
|
|
}
|
|
|
|
static inline void mte_enable_kernel_sync(void)
|
|
{
|
|
}
|
|
|
|
static inline void mte_enable_kernel_async(void)
|
|
{
|
|
}
|
|
|
|
static inline void mte_init_tags(u64 max_tag)
|
|
{
|
|
}
|
|
|
|
static inline void mte_set_report_once(bool state)
|
|
{
|
|
}
|
|
|
|
static inline bool mte_report_once(void)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
#endif /* CONFIG_ARM64_MTE */
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif /* __ASM_MTE_KASAN_H */
|