2019-06-04 08:11:33 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2015-10-12 15:52:58 +00:00
|
|
|
/*
|
|
|
|
|
* This file contains kasan initialization code for ARM64.
|
|
|
|
|
*
|
|
|
|
|
* Copyright (c) 2015 Samsung Electronics Co., Ltd.
|
|
|
|
|
* Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#define pr_fmt(fmt) "kasan: " fmt
|
|
|
|
|
#include <linux/kasan.h>
|
|
|
|
|
#include <linux/kernel.h>
|
2017-02-04 00:20:53 +00:00
|
|
|
#include <linux/sched/task.h>
|
2015-10-12 15:52:58 +00:00
|
|
|
#include <linux/memblock.h>
|
|
|
|
|
#include <linux/start_kernel.h>
|
2017-01-10 21:35:49 +00:00
|
|
|
#include <linux/mm.h>
|
2015-10-12 15:52:58 +00:00
|
|
|
|
2016-01-25 11:45:02 +00:00
|
|
|
#include <asm/mmu_context.h>
|
2016-02-16 12:52:40 +00:00
|
|
|
#include <asm/kernel-pgtable.h>
|
2015-10-12 15:52:58 +00:00
|
|
|
#include <asm/page.h>
|
|
|
|
|
#include <asm/pgalloc.h>
|
2016-02-16 12:52:40 +00:00
|
|
|
#include <asm/sections.h>
|
2015-10-12 15:52:58 +00:00
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
|
|
|
|
|
|
static pgd_t tmp_pg_dir[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
|
|
|
|
|
|
2017-01-10 21:35:49 +00:00
|
|
|
/*
|
|
|
|
|
* The p*d_populate functions call virt_to_phys implicitly so they can't be used
|
|
|
|
|
* directly on kernel symbols (bm_p*d). All the early functions are called too
|
|
|
|
|
* early to use lm_alias so __p*d_populate functions must be used to populate
|
|
|
|
|
* with the physical address from __pa_symbol.
|
|
|
|
|
*/
|
|
|
|
|
|
2017-11-16 01:36:40 +00:00
|
|
|
static phys_addr_t __init kasan_alloc_zeroed_page(int node)
|
2015-10-12 15:52:58 +00:00
|
|
|
{
|
2018-10-30 22:08:04 +00:00
|
|
|
void *p = memblock_alloc_try_nid(PAGE_SIZE, PAGE_SIZE,
|
2017-11-16 01:36:40 +00:00
|
|
|
__pa(MAX_DMA_ADDRESS),
|
mm/memblock.c: skip kmemleak for kasan_init()
Kmemleak does not play well with KASAN (tested on both HPE Apollo 70 and
Huawei TaiShan 2280 aarch64 servers).
After calling start_kernel()->setup_arch()->kasan_init(), kmemleak early
log buffer went from something like 280 to 260000 which caused kmemleak
disabled and crash dump memory reservation failed. The multitude of
kmemleak_alloc() calls is from nested loops while KASAN is setting up full
memory mappings, so let early kmemleak allocations skip those
memblock_alloc_internal() calls came from kasan_init() given that those
early KASAN memory mappings should not reference to other memory. Hence,
no kmemleak false positives.
kasan_init
kasan_map_populate [1]
kasan_pgd_populate [2]
kasan_pud_populate [3]
kasan_pmd_populate [4]
kasan_pte_populate [5]
kasan_alloc_zeroed_page
memblock_alloc_try_nid
memblock_alloc_internal
kmemleak_alloc
[1] for_each_memblock(memory, reg)
[2] while (pgdp++, addr = next, addr != end)
[3] while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp)))
[4] while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp)))
[5] while (ptep++, addr = next, addr != end && pte_none(READ_ONCE(*ptep)))
Link: http://lkml.kernel.org/r/1543442925-17794-1-git-send-email-cai@gmx.us
Signed-off-by: Qian Cai <cai@gmx.us>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@linux.vnet.ibm.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2018-12-28 08:36:29 +00:00
|
|
|
MEMBLOCK_ALLOC_KASAN, node);
|
2019-03-12 06:30:31 +00:00
|
|
|
if (!p)
|
|
|
|
|
panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%llx\n",
|
|
|
|
|
__func__, PAGE_SIZE, PAGE_SIZE, node,
|
|
|
|
|
__pa(MAX_DMA_ADDRESS));
|
|
|
|
|
|
2017-11-16 01:36:40 +00:00
|
|
|
return __pa(p);
|
|
|
|
|
}
|
|
|
|
|
|
2018-12-28 08:30:09 +00:00
|
|
|
static phys_addr_t __init kasan_alloc_raw_page(int node)
|
|
|
|
|
{
|
|
|
|
|
void *p = memblock_alloc_try_nid_raw(PAGE_SIZE, PAGE_SIZE,
|
|
|
|
|
__pa(MAX_DMA_ADDRESS),
|
|
|
|
|
MEMBLOCK_ALLOC_KASAN, node);
|
2019-03-12 06:30:31 +00:00
|
|
|
if (!p)
|
|
|
|
|
panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%llx\n",
|
|
|
|
|
__func__, PAGE_SIZE, PAGE_SIZE, node,
|
|
|
|
|
__pa(MAX_DMA_ADDRESS));
|
|
|
|
|
|
2018-12-28 08:30:09 +00:00
|
|
|
return __pa(p);
|
|
|
|
|
}
|
|
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node,
|
2017-11-16 01:36:40 +00:00
|
|
|
bool early)
|
|
|
|
|
{
|
2018-02-15 11:14:56 +00:00
|
|
|
if (pmd_none(READ_ONCE(*pmdp))) {
|
2018-12-28 08:30:01 +00:00
|
|
|
phys_addr_t pte_phys = early ?
|
|
|
|
|
__pa_symbol(kasan_early_shadow_pte)
|
|
|
|
|
: kasan_alloc_zeroed_page(node);
|
2018-02-15 11:14:56 +00:00
|
|
|
__pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE);
|
2017-11-16 01:36:40 +00:00
|
|
|
}
|
|
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
return early ? pte_offset_kimg(pmdp, addr)
|
|
|
|
|
: pte_offset_kernel(pmdp, addr);
|
2017-11-16 01:36:40 +00:00
|
|
|
}
|
2015-10-12 15:52:58 +00:00
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node,
|
2017-11-16 01:36:40 +00:00
|
|
|
bool early)
|
|
|
|
|
{
|
2018-02-15 11:14:56 +00:00
|
|
|
if (pud_none(READ_ONCE(*pudp))) {
|
2018-12-28 08:30:01 +00:00
|
|
|
phys_addr_t pmd_phys = early ?
|
|
|
|
|
__pa_symbol(kasan_early_shadow_pmd)
|
|
|
|
|
: kasan_alloc_zeroed_page(node);
|
2018-02-15 11:14:56 +00:00
|
|
|
__pud_populate(pudp, pmd_phys, PMD_TYPE_TABLE);
|
2017-11-16 01:36:40 +00:00
|
|
|
}
|
|
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr);
|
2017-11-16 01:36:40 +00:00
|
|
|
}
|
|
|
|
|
|
2020-06-04 23:46:23 +00:00
|
|
|
static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node,
|
2017-11-16 01:36:40 +00:00
|
|
|
bool early)
|
|
|
|
|
{
|
2020-06-04 23:46:23 +00:00
|
|
|
if (p4d_none(READ_ONCE(*p4dp))) {
|
2018-12-28 08:30:01 +00:00
|
|
|
phys_addr_t pud_phys = early ?
|
|
|
|
|
__pa_symbol(kasan_early_shadow_pud)
|
|
|
|
|
: kasan_alloc_zeroed_page(node);
|
2020-06-04 23:46:23 +00:00
|
|
|
__p4d_populate(p4dp, pud_phys, PMD_TYPE_TABLE);
|
2017-11-16 01:36:40 +00:00
|
|
|
}
|
|
|
|
|
|
2020-06-04 23:46:23 +00:00
|
|
|
return early ? pud_offset_kimg(p4dp, addr) : pud_offset(p4dp, addr);
|
2017-11-16 01:36:40 +00:00
|
|
|
}
|
|
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
|
2017-11-16 01:36:40 +00:00
|
|
|
unsigned long end, int node, bool early)
|
|
|
|
|
{
|
|
|
|
|
unsigned long next;
|
2018-02-15 11:14:56 +00:00
|
|
|
pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early);
|
2015-10-12 15:52:58 +00:00
|
|
|
|
|
|
|
|
do {
|
2018-12-28 08:30:01 +00:00
|
|
|
phys_addr_t page_phys = early ?
|
|
|
|
|
__pa_symbol(kasan_early_shadow_page)
|
2018-12-28 08:30:09 +00:00
|
|
|
: kasan_alloc_raw_page(node);
|
|
|
|
|
if (!early)
|
|
|
|
|
memset(__va(page_phys), KASAN_SHADOW_INIT, PAGE_SIZE);
|
2015-10-12 15:52:58 +00:00
|
|
|
next = addr + PAGE_SIZE;
|
2018-02-15 11:14:56 +00:00
|
|
|
set_pte(ptep, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL));
|
|
|
|
|
} while (ptep++, addr = next, addr != end && pte_none(READ_ONCE(*ptep)));
|
2015-10-12 15:52:58 +00:00
|
|
|
}
|
|
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
|
2017-11-16 01:36:40 +00:00
|
|
|
unsigned long end, int node, bool early)
|
2015-10-12 15:52:58 +00:00
|
|
|
{
|
|
|
|
|
unsigned long next;
|
2018-02-15 11:14:56 +00:00
|
|
|
pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early);
|
2015-10-12 15:52:58 +00:00
|
|
|
|
|
|
|
|
do {
|
|
|
|
|
next = pmd_addr_end(addr, end);
|
2018-02-15 11:14:56 +00:00
|
|
|
kasan_pte_populate(pmdp, addr, next, node, early);
|
|
|
|
|
} while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp)));
|
2015-10-12 15:52:58 +00:00
|
|
|
}
|
|
|
|
|
|
2020-06-04 23:46:23 +00:00
|
|
|
static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr,
|
2017-11-16 01:36:40 +00:00
|
|
|
unsigned long end, int node, bool early)
|
2015-10-12 15:52:58 +00:00
|
|
|
{
|
|
|
|
|
unsigned long next;
|
2020-06-04 23:46:23 +00:00
|
|
|
pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early);
|
2015-10-12 15:52:58 +00:00
|
|
|
|
|
|
|
|
do {
|
|
|
|
|
next = pud_addr_end(addr, end);
|
2018-02-15 11:14:56 +00:00
|
|
|
kasan_pmd_populate(pudp, addr, next, node, early);
|
|
|
|
|
} while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp)));
|
2015-10-12 15:52:58 +00:00
|
|
|
}
|
|
|
|
|
|
2020-06-04 23:46:23 +00:00
|
|
|
static void __init kasan_p4d_populate(pgd_t *pgdp, unsigned long addr,
|
|
|
|
|
unsigned long end, int node, bool early)
|
|
|
|
|
{
|
|
|
|
|
unsigned long next;
|
|
|
|
|
p4d_t *p4dp = p4d_offset(pgdp, addr);
|
|
|
|
|
|
|
|
|
|
do {
|
|
|
|
|
next = p4d_addr_end(addr, end);
|
|
|
|
|
kasan_pud_populate(p4dp, addr, next, node, early);
|
|
|
|
|
} while (p4dp++, addr = next, addr != end);
|
|
|
|
|
}
|
|
|
|
|
|
2017-11-16 01:36:40 +00:00
|
|
|
static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
|
|
|
|
|
int node, bool early)
|
2015-10-12 15:52:58 +00:00
|
|
|
{
|
|
|
|
|
unsigned long next;
|
2018-02-15 11:14:56 +00:00
|
|
|
pgd_t *pgdp;
|
2015-10-12 15:52:58 +00:00
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
pgdp = pgd_offset_k(addr);
|
2015-10-12 15:52:58 +00:00
|
|
|
do {
|
|
|
|
|
next = pgd_addr_end(addr, end);
|
2020-06-04 23:46:23 +00:00
|
|
|
kasan_p4d_populate(pgdp, addr, next, node, early);
|
2018-02-15 11:14:56 +00:00
|
|
|
} while (pgdp++, addr = next, addr != end);
|
2015-10-12 15:52:58 +00:00
|
|
|
}
|
|
|
|
|
|
2017-11-16 01:36:40 +00:00
|
|
|
/* The early shadow maps everything to a single page of zeroes */
|
2015-10-13 13:01:06 +00:00
|
|
|
asmlinkage void __init kasan_early_init(void)
|
2015-10-12 15:52:58 +00:00
|
|
|
{
|
2018-02-06 23:36:44 +00:00
|
|
|
BUILD_BUG_ON(KASAN_SHADOW_OFFSET !=
|
|
|
|
|
KASAN_SHADOW_END - (1UL << (64 - KASAN_SHADOW_SCALE_SHIFT)));
|
2019-08-07 15:55:17 +00:00
|
|
|
BUILD_BUG_ON(!IS_ALIGNED(_KASAN_SHADOW_START(VA_BITS), PGDIR_SIZE));
|
|
|
|
|
BUILD_BUG_ON(!IS_ALIGNED(_KASAN_SHADOW_START(VA_BITS_MIN), PGDIR_SIZE));
|
2015-10-12 15:52:58 +00:00
|
|
|
BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE));
|
2017-11-16 01:36:40 +00:00
|
|
|
kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE,
|
|
|
|
|
true);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Set up full kasan mappings, ensuring that the mapped pages are zeroed */
|
|
|
|
|
static void __init kasan_map_populate(unsigned long start, unsigned long end,
|
|
|
|
|
int node)
|
|
|
|
|
{
|
|
|
|
|
kasan_pgd_populate(start & PAGE_MASK, PAGE_ALIGN(end), node, false);
|
2015-10-12 15:52:58 +00:00
|
|
|
}
|
|
|
|
|
|
arm64: mm: create new fine-grained mappings at boot
At boot we may change the granularity of the tables mapping the kernel
(by splitting or making sections). This may happen when we create the
linear mapping (in __map_memblock), or at any point we try to apply
fine-grained permissions to the kernel (e.g. fixup_executable,
mark_rodata_ro, fixup_init).
Changing the active page tables in this manner may result in multiple
entries for the same address being allocated into TLBs, risking problems
such as TLB conflict aborts or issues derived from the amalgamation of
TLB entries. Generally, a break-before-make (BBM) approach is necessary
to avoid conflicts, but we cannot do this for the kernel tables as it
risks unmapping text or data being used to do so.
Instead, we can create a new set of tables from scratch in the safety of
the existing mappings, and subsequently migrate over to these using the
new cpu_replace_ttbr1 helper, which avoids the two sets of tables being
active simultaneously.
To avoid issues when we later modify permissions of the page tables
(e.g. in fixup_init), we must create the page tables at a granularity
such that later modification does not result in splitting of tables.
This patch applies this strategy, creating a new set of fine-grained
page tables from scratch, and safely migrating to them. The existing
fixmap and kasan shadow page tables are reused in the new fine-grained
tables.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Tested-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: Jeremy Linton <jeremy.linton@arm.com>
Cc: Laura Abbott <labbott@fedoraproject.org>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-01-25 11:45:12 +00:00
|
|
|
/*
|
|
|
|
|
* Copy the current shadow region into a new pgdir.
|
|
|
|
|
*/
|
|
|
|
|
void __init kasan_copy_shadow(pgd_t *pgdir)
|
|
|
|
|
{
|
2018-02-15 11:14:56 +00:00
|
|
|
pgd_t *pgdp, *pgdp_new, *pgdp_end;
|
arm64: mm: create new fine-grained mappings at boot
At boot we may change the granularity of the tables mapping the kernel
(by splitting or making sections). This may happen when we create the
linear mapping (in __map_memblock), or at any point we try to apply
fine-grained permissions to the kernel (e.g. fixup_executable,
mark_rodata_ro, fixup_init).
Changing the active page tables in this manner may result in multiple
entries for the same address being allocated into TLBs, risking problems
such as TLB conflict aborts or issues derived from the amalgamation of
TLB entries. Generally, a break-before-make (BBM) approach is necessary
to avoid conflicts, but we cannot do this for the kernel tables as it
risks unmapping text or data being used to do so.
Instead, we can create a new set of tables from scratch in the safety of
the existing mappings, and subsequently migrate over to these using the
new cpu_replace_ttbr1 helper, which avoids the two sets of tables being
active simultaneously.
To avoid issues when we later modify permissions of the page tables
(e.g. in fixup_init), we must create the page tables at a granularity
such that later modification does not result in splitting of tables.
This patch applies this strategy, creating a new set of fine-grained
page tables from scratch, and safely migrating to them. The existing
fixmap and kasan shadow page tables are reused in the new fine-grained
tables.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Tested-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: Jeremy Linton <jeremy.linton@arm.com>
Cc: Laura Abbott <labbott@fedoraproject.org>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-01-25 11:45:12 +00:00
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
pgdp = pgd_offset_k(KASAN_SHADOW_START);
|
|
|
|
|
pgdp_end = pgd_offset_k(KASAN_SHADOW_END);
|
2020-06-09 04:33:10 +00:00
|
|
|
pgdp_new = pgd_offset_pgd(pgdir, KASAN_SHADOW_START);
|
arm64: mm: create new fine-grained mappings at boot
At boot we may change the granularity of the tables mapping the kernel
(by splitting or making sections). This may happen when we create the
linear mapping (in __map_memblock), or at any point we try to apply
fine-grained permissions to the kernel (e.g. fixup_executable,
mark_rodata_ro, fixup_init).
Changing the active page tables in this manner may result in multiple
entries for the same address being allocated into TLBs, risking problems
such as TLB conflict aborts or issues derived from the amalgamation of
TLB entries. Generally, a break-before-make (BBM) approach is necessary
to avoid conflicts, but we cannot do this for the kernel tables as it
risks unmapping text or data being used to do so.
Instead, we can create a new set of tables from scratch in the safety of
the existing mappings, and subsequently migrate over to these using the
new cpu_replace_ttbr1 helper, which avoids the two sets of tables being
active simultaneously.
To avoid issues when we later modify permissions of the page tables
(e.g. in fixup_init), we must create the page tables at a granularity
such that later modification does not result in splitting of tables.
This patch applies this strategy, creating a new set of fine-grained
page tables from scratch, and safely migrating to them. The existing
fixmap and kasan shadow page tables are reused in the new fine-grained
tables.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Tested-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: Jeremy Linton <jeremy.linton@arm.com>
Cc: Laura Abbott <labbott@fedoraproject.org>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-01-25 11:45:12 +00:00
|
|
|
do {
|
2018-02-15 11:14:56 +00:00
|
|
|
set_pgd(pgdp_new, READ_ONCE(*pgdp));
|
|
|
|
|
} while (pgdp++, pgdp_new++, pgdp != pgdp_end);
|
arm64: mm: create new fine-grained mappings at boot
At boot we may change the granularity of the tables mapping the kernel
(by splitting or making sections). This may happen when we create the
linear mapping (in __map_memblock), or at any point we try to apply
fine-grained permissions to the kernel (e.g. fixup_executable,
mark_rodata_ro, fixup_init).
Changing the active page tables in this manner may result in multiple
entries for the same address being allocated into TLBs, risking problems
such as TLB conflict aborts or issues derived from the amalgamation of
TLB entries. Generally, a break-before-make (BBM) approach is necessary
to avoid conflicts, but we cannot do this for the kernel tables as it
risks unmapping text or data being used to do so.
Instead, we can create a new set of tables from scratch in the safety of
the existing mappings, and subsequently migrate over to these using the
new cpu_replace_ttbr1 helper, which avoids the two sets of tables being
active simultaneously.
To avoid issues when we later modify permissions of the page tables
(e.g. in fixup_init), we must create the page tables at a granularity
such that later modification does not result in splitting of tables.
This patch applies this strategy, creating a new set of fine-grained
page tables from scratch, and safely migrating to them. The existing
fixmap and kasan shadow page tables are reused in the new fine-grained
tables.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Tested-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: Jeremy Linton <jeremy.linton@arm.com>
Cc: Laura Abbott <labbott@fedoraproject.org>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-01-25 11:45:12 +00:00
|
|
|
}
|
|
|
|
|
|
2015-10-12 15:52:58 +00:00
|
|
|
static void __init clear_pgds(unsigned long start,
|
|
|
|
|
unsigned long end)
|
|
|
|
|
{
|
|
|
|
|
/*
|
|
|
|
|
* Remove references to kasan page tables from
|
|
|
|
|
* swapper_pg_dir. pgd_clear() can't be used
|
|
|
|
|
* here because it's nop on 2,3-level pagetable setups
|
|
|
|
|
*/
|
|
|
|
|
for (; start < end; start += PGDIR_SIZE)
|
|
|
|
|
set_pgd(pgd_offset_k(start), __pgd(0));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void __init kasan_init(void)
|
|
|
|
|
{
|
2016-02-16 12:52:40 +00:00
|
|
|
u64 kimg_shadow_start, kimg_shadow_end;
|
arm64: add support for kernel ASLR
This adds support for KASLR is implemented, based on entropy provided by
the bootloader in the /chosen/kaslr-seed DT property. Depending on the size
of the address space (VA_BITS) and the page size, the entropy in the
virtual displacement is up to 13 bits (16k/2 levels) and up to 25 bits (all
4 levels), with the sidenote that displacements that result in the kernel
image straddling a 1GB/32MB/512MB alignment boundary (for 4KB/16KB/64KB
granule kernels, respectively) are not allowed, and will be rounded up to
an acceptable value.
If CONFIG_RANDOMIZE_MODULE_REGION_FULL is enabled, the module region is
randomized independently from the core kernel. This makes it less likely
that the location of core kernel data structures can be determined by an
adversary, but causes all function calls from modules into the core kernel
to be resolved via entries in the module PLTs.
If CONFIG_RANDOMIZE_MODULE_REGION_FULL is not enabled, the module region is
randomized by choosing a page aligned 128 MB region inside the interval
[_etext - 128 MB, _stext + 128 MB). This gives between 10 and 14 bits of
entropy (depending on page size), independently of the kernel randomization,
but still guarantees that modules are within the range of relative branch
and jump instructions (with the caveat that, since the module region is
shared with other uses of the vmalloc area, modules may need to be loaded
further away if the module region is exhausted)
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-01-26 13:12:01 +00:00
|
|
|
u64 mod_shadow_start, mod_shadow_end;
|
2020-10-13 23:58:08 +00:00
|
|
|
phys_addr_t pa_start, pa_end;
|
|
|
|
|
u64 i;
|
2015-10-12 15:52:58 +00:00
|
|
|
|
2017-11-16 01:36:40 +00:00
|
|
|
kimg_shadow_start = (u64)kasan_mem_to_shadow(_text) & PAGE_MASK;
|
|
|
|
|
kimg_shadow_end = PAGE_ALIGN((u64)kasan_mem_to_shadow(_end));
|
2016-02-16 12:52:40 +00:00
|
|
|
|
arm64: add support for kernel ASLR
This adds support for KASLR is implemented, based on entropy provided by
the bootloader in the /chosen/kaslr-seed DT property. Depending on the size
of the address space (VA_BITS) and the page size, the entropy in the
virtual displacement is up to 13 bits (16k/2 levels) and up to 25 bits (all
4 levels), with the sidenote that displacements that result in the kernel
image straddling a 1GB/32MB/512MB alignment boundary (for 4KB/16KB/64KB
granule kernels, respectively) are not allowed, and will be rounded up to
an acceptable value.
If CONFIG_RANDOMIZE_MODULE_REGION_FULL is enabled, the module region is
randomized independently from the core kernel. This makes it less likely
that the location of core kernel data structures can be determined by an
adversary, but causes all function calls from modules into the core kernel
to be resolved via entries in the module PLTs.
If CONFIG_RANDOMIZE_MODULE_REGION_FULL is not enabled, the module region is
randomized by choosing a page aligned 128 MB region inside the interval
[_etext - 128 MB, _stext + 128 MB). This gives between 10 and 14 bits of
entropy (depending on page size), independently of the kernel randomization,
but still guarantees that modules are within the range of relative branch
and jump instructions (with the caveat that, since the module region is
shared with other uses of the vmalloc area, modules may need to be loaded
further away if the module region is exhausted)
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-01-26 13:12:01 +00:00
|
|
|
mod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR);
|
|
|
|
|
mod_shadow_end = (u64)kasan_mem_to_shadow((void *)MODULES_END);
|
|
|
|
|
|
2015-10-12 15:52:58 +00:00
|
|
|
/*
|
|
|
|
|
* We are going to perform proper setup of shadow memory.
|
2018-10-04 16:06:46 +00:00
|
|
|
* At first we should unmap early shadow (clear_pgds() call below).
|
2015-10-12 15:52:58 +00:00
|
|
|
* However, instrumented code couldn't execute without shadow memory.
|
|
|
|
|
* tmp_pg_dir used to keep early shadow mapped until full shadow
|
|
|
|
|
* setup will be finished.
|
|
|
|
|
*/
|
|
|
|
|
memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir));
|
2016-01-25 11:45:02 +00:00
|
|
|
dsb(ishst);
|
2017-01-10 21:35:49 +00:00
|
|
|
cpu_replace_ttbr1(lm_alias(tmp_pg_dir));
|
2015-10-12 15:52:58 +00:00
|
|
|
|
|
|
|
|
clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
|
|
|
|
|
|
2017-11-16 01:36:40 +00:00
|
|
|
kasan_map_populate(kimg_shadow_start, kimg_shadow_end,
|
arm64: kasan: avoid pfn_to_nid() before page array is initialized
In arm64's kasan_init(), we use pfn_to_nid() to find the NUMA node a
span of memory is in, hoping to allocate shadow from the same NUMA node.
However, at this point, the page array has not been initialized, and
thus this is bogus.
Since commit:
f165b378bbdf6c8a ("mm: uninitialized struct page poisoning sanity")
... accessing fields of the page array results in a boot time Oops(),
highlighting this problem:
[ 0.000000] Unable to handle kernel paging request at virtual address dfff200000000000
[ 0.000000] Mem abort info:
[ 0.000000] ESR = 0x96000004
[ 0.000000] Exception class = DABT (current EL), IL = 32 bits
[ 0.000000] SET = 0, FnV = 0
[ 0.000000] EA = 0, S1PTW = 0
[ 0.000000] Data abort info:
[ 0.000000] ISV = 0, ISS = 0x00000004
[ 0.000000] CM = 0, WnR = 0
[ 0.000000] [dfff200000000000] address between user and kernel address ranges
[ 0.000000] Internal error: Oops: 96000004 [#1] PREEMPT SMP
[ 0.000000] Modules linked in:
[ 0.000000] CPU: 0 PID: 0 Comm: swapper Not tainted 4.16.0-07317-gf165b378bbdf #42
[ 0.000000] Hardware name: ARM Juno development board (r1) (DT)
[ 0.000000] pstate: 80000085 (Nzcv daIf -PAN -UAO)
[ 0.000000] pc : __asan_load8+0x8c/0xa8
[ 0.000000] lr : __dump_page+0x3c/0x3b8
[ 0.000000] sp : ffff2000099b7ca0
[ 0.000000] x29: ffff2000099b7ca0 x28: ffff20000a1762c0
[ 0.000000] x27: ffff7e0000000000 x26: ffff2000099dd000
[ 0.000000] x25: ffff200009a3f960 x24: ffff200008f9c38c
[ 0.000000] x23: ffff20000a9d3000 x22: ffff200009735430
[ 0.000000] x21: fffffffffffffffe x20: ffff7e0001e50420
[ 0.000000] x19: ffff7e0001e50400 x18: 0000000000001840
[ 0.000000] x17: ffffffffffff8270 x16: 0000000000001840
[ 0.000000] x15: 0000000000001920 x14: 0000000000000004
[ 0.000000] x13: 0000000000000000 x12: 0000000000000800
[ 0.000000] x11: 1ffff0012d0f89ff x10: ffff10012d0f89ff
[ 0.000000] x9 : 0000000000000000 x8 : ffff8009687c5000
[ 0.000000] x7 : 0000000000000000 x6 : ffff10000f282000
[ 0.000000] x5 : 0000000000000040 x4 : fffffffffffffffe
[ 0.000000] x3 : 0000000000000000 x2 : dfff200000000000
[ 0.000000] x1 : 0000000000000005 x0 : 0000000000000000
[ 0.000000] Process swapper (pid: 0, stack limit = 0x (ptrval))
[ 0.000000] Call trace:
[ 0.000000] __asan_load8+0x8c/0xa8
[ 0.000000] __dump_page+0x3c/0x3b8
[ 0.000000] dump_page+0xc/0x18
[ 0.000000] kasan_init+0x2e8/0x5a8
[ 0.000000] setup_arch+0x294/0x71c
[ 0.000000] start_kernel+0xdc/0x500
[ 0.000000] Code: aa0403e0 9400063c 17ffffee d343fc00 (38e26800)
[ 0.000000] ---[ end trace 67064f0e9c0cc338 ]---
[ 0.000000] Kernel panic - not syncing: Attempted to kill the idle task!
[ 0.000000] ---[ end Kernel panic - not syncing: Attempted to kill the idle task! ]---
Let's fix this by using early_pfn_to_nid(), as other architectures do in
their kasan init code. Note that early_pfn_to_nid acquires the nid from
the memblock array, which we iterate over in kasan_init(), so this
should be fine.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Fixes: 39d114ddc6822302 ("arm64: add KASAN support")
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-04-16 13:44:41 +00:00
|
|
|
early_pfn_to_nid(virt_to_pfn(lm_alias(_text))));
|
2016-02-16 12:52:40 +00:00
|
|
|
|
2019-08-14 13:28:48 +00:00
|
|
|
kasan_populate_early_shadow(kasan_mem_to_shadow((void *)PAGE_END),
|
2019-08-07 15:55:14 +00:00
|
|
|
(void *)mod_shadow_start);
|
2018-12-28 08:30:01 +00:00
|
|
|
kasan_populate_early_shadow((void *)kimg_shadow_end,
|
2019-08-07 15:55:14 +00:00
|
|
|
(void *)KASAN_SHADOW_END);
|
arm64: add support for kernel ASLR
This adds support for KASLR is implemented, based on entropy provided by
the bootloader in the /chosen/kaslr-seed DT property. Depending on the size
of the address space (VA_BITS) and the page size, the entropy in the
virtual displacement is up to 13 bits (16k/2 levels) and up to 25 bits (all
4 levels), with the sidenote that displacements that result in the kernel
image straddling a 1GB/32MB/512MB alignment boundary (for 4KB/16KB/64KB
granule kernels, respectively) are not allowed, and will be rounded up to
an acceptable value.
If CONFIG_RANDOMIZE_MODULE_REGION_FULL is enabled, the module region is
randomized independently from the core kernel. This makes it less likely
that the location of core kernel data structures can be determined by an
adversary, but causes all function calls from modules into the core kernel
to be resolved via entries in the module PLTs.
If CONFIG_RANDOMIZE_MODULE_REGION_FULL is not enabled, the module region is
randomized by choosing a page aligned 128 MB region inside the interval
[_etext - 128 MB, _stext + 128 MB). This gives between 10 and 14 bits of
entropy (depending on page size), independently of the kernel randomization,
but still guarantees that modules are within the range of relative branch
and jump instructions (with the caveat that, since the module region is
shared with other uses of the vmalloc area, modules may need to be loaded
further away if the module region is exhausted)
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-01-26 13:12:01 +00:00
|
|
|
|
|
|
|
|
if (kimg_shadow_start > mod_shadow_end)
|
2018-12-28 08:30:01 +00:00
|
|
|
kasan_populate_early_shadow((void *)mod_shadow_end,
|
|
|
|
|
(void *)kimg_shadow_start);
|
2015-10-12 15:52:58 +00:00
|
|
|
|
2020-10-13 23:58:08 +00:00
|
|
|
for_each_mem_range(i, &pa_start, &pa_end) {
|
|
|
|
|
void *start = (void *)__phys_to_virt(pa_start);
|
|
|
|
|
void *end = (void *)__phys_to_virt(pa_end);
|
2015-10-12 15:52:58 +00:00
|
|
|
|
|
|
|
|
if (start >= end)
|
|
|
|
|
break;
|
|
|
|
|
|
2017-11-16 01:36:40 +00:00
|
|
|
kasan_map_populate((unsigned long)kasan_mem_to_shadow(start),
|
|
|
|
|
(unsigned long)kasan_mem_to_shadow(end),
|
arm64: kasan: avoid pfn_to_nid() before page array is initialized
In arm64's kasan_init(), we use pfn_to_nid() to find the NUMA node a
span of memory is in, hoping to allocate shadow from the same NUMA node.
However, at this point, the page array has not been initialized, and
thus this is bogus.
Since commit:
f165b378bbdf6c8a ("mm: uninitialized struct page poisoning sanity")
... accessing fields of the page array results in a boot time Oops(),
highlighting this problem:
[ 0.000000] Unable to handle kernel paging request at virtual address dfff200000000000
[ 0.000000] Mem abort info:
[ 0.000000] ESR = 0x96000004
[ 0.000000] Exception class = DABT (current EL), IL = 32 bits
[ 0.000000] SET = 0, FnV = 0
[ 0.000000] EA = 0, S1PTW = 0
[ 0.000000] Data abort info:
[ 0.000000] ISV = 0, ISS = 0x00000004
[ 0.000000] CM = 0, WnR = 0
[ 0.000000] [dfff200000000000] address between user and kernel address ranges
[ 0.000000] Internal error: Oops: 96000004 [#1] PREEMPT SMP
[ 0.000000] Modules linked in:
[ 0.000000] CPU: 0 PID: 0 Comm: swapper Not tainted 4.16.0-07317-gf165b378bbdf #42
[ 0.000000] Hardware name: ARM Juno development board (r1) (DT)
[ 0.000000] pstate: 80000085 (Nzcv daIf -PAN -UAO)
[ 0.000000] pc : __asan_load8+0x8c/0xa8
[ 0.000000] lr : __dump_page+0x3c/0x3b8
[ 0.000000] sp : ffff2000099b7ca0
[ 0.000000] x29: ffff2000099b7ca0 x28: ffff20000a1762c0
[ 0.000000] x27: ffff7e0000000000 x26: ffff2000099dd000
[ 0.000000] x25: ffff200009a3f960 x24: ffff200008f9c38c
[ 0.000000] x23: ffff20000a9d3000 x22: ffff200009735430
[ 0.000000] x21: fffffffffffffffe x20: ffff7e0001e50420
[ 0.000000] x19: ffff7e0001e50400 x18: 0000000000001840
[ 0.000000] x17: ffffffffffff8270 x16: 0000000000001840
[ 0.000000] x15: 0000000000001920 x14: 0000000000000004
[ 0.000000] x13: 0000000000000000 x12: 0000000000000800
[ 0.000000] x11: 1ffff0012d0f89ff x10: ffff10012d0f89ff
[ 0.000000] x9 : 0000000000000000 x8 : ffff8009687c5000
[ 0.000000] x7 : 0000000000000000 x6 : ffff10000f282000
[ 0.000000] x5 : 0000000000000040 x4 : fffffffffffffffe
[ 0.000000] x3 : 0000000000000000 x2 : dfff200000000000
[ 0.000000] x1 : 0000000000000005 x0 : 0000000000000000
[ 0.000000] Process swapper (pid: 0, stack limit = 0x (ptrval))
[ 0.000000] Call trace:
[ 0.000000] __asan_load8+0x8c/0xa8
[ 0.000000] __dump_page+0x3c/0x3b8
[ 0.000000] dump_page+0xc/0x18
[ 0.000000] kasan_init+0x2e8/0x5a8
[ 0.000000] setup_arch+0x294/0x71c
[ 0.000000] start_kernel+0xdc/0x500
[ 0.000000] Code: aa0403e0 9400063c 17ffffee d343fc00 (38e26800)
[ 0.000000] ---[ end trace 67064f0e9c0cc338 ]---
[ 0.000000] Kernel panic - not syncing: Attempted to kill the idle task!
[ 0.000000] ---[ end Kernel panic - not syncing: Attempted to kill the idle task! ]---
Let's fix this by using early_pfn_to_nid(), as other architectures do in
their kasan init code. Note that early_pfn_to_nid acquires the nid from
the memblock array, which we iterate over in kasan_init(), so this
should be fine.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Fixes: 39d114ddc6822302 ("arm64: add KASAN support")
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-04-16 13:44:41 +00:00
|
|
|
early_pfn_to_nid(virt_to_pfn(start)));
|
2015-10-12 15:52:58 +00:00
|
|
|
}
|
|
|
|
|
|
2016-01-11 13:50:21 +00:00
|
|
|
/*
|
2018-12-28 08:30:01 +00:00
|
|
|
* KAsan may reuse the contents of kasan_early_shadow_pte directly,
|
|
|
|
|
* so we should make sure that it maps the zero page read-only.
|
2016-01-11 13:50:21 +00:00
|
|
|
*/
|
|
|
|
|
for (i = 0; i < PTRS_PER_PTE; i++)
|
2018-12-28 08:30:01 +00:00
|
|
|
set_pte(&kasan_early_shadow_pte[i],
|
|
|
|
|
pfn_pte(sym_to_pfn(kasan_early_shadow_page),
|
|
|
|
|
PAGE_KERNEL_RO));
|
2016-01-11 13:50:21 +00:00
|
|
|
|
2018-12-28 08:30:09 +00:00
|
|
|
memset(kasan_early_shadow_page, KASAN_SHADOW_INIT, PAGE_SIZE);
|
2017-01-10 21:35:49 +00:00
|
|
|
cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
|
2015-10-12 15:52:58 +00:00
|
|
|
|
|
|
|
|
/* At this point kasan is fully initialized. Enable error messages */
|
|
|
|
|
init_task.kasan_depth = 0;
|
|
|
|
|
pr_info("KernelAddressSanitizer initialized\n");
|
|
|
|
|
}
|
