Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-12-03 17:41:22 +00:00
Code Issues Packages Projects Releases Wiki Activity
dbae2736fd
linux/arch/s390/mm/kasan_init.c
Vasily Gorbik c360c9a238 s390/kasan: support protvirt with 4-level paging 
Currently the kernel crashes in Kasan instrumentation code if
CONFIG_KASAN_S390_4_LEVEL_PAGING is used on protected virtualization
capable machine where the ultravisor imposes addressing limitations on
the host and those limitations are lower then KASAN_SHADOW_OFFSET.

The problem is that Kasan has to know in advance where vmalloc/modules
areas would be. With protected virtualization enabled vmalloc/modules
areas are moved down to the ultravisor secure storage limit while kasan
still expects them at the very end of 4-level paging address space.

To fix that make Kasan recognize when protected virtualization is enabled
and predefine vmalloc/modules areas position which are compliant with
ultravisor secure storage limit.

Kasan shadow itself stays in place and might reside above that ultravisor
secure storage limit.

One slight difference compaired to a kernel without Kasan enabled is that
vmalloc/modules areas position is not reverted to default if ultravisor
initialization fails. It would still be below the ultravisor secure
storage limit.

Kernel layout with kasan, 4-level paging and protected virtualization
enabled (ultravisor secure storage limit is at 0x0000800000000000):
---[ vmemmap Area Start ]---
0x0000400000000000-0x0000400080000000
---[ vmemmap Area End ]---
---[ vmalloc Area Start ]---
0x00007fe000000000-0x00007fff80000000
---[ vmalloc Area End ]---
---[ Modules Area Start ]---
0x00007fff80000000-0x0000800000000000
---[ Modules Area End ]---
---[ Kasan Shadow Start ]---
0x0018000000000000-0x001c000000000000
---[ Kasan Shadow End ]---
0x001c000000000000-0x0020000000000000         1P PGD I

Kernel layout with kasan, 4-level paging and protected virtualization
disabled/unsupported:
---[ vmemmap Area Start ]---
0x0000400000000000-0x0000400060000000
---[ vmemmap Area End ]---
---[ Kasan Shadow Start ]---
0x0018000000000000-0x001c000000000000
---[ Kasan Shadow End ]---
---[ vmalloc Area Start ]---
0x001fffe000000000-0x001fffff80000000
---[ vmalloc Area End ]---
---[ Modules Area Start ]---
0x001fffff80000000-0x0020000000000000
---[ Modules Area End ]---

Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
2020-09-16 14:08:48 +02:00

455 lines
13 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0
#include <linux/kasan.h>
#include <linux/sched/task.h>
#include <linux/memblock.h>
#include <linux/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/kasan.h>
#include <asm/mem_detect.h>
#include <asm/processor.h>
#include <asm/sclp.h>
#include <asm/facility.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/uv.h>
unsigned long kasan_vmax;
static unsigned long segment_pos __initdata;
static unsigned long segment_low __initdata;
static unsigned long pgalloc_pos __initdata;
static unsigned long pgalloc_low __initdata;
static unsigned long pgalloc_freeable __initdata;
static bool has_edat __initdata;
static bool has_nx __initdata;
#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
static void __init kasan_early_panic(const char *reason)
{
sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
sclp_early_printk(reason);
disabled_wait();
}
static void * __init kasan_early_alloc_segment(void)
{
segment_pos -= _SEGMENT_SIZE;
if (segment_pos < segment_low)
kasan_early_panic("out of memory during initialisation\n");
return (void *)segment_pos;
}
static void * __init kasan_early_alloc_pages(unsigned int order)
{
pgalloc_pos -= (PAGE_SIZE << order);
if (pgalloc_pos < pgalloc_low)
kasan_early_panic("out of memory during initialisation\n");
return (void *)pgalloc_pos;
}
static void * __init kasan_early_crst_alloc(unsigned long val)
{
unsigned long *table;
table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
if (table)
crst_table_init(table, val);
return table;
}
static pte_t * __init kasan_early_pte_alloc(void)
{
static void *pte_leftover;
pte_t *pte;
BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);
if (!pte_leftover) {
pte_leftover = kasan_early_alloc_pages(0);
pte = pte_leftover + _PAGE_TABLE_SIZE;
} else {
pte = pte_leftover;
pte_leftover = NULL;
}
memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
return pte;
}
enum populate_mode {
POPULATE_ONE2ONE,
POPULATE_MAP,
POPULATE_ZERO_SHADOW,
POPULATE_SHALLOW
};
static void __init kasan_early_vmemmap_populate(unsigned long address,
unsigned long end,
enum populate_mode mode)
{
unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
pgd_t *pg_dir;
p4d_t *p4_dir;
pud_t *pu_dir;
pmd_t *pm_dir;
pte_t *pt_dir;
pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
if (!has_nx)
pgt_prot_zero &= ~_PAGE_NOEXEC;
pgt_prot = pgprot_val(PAGE_KERNEL);
sgt_prot = pgprot_val(SEGMENT_KERNEL);
if (!has_nx || mode == POPULATE_ONE2ONE) {
pgt_prot &= ~_PAGE_NOEXEC;
sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
}
while (address < end) {
pg_dir = pgd_offset_k(address);
if (pgd_none(*pg_dir)) {
if (mode == POPULATE_ZERO_SHADOW &&
IS_ALIGNED(address, PGDIR_SIZE) &&
end - address >= PGDIR_SIZE) {
pgd_populate(&init_mm, pg_dir,
kasan_early_shadow_p4d);
address = (address + PGDIR_SIZE) & PGDIR_MASK;
continue;
}
p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
pgd_populate(&init_mm, pg_dir, p4_dir);
}
if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) &&
mode == POPULATE_SHALLOW) {
address = (address + P4D_SIZE) & P4D_MASK;
continue;
}
p4_dir = p4d_offset(pg_dir, address);
if (p4d_none(*p4_dir)) {
if (mode == POPULATE_ZERO_SHADOW &&
IS_ALIGNED(address, P4D_SIZE) &&
end - address >= P4D_SIZE) {
p4d_populate(&init_mm, p4_dir,
kasan_early_shadow_pud);
address = (address + P4D_SIZE) & P4D_MASK;
continue;
}
pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
p4d_populate(&init_mm, p4_dir, pu_dir);
}
if (!IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) &&
mode == POPULATE_SHALLOW) {
address = (address + PUD_SIZE) & PUD_MASK;
continue;
}
pu_dir = pud_offset(p4_dir, address);
if (pud_none(*pu_dir)) {
if (mode == POPULATE_ZERO_SHADOW &&
IS_ALIGNED(address, PUD_SIZE) &&
end - address >= PUD_SIZE) {
pud_populate(&init_mm, pu_dir,
kasan_early_shadow_pmd);
address = (address + PUD_SIZE) & PUD_MASK;
continue;
}
pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
pud_populate(&init_mm, pu_dir, pm_dir);
}
pm_dir = pmd_offset(pu_dir, address);
if (pmd_none(*pm_dir)) {
if (mode == POPULATE_ZERO_SHADOW &&
IS_ALIGNED(address, PMD_SIZE) &&
end - address >= PMD_SIZE) {
pmd_populate(&init_mm, pm_dir,
kasan_early_shadow_pte);
address = (address + PMD_SIZE) & PMD_MASK;
continue;
}
/* the first megabyte of 1:1 is mapped with 4k pages */
if (has_edat && address && end - address >= PMD_SIZE &&
mode != POPULATE_ZERO_SHADOW) {
void *page;
if (mode == POPULATE_ONE2ONE) {
page = (void *)address;
} else {
page = kasan_early_alloc_segment();
memset(page, 0, _SEGMENT_SIZE);
}
pmd_val(*pm_dir) = __pa(page) | sgt_prot;
address = (address + PMD_SIZE) & PMD_MASK;
continue;
}
pt_dir = kasan_early_pte_alloc();
pmd_populate(&init_mm, pm_dir, pt_dir);
} else if (pmd_large(*pm_dir)) {
address = (address + PMD_SIZE) & PMD_MASK;
continue;
}
pt_dir = pte_offset_kernel(pm_dir, address);
if (pte_none(*pt_dir)) {
void *page;
switch (mode) {
case POPULATE_ONE2ONE:
page = (void *)address;
pte_val(*pt_dir) = __pa(page) | pgt_prot;
break;
case POPULATE_MAP:
page = kasan_early_alloc_pages(0);
memset(page, 0, PAGE_SIZE);
pte_val(*pt_dir) = __pa(page) | pgt_prot;
break;
case POPULATE_ZERO_SHADOW:
page = kasan_early_shadow_page;
pte_val(*pt_dir) = __pa(page) | pgt_prot_zero;
break;
case POPULATE_SHALLOW:
/* should never happen */
break;
}
}
address += PAGE_SIZE;
}
}
static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
{
unsigned long asce_bits;
asce_bits = asce_type | _ASCE_TABLE_LENGTH;
S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) | asce_bits;
S390_lowcore.user_asce = S390_lowcore.kernel_asce;
__ctl_load(S390_lowcore.kernel_asce, 1, 1);
__ctl_load(S390_lowcore.kernel_asce, 7, 7);
__ctl_load(S390_lowcore.kernel_asce, 13, 13);
}
static void __init kasan_enable_dat(void)
{
psw_t psw;
psw.mask = __extract_psw();
psw_bits(psw).dat = 1;
psw_bits(psw).as = PSW_BITS_AS_HOME;
__load_psw_mask(psw.mask);
}
static void __init kasan_early_detect_facilities(void)
{
if (test_facility(8)) {
has_edat = true;
__ctl_set_bit(0, 23);
}
if (!noexec_disabled && test_facility(130)) {
has_nx = true;
__ctl_set_bit(0, 20);
}
}
static bool __init has_uv_sec_stor_limit(void)
{
/*
* keep these conditions in line with setup_uv()
*/
if (!is_prot_virt_host())
return false;
if (is_prot_virt_guest())
return false;
if (!test_facility(158))
return false;
return !!uv_info.max_sec_stor_addr;
}
void __init kasan_early_init(void)
{
unsigned long untracked_mem_end;
unsigned long shadow_alloc_size;
unsigned long vmax_unlimited;
unsigned long initrd_end;
unsigned long asce_type;
unsigned long memsize;
unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
pte_t pte_z;
pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);
kasan_early_detect_facilities();
if (!has_nx)
pgt_prot &= ~_PAGE_NOEXEC;
pte_z = __pte(__pa(kasan_early_shadow_page) | pgt_prot);
memsize = get_mem_detect_end();
if (!memsize)
kasan_early_panic("cannot detect physical memory size\n");
/* respect mem= cmdline parameter */
if (memory_end_set && memsize > memory_end)
memsize = memory_end;
if (IS_ENABLED(CONFIG_CRASH_DUMP) && OLDMEM_BASE)
memsize = min(memsize, OLDMEM_SIZE);
memsize = min(memsize, KASAN_SHADOW_START);
if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING)) {
/* 4 level paging */
BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
crst_table_init((unsigned long *)early_pg_dir,
_REGION2_ENTRY_EMPTY);
untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE;
if (has_uv_sec_stor_limit())
kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr);
asce_type = _ASCE_TYPE_REGION2;
} else {
/* 3 level paging */
BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PUD_SIZE));
BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PUD_SIZE));
crst_table_init((unsigned long *)early_pg_dir,
_REGION3_ENTRY_EMPTY);
untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION2_SIZE;
asce_type = _ASCE_TYPE_REGION3;
}
/* init kasan zero shadow */
crst_table_init((unsigned long *)kasan_early_shadow_p4d,
p4d_val(p4d_z));
crst_table_init((unsigned long *)kasan_early_shadow_pud,
pud_val(pud_z));
crst_table_init((unsigned long *)kasan_early_shadow_pmd,
pmd_val(pmd_z));
memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
pgalloc_low = round_up((unsigned long)_end, _SEGMENT_SIZE);
if (IS_ENABLED(CONFIG_BLK_DEV_INITRD)) {
initrd_end =
round_up(INITRD_START + INITRD_SIZE, _SEGMENT_SIZE);
pgalloc_low = max(pgalloc_low, initrd_end);
}
if (pgalloc_low + shadow_alloc_size > memsize)
kasan_early_panic("out of memory during initialisation\n");
if (has_edat) {
segment_pos = round_down(memsize, _SEGMENT_SIZE);
segment_low = segment_pos - shadow_alloc_size;
pgalloc_pos = segment_low;
} else {
pgalloc_pos = memsize;
}
init_mm.pgd = early_pg_dir;
/*
* Current memory layout:
* +- 0 -------------+ +- shadow start -+
* | 1:1 ram mapping | /| 1/8 ram |
* | | / | |
* +- end of ram ----+ / +----------------+
* | ... gap ... | / | |
* | |/ | kasan |
* +- shadow start --+ | zero |
* | 1/8 addr space | | page |
* +- shadow end -+ | mapping |
* | ... gap ... |\ | (untracked) |
* +- vmalloc area -+ \ | |
* | vmalloc_size | \ | |
* +- modules vaddr -+ \ +----------------+
* | 2Gb | \| unmapped | allocated per module
* +-----------------+ +- shadow end ---+
*
* Current memory layout (KASAN_VMALLOC):
* +- 0 -------------+ +- shadow start -+
* | 1:1 ram mapping | /| 1/8 ram |
* | | / | |
* +- end of ram ----+ / +----------------+
* | ... gap ... | / | kasan |
* | |/ | zero |
* +- shadow start --+ | page |
* | 1/8 addr space | | mapping |
* +- shadow end -+ | (untracked) |
* | ... gap ... |\ | |
* +- vmalloc area -+ \ +- vmalloc area -+
* | vmalloc_size | \ |shallow populate|
* +- modules vaddr -+ \ +- modules area -+
* | 2Gb | \|shallow populate|
* +-----------------+ +- shadow end ---+
*/
/* populate kasan shadow (for identity mapping and zero page mapping) */
kasan_early_vmemmap_populate(__sha(0), __sha(memsize), POPULATE_MAP);
if (IS_ENABLED(CONFIG_MODULES))
untracked_mem_end = kasan_vmax - MODULES_LEN;
if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
untracked_mem_end = kasan_vmax - vmalloc_size - MODULES_LEN;
/* shallowly populate kasan shadow for vmalloc and modules */
kasan_early_vmemmap_populate(__sha(untracked_mem_end),
__sha(kasan_vmax), POPULATE_SHALLOW);
}
/* populate kasan shadow for untracked memory */
kasan_early_vmemmap_populate(__sha(max_physmem_end),
__sha(untracked_mem_end),
POPULATE_ZERO_SHADOW);
kasan_early_vmemmap_populate(__sha(kasan_vmax),
__sha(vmax_unlimited),
POPULATE_ZERO_SHADOW);
/* memory allocated for identity mapping structs will be freed later */
pgalloc_freeable = pgalloc_pos;
/* populate identity mapping */
kasan_early_vmemmap_populate(0, memsize, POPULATE_ONE2ONE);
kasan_set_pgd(early_pg_dir, asce_type);
kasan_enable_dat();
/* enable kasan */
init_task.kasan_depth = 0;
memblock_reserve(pgalloc_pos, memsize - pgalloc_pos);
sclp_early_printk("KernelAddressSanitizer initialized\n");
}
void __init kasan_copy_shadow(pgd_t *pg_dir)
{
/*
* At this point we are still running on early pages setup early_pg_dir,
* while swapper_pg_dir has just been initialized with identity mapping.
* Carry over shadow memory region from early_pg_dir to swapper_pg_dir.
*/
pgd_t *pg_dir_src;
pgd_t *pg_dir_dst;
p4d_t *p4_dir_src;
p4d_t *p4_dir_dst;
pud_t *pu_dir_src;
pud_t *pu_dir_dst;
pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START);
pg_dir_dst = pgd_offset_raw(pg_dir, KASAN_SHADOW_START);
p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START);
p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START);
if (!p4d_folded(*p4_dir_src)) {
/* 4 level paging */
memcpy(p4_dir_dst, p4_dir_src,
(KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t));
return;
}
/* 3 level paging */
pu_dir_src = pud_offset(p4_dir_src, KASAN_SHADOW_START);
pu_dir_dst = pud_offset(p4_dir_dst, KASAN_SHADOW_START);
memcpy(pu_dir_dst, pu_dir_src,
(KASAN_SHADOW_SIZE >> PUD_SHIFT) * sizeof(pud_t));
}
void __init kasan_free_early_identity(void)
{
memblock_free(pgalloc_pos, pgalloc_freeable - pgalloc_pos);
}
Reference in New Issue View Git Blame Copy Permalink