linux/arch/x86/mm/mem_encrypt.c
Kirill A. Shutemov 1cd9c22fee x86/mm/encrypt: Move page table helpers into separate translation unit
There are bunch of functions in mem_encrypt.c that operate on the
identity mapping, which means they want virtual addresses to be equal to
physical one, without PAGE_OFFSET shift.

We also need to avoid paravirtualizaion call there.

Getting this done is tricky. We cannot use usual page table helpers.
It forces us to open-code a lot of things. It makes code ugly and hard
to modify.

We can get it work with the page table helpers, but it requires few
preprocessor tricks. These tricks may have side effects for the rest of
the file.

Let's isolate such functions into own translation unit.

Tested-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20180131135404.40692-2-kirill.shutemov@linux.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-02-13 15:59:48 +01:00

461 lines
12 KiB
C

/*
* AMD Memory Encryption Support
*
* Copyright (C) 2016 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#define DISABLE_BRANCH_PROFILING
#include <linux/linkage.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/dma-direct.h>
#include <linux/swiotlb.h>
#include <linux/mem_encrypt.h>
#include <asm/tlbflush.h>
#include <asm/fixmap.h>
#include <asm/setup.h>
#include <asm/bootparam.h>
#include <asm/set_memory.h>
#include <asm/cacheflush.h>
#include <asm/processor-flags.h>
#include <asm/msr.h>
#include <asm/cmdline.h>
#include "mm_internal.h"
/*
* Since SME related variables are set early in the boot process they must
* reside in the .data section so as not to be zeroed out when the .bss
* section is later cleared.
*/
u64 sme_me_mask __section(.data) = 0;
EXPORT_SYMBOL(sme_me_mask);
DEFINE_STATIC_KEY_FALSE(sev_enable_key);
EXPORT_SYMBOL_GPL(sev_enable_key);
bool sev_enabled __section(.data);
/* Buffer used for early in-place encryption by BSP, no locking needed */
static char sme_early_buffer[PAGE_SIZE] __aligned(PAGE_SIZE);
/*
* This routine does not change the underlying encryption setting of the
* page(s) that map this memory. It assumes that eventually the memory is
* meant to be accessed as either encrypted or decrypted but the contents
* are currently not in the desired state.
*
* This routine follows the steps outlined in the AMD64 Architecture
* Programmer's Manual Volume 2, Section 7.10.8 Encrypt-in-Place.
*/
static void __init __sme_early_enc_dec(resource_size_t paddr,
unsigned long size, bool enc)
{
void *src, *dst;
size_t len;
if (!sme_me_mask)
return;
wbinvd();
/*
* There are limited number of early mapping slots, so map (at most)
* one page at time.
*/
while (size) {
len = min_t(size_t, sizeof(sme_early_buffer), size);
/*
* Create mappings for the current and desired format of
* the memory. Use a write-protected mapping for the source.
*/
src = enc ? early_memremap_decrypted_wp(paddr, len) :
early_memremap_encrypted_wp(paddr, len);
dst = enc ? early_memremap_encrypted(paddr, len) :
early_memremap_decrypted(paddr, len);
/*
* If a mapping can't be obtained to perform the operation,
* then eventual access of that area in the desired mode
* will cause a crash.
*/
BUG_ON(!src || !dst);
/*
* Use a temporary buffer, of cache-line multiple size, to
* avoid data corruption as documented in the APM.
*/
memcpy(sme_early_buffer, src, len);
memcpy(dst, sme_early_buffer, len);
early_memunmap(dst, len);
early_memunmap(src, len);
paddr += len;
size -= len;
}
}
void __init sme_early_encrypt(resource_size_t paddr, unsigned long size)
{
__sme_early_enc_dec(paddr, size, true);
}
void __init sme_early_decrypt(resource_size_t paddr, unsigned long size)
{
__sme_early_enc_dec(paddr, size, false);
}
static void __init __sme_early_map_unmap_mem(void *vaddr, unsigned long size,
bool map)
{
unsigned long paddr = (unsigned long)vaddr - __PAGE_OFFSET;
pmdval_t pmd_flags, pmd;
/* Use early_pmd_flags but remove the encryption mask */
pmd_flags = __sme_clr(early_pmd_flags);
do {
pmd = map ? (paddr & PMD_MASK) + pmd_flags : 0;
__early_make_pgtable((unsigned long)vaddr, pmd);
vaddr += PMD_SIZE;
paddr += PMD_SIZE;
size = (size <= PMD_SIZE) ? 0 : size - PMD_SIZE;
} while (size);
__native_flush_tlb();
}
void __init sme_unmap_bootdata(char *real_mode_data)
{
struct boot_params *boot_data;
unsigned long cmdline_paddr;
if (!sme_active())
return;
/* Get the command line address before unmapping the real_mode_data */
boot_data = (struct boot_params *)real_mode_data;
cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
__sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), false);
if (!cmdline_paddr)
return;
__sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, false);
}
void __init sme_map_bootdata(char *real_mode_data)
{
struct boot_params *boot_data;
unsigned long cmdline_paddr;
if (!sme_active())
return;
__sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), true);
/* Get the command line address after mapping the real_mode_data */
boot_data = (struct boot_params *)real_mode_data;
cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
if (!cmdline_paddr)
return;
__sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, true);
}
void __init sme_early_init(void)
{
unsigned int i;
if (!sme_me_mask)
return;
early_pmd_flags = __sme_set(early_pmd_flags);
__supported_pte_mask = __sme_set(__supported_pte_mask);
/* Update the protection map with memory encryption mask */
for (i = 0; i < ARRAY_SIZE(protection_map); i++)
protection_map[i] = pgprot_encrypted(protection_map[i]);
if (sev_active())
swiotlb_force = SWIOTLB_FORCE;
}
static void *sev_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
gfp_t gfp, unsigned long attrs)
{
unsigned long dma_mask;
unsigned int order;
struct page *page;
void *vaddr = NULL;
dma_mask = dma_alloc_coherent_mask(dev, gfp);
order = get_order(size);
/*
* Memory will be memset to zero after marking decrypted, so don't
* bother clearing it before.
*/
gfp &= ~__GFP_ZERO;
page = alloc_pages_node(dev_to_node(dev), gfp, order);
if (page) {
dma_addr_t addr;
/*
* Since we will be clearing the encryption bit, check the
* mask with it already cleared.
*/
addr = __sme_clr(phys_to_dma(dev, page_to_phys(page)));
if ((addr + size) > dma_mask) {
__free_pages(page, get_order(size));
} else {
vaddr = page_address(page);
*dma_handle = addr;
}
}
if (!vaddr)
vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
if (!vaddr)
return NULL;
/* Clear the SME encryption bit for DMA use if not swiotlb area */
if (!is_swiotlb_buffer(dma_to_phys(dev, *dma_handle))) {
set_memory_decrypted((unsigned long)vaddr, 1 << order);
memset(vaddr, 0, PAGE_SIZE << order);
*dma_handle = __sme_clr(*dma_handle);
}
return vaddr;
}
static void sev_free(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_handle, unsigned long attrs)
{
/* Set the SME encryption bit for re-use if not swiotlb area */
if (!is_swiotlb_buffer(dma_to_phys(dev, dma_handle)))
set_memory_encrypted((unsigned long)vaddr,
1 << get_order(size));
swiotlb_free_coherent(dev, size, vaddr, dma_handle);
}
static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)
{
pgprot_t old_prot, new_prot;
unsigned long pfn, pa, size;
pte_t new_pte;
switch (level) {
case PG_LEVEL_4K:
pfn = pte_pfn(*kpte);
old_prot = pte_pgprot(*kpte);
break;
case PG_LEVEL_2M:
pfn = pmd_pfn(*(pmd_t *)kpte);
old_prot = pmd_pgprot(*(pmd_t *)kpte);
break;
case PG_LEVEL_1G:
pfn = pud_pfn(*(pud_t *)kpte);
old_prot = pud_pgprot(*(pud_t *)kpte);
break;
default:
return;
}
new_prot = old_prot;
if (enc)
pgprot_val(new_prot) |= _PAGE_ENC;
else
pgprot_val(new_prot) &= ~_PAGE_ENC;
/* If prot is same then do nothing. */
if (pgprot_val(old_prot) == pgprot_val(new_prot))
return;
pa = pfn << page_level_shift(level);
size = page_level_size(level);
/*
* We are going to perform in-place en-/decryption and change the
* physical page attribute from C=1 to C=0 or vice versa. Flush the
* caches to ensure that data gets accessed with the correct C-bit.
*/
clflush_cache_range(__va(pa), size);
/* Encrypt/decrypt the contents in-place */
if (enc)
sme_early_encrypt(pa, size);
else
sme_early_decrypt(pa, size);
/* Change the page encryption mask. */
new_pte = pfn_pte(pfn, new_prot);
set_pte_atomic(kpte, new_pte);
}
static int __init early_set_memory_enc_dec(unsigned long vaddr,
unsigned long size, bool enc)
{
unsigned long vaddr_end, vaddr_next;
unsigned long psize, pmask;
int split_page_size_mask;
int level, ret;
pte_t *kpte;
vaddr_next = vaddr;
vaddr_end = vaddr + size;
for (; vaddr < vaddr_end; vaddr = vaddr_next) {
kpte = lookup_address(vaddr, &level);
if (!kpte || pte_none(*kpte)) {
ret = 1;
goto out;
}
if (level == PG_LEVEL_4K) {
__set_clr_pte_enc(kpte, level, enc);
vaddr_next = (vaddr & PAGE_MASK) + PAGE_SIZE;
continue;
}
psize = page_level_size(level);
pmask = page_level_mask(level);
/*
* Check whether we can change the large page in one go.
* We request a split when the address is not aligned and
* the number of pages to set/clear encryption bit is smaller
* than the number of pages in the large page.
*/
if (vaddr == (vaddr & pmask) &&
((vaddr_end - vaddr) >= psize)) {
__set_clr_pte_enc(kpte, level, enc);
vaddr_next = (vaddr & pmask) + psize;
continue;
}
/*
* The virtual address is part of a larger page, create the next
* level page table mapping (4K or 2M). If it is part of a 2M
* page then we request a split of the large page into 4K
* chunks. A 1GB large page is split into 2M pages, resp.
*/
if (level == PG_LEVEL_2M)
split_page_size_mask = 0;
else
split_page_size_mask = 1 << PG_LEVEL_2M;
kernel_physical_mapping_init(__pa(vaddr & pmask),
__pa((vaddr_end & pmask) + psize),
split_page_size_mask);
}
ret = 0;
out:
__flush_tlb_all();
return ret;
}
int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size)
{
return early_set_memory_enc_dec(vaddr, size, false);
}
int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size)
{
return early_set_memory_enc_dec(vaddr, size, true);
}
/*
* SME and SEV are very similar but they are not the same, so there are
* times that the kernel will need to distinguish between SME and SEV. The
* sme_active() and sev_active() functions are used for this. When a
* distinction isn't needed, the mem_encrypt_active() function can be used.
*
* The trampoline code is a good example for this requirement. Before
* paging is activated, SME will access all memory as decrypted, but SEV
* will access all memory as encrypted. So, when APs are being brought
* up under SME the trampoline area cannot be encrypted, whereas under SEV
* the trampoline area must be encrypted.
*/
bool sme_active(void)
{
return sme_me_mask && !sev_enabled;
}
EXPORT_SYMBOL(sme_active);
bool sev_active(void)
{
return sme_me_mask && sev_enabled;
}
EXPORT_SYMBOL(sev_active);
static const struct dma_map_ops sev_dma_ops = {
.alloc = sev_alloc,
.free = sev_free,
.map_page = swiotlb_map_page,
.unmap_page = swiotlb_unmap_page,
.map_sg = swiotlb_map_sg_attrs,
.unmap_sg = swiotlb_unmap_sg_attrs,
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
.sync_single_for_device = swiotlb_sync_single_for_device,
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
.sync_sg_for_device = swiotlb_sync_sg_for_device,
.mapping_error = swiotlb_dma_mapping_error,
};
/* Architecture __weak replacement functions */
void __init mem_encrypt_init(void)
{
if (!sme_me_mask)
return;
/* Call into SWIOTLB to update the SWIOTLB DMA buffers */
swiotlb_update_mem_attributes();
/*
* With SEV, DMA operations cannot use encryption. New DMA ops
* are required in order to mark the DMA areas as decrypted or
* to use bounce buffers.
*/
if (sev_active())
dma_ops = &sev_dma_ops;
/*
* With SEV, we need to unroll the rep string I/O instructions.
*/
if (sev_active())
static_branch_enable(&sev_enable_key);
pr_info("AMD %s active\n",
sev_active() ? "Secure Encrypted Virtualization (SEV)"
: "Secure Memory Encryption (SME)");
}
void swiotlb_set_mem_attributes(void *vaddr, unsigned long size)
{
WARN(PAGE_ALIGN(size) != size,
"size is not page-aligned (%#lx)\n", size);
/* Make the SWIOTLB buffer area decrypted */
set_memory_decrypted((unsigned long)vaddr, size >> PAGE_SHIFT);
}