2022-01-14 22:06:37 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Copyright (c) 2021, Google LLC.
|
|
|
|
* Pasha Tatashin <pasha.tatashin@soleen.com>
|
|
|
|
*/
|
2022-11-01 21:14:09 +00:00
|
|
|
#include <linux/kstrtox.h>
|
2022-01-14 22:06:37 +00:00
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/page_table_check.h>
|
|
|
|
|
|
|
|
#undef pr_fmt
|
|
|
|
#define pr_fmt(fmt) "page_table_check: " fmt
|
|
|
|
|
|
|
|
struct page_table_check {
|
|
|
|
atomic_t anon_map_count;
|
|
|
|
atomic_t file_map_count;
|
|
|
|
};
|
|
|
|
|
|
|
|
static bool __page_table_check_enabled __initdata =
|
|
|
|
IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
|
|
|
|
|
|
|
|
DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
|
|
|
|
EXPORT_SYMBOL(page_table_check_disabled);
|
|
|
|
|
|
|
|
static int __init early_page_table_check_param(char *buf)
|
|
|
|
{
|
2022-11-01 21:14:09 +00:00
|
|
|
return kstrtobool(buf, &__page_table_check_enabled);
|
2022-01-14 22:06:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
early_param("page_table_check", early_page_table_check_param);
|
|
|
|
|
|
|
|
static bool __init need_page_table_check(void)
|
|
|
|
{
|
|
|
|
return __page_table_check_enabled;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __init init_page_table_check(void)
|
|
|
|
{
|
|
|
|
if (!__page_table_check_enabled)
|
|
|
|
return;
|
|
|
|
static_branch_disable(&page_table_check_disabled);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct page_ext_operations page_table_check_ops = {
|
|
|
|
.size = sizeof(struct page_table_check),
|
|
|
|
.need = need_page_table_check,
|
|
|
|
.init = init_page_table_check,
|
2023-01-13 15:42:53 +00:00
|
|
|
.need_shared_flags = false,
|
2022-01-14 22:06:37 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
|
|
|
|
{
|
|
|
|
BUG_ON(!page_ext);
|
|
|
|
return (void *)(page_ext) + page_table_check_ops.offset;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2022-09-16 09:04:34 +00:00
|
|
|
* An entry is removed from the page table, decrement the counters for that page
|
2022-01-14 22:06:37 +00:00
|
|
|
* verify that it is of correct type and counters do not become negative.
|
|
|
|
*/
|
|
|
|
static void page_table_check_clear(struct mm_struct *mm, unsigned long addr,
|
|
|
|
unsigned long pfn, unsigned long pgcnt)
|
|
|
|
{
|
|
|
|
struct page_ext *page_ext;
|
|
|
|
struct page *page;
|
2022-02-04 04:49:15 +00:00
|
|
|
unsigned long i;
|
2022-01-14 22:06:37 +00:00
|
|
|
bool anon;
|
|
|
|
|
|
|
|
if (!pfn_valid(pfn))
|
|
|
|
return;
|
|
|
|
|
|
|
|
page = pfn_to_page(pfn);
|
2022-08-18 13:50:00 +00:00
|
|
|
page_ext = page_ext_get(page);
|
2022-01-14 22:06:37 +00:00
|
|
|
anon = PageAnon(page);
|
|
|
|
|
|
|
|
for (i = 0; i < pgcnt; i++) {
|
|
|
|
struct page_table_check *ptc = get_page_table_check(page_ext);
|
|
|
|
|
|
|
|
if (anon) {
|
|
|
|
BUG_ON(atomic_read(&ptc->file_map_count));
|
|
|
|
BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
|
|
|
|
} else {
|
|
|
|
BUG_ON(atomic_read(&ptc->anon_map_count));
|
|
|
|
BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
|
|
|
|
}
|
|
|
|
page_ext = page_ext_next(page_ext);
|
|
|
|
}
|
2022-08-18 13:50:00 +00:00
|
|
|
page_ext_put(page_ext);
|
2022-01-14 22:06:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2022-09-16 09:04:34 +00:00
|
|
|
* A new entry is added to the page table, increment the counters for that page
|
2022-01-14 22:06:37 +00:00
|
|
|
* verify that it is of correct type and is not being mapped with a different
|
|
|
|
* type to a different process.
|
|
|
|
*/
|
|
|
|
static void page_table_check_set(struct mm_struct *mm, unsigned long addr,
|
|
|
|
unsigned long pfn, unsigned long pgcnt,
|
|
|
|
bool rw)
|
|
|
|
{
|
|
|
|
struct page_ext *page_ext;
|
|
|
|
struct page *page;
|
2022-02-04 04:49:15 +00:00
|
|
|
unsigned long i;
|
2022-01-14 22:06:37 +00:00
|
|
|
bool anon;
|
|
|
|
|
|
|
|
if (!pfn_valid(pfn))
|
|
|
|
return;
|
|
|
|
|
|
|
|
page = pfn_to_page(pfn);
|
2022-08-18 13:50:00 +00:00
|
|
|
page_ext = page_ext_get(page);
|
2022-01-14 22:06:37 +00:00
|
|
|
anon = PageAnon(page);
|
|
|
|
|
|
|
|
for (i = 0; i < pgcnt; i++) {
|
|
|
|
struct page_table_check *ptc = get_page_table_check(page_ext);
|
|
|
|
|
|
|
|
if (anon) {
|
|
|
|
BUG_ON(atomic_read(&ptc->file_map_count));
|
|
|
|
BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
|
|
|
|
} else {
|
|
|
|
BUG_ON(atomic_read(&ptc->anon_map_count));
|
|
|
|
BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
|
|
|
|
}
|
|
|
|
page_ext = page_ext_next(page_ext);
|
|
|
|
}
|
2022-08-18 13:50:00 +00:00
|
|
|
page_ext_put(page_ext);
|
2022-01-14 22:06:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* page is on free list, or is being allocated, verify that counters are zeroes
|
|
|
|
* crash if they are not.
|
|
|
|
*/
|
|
|
|
void __page_table_check_zero(struct page *page, unsigned int order)
|
|
|
|
{
|
2022-08-18 13:50:00 +00:00
|
|
|
struct page_ext *page_ext;
|
2022-02-04 04:49:15 +00:00
|
|
|
unsigned long i;
|
2022-01-14 22:06:37 +00:00
|
|
|
|
2022-08-18 13:50:00 +00:00
|
|
|
page_ext = page_ext_get(page);
|
2022-01-14 22:06:37 +00:00
|
|
|
BUG_ON(!page_ext);
|
2022-02-04 04:49:15 +00:00
|
|
|
for (i = 0; i < (1ul << order); i++) {
|
2022-01-14 22:06:37 +00:00
|
|
|
struct page_table_check *ptc = get_page_table_check(page_ext);
|
|
|
|
|
|
|
|
BUG_ON(atomic_read(&ptc->anon_map_count));
|
|
|
|
BUG_ON(atomic_read(&ptc->file_map_count));
|
|
|
|
page_ext = page_ext_next(page_ext);
|
|
|
|
}
|
2022-08-18 13:50:00 +00:00
|
|
|
page_ext_put(page_ext);
|
2022-01-14 22:06:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pte_t pte)
|
|
|
|
{
|
|
|
|
if (&init_mm == mm)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (pte_user_accessible_page(pte)) {
|
|
|
|
page_table_check_clear(mm, addr, pte_pfn(pte),
|
|
|
|
PAGE_SIZE >> PAGE_SHIFT);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__page_table_check_pte_clear);
|
|
|
|
|
|
|
|
void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pmd_t pmd)
|
|
|
|
{
|
|
|
|
if (&init_mm == mm)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (pmd_user_accessible_page(pmd)) {
|
|
|
|
page_table_check_clear(mm, addr, pmd_pfn(pmd),
|
2022-05-13 03:23:06 +00:00
|
|
|
PMD_SIZE >> PAGE_SHIFT);
|
2022-01-14 22:06:37 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__page_table_check_pmd_clear);
|
|
|
|
|
|
|
|
void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pud_t pud)
|
|
|
|
{
|
|
|
|
if (&init_mm == mm)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (pud_user_accessible_page(pud)) {
|
|
|
|
page_table_check_clear(mm, addr, pud_pfn(pud),
|
2022-05-13 03:23:06 +00:00
|
|
|
PUD_SIZE >> PAGE_SHIFT);
|
2022-01-14 22:06:37 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__page_table_check_pud_clear);
|
|
|
|
|
|
|
|
void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pte_t *ptep, pte_t pte)
|
|
|
|
{
|
|
|
|
if (&init_mm == mm)
|
|
|
|
return;
|
|
|
|
|
2022-02-04 04:49:15 +00:00
|
|
|
__page_table_check_pte_clear(mm, addr, *ptep);
|
2022-01-14 22:06:37 +00:00
|
|
|
if (pte_user_accessible_page(pte)) {
|
|
|
|
page_table_check_set(mm, addr, pte_pfn(pte),
|
|
|
|
PAGE_SIZE >> PAGE_SHIFT,
|
|
|
|
pte_write(pte));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__page_table_check_pte_set);
|
|
|
|
|
|
|
|
void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pmd_t *pmdp, pmd_t pmd)
|
|
|
|
{
|
|
|
|
if (&init_mm == mm)
|
|
|
|
return;
|
|
|
|
|
2022-02-04 04:49:15 +00:00
|
|
|
__page_table_check_pmd_clear(mm, addr, *pmdp);
|
2022-01-14 22:06:37 +00:00
|
|
|
if (pmd_user_accessible_page(pmd)) {
|
|
|
|
page_table_check_set(mm, addr, pmd_pfn(pmd),
|
2022-05-13 03:23:06 +00:00
|
|
|
PMD_SIZE >> PAGE_SHIFT,
|
2022-01-14 22:06:37 +00:00
|
|
|
pmd_write(pmd));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__page_table_check_pmd_set);
|
|
|
|
|
|
|
|
void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pud_t *pudp, pud_t pud)
|
|
|
|
{
|
|
|
|
if (&init_mm == mm)
|
|
|
|
return;
|
|
|
|
|
2022-02-04 04:49:15 +00:00
|
|
|
__page_table_check_pud_clear(mm, addr, *pudp);
|
2022-01-14 22:06:37 +00:00
|
|
|
if (pud_user_accessible_page(pud)) {
|
|
|
|
page_table_check_set(mm, addr, pud_pfn(pud),
|
2022-05-13 03:23:06 +00:00
|
|
|
PUD_SIZE >> PAGE_SHIFT,
|
2022-01-14 22:06:37 +00:00
|
|
|
pud_write(pud));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__page_table_check_pud_set);
|
2022-02-04 04:49:24 +00:00
|
|
|
|
|
|
|
void __page_table_check_pte_clear_range(struct mm_struct *mm,
|
|
|
|
unsigned long addr,
|
|
|
|
pmd_t pmd)
|
|
|
|
{
|
|
|
|
if (&init_mm == mm)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
|
|
|
|
pte_t *ptep = pte_offset_map(&pmd, addr);
|
|
|
|
unsigned long i;
|
|
|
|
|
|
|
|
for (i = 0; i < PTRS_PER_PTE; i++) {
|
|
|
|
__page_table_check_pte_clear(mm, addr, *ptep);
|
|
|
|
addr += PAGE_SIZE;
|
|
|
|
ptep++;
|
|
|
|
}
|
2022-05-26 11:33:50 +00:00
|
|
|
pte_unmap(ptep - PTRS_PER_PTE);
|
2022-02-04 04:49:24 +00:00
|
|
|
}
|
|
|
|
}
|