diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 193ed34a2780..7d8f642c498d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -991,12 +991,65 @@ out: return ret; } -static bool free_pages_prepare(struct page *page, unsigned int order); +static __always_inline bool free_pages_prepare(struct page *page, + unsigned int order, bool check_free) +{ + int bad = 0; + + VM_BUG_ON_PAGE(PageTail(page), page); + + trace_mm_page_free(page, order); + kmemcheck_free_shadow(page, order); + kasan_free_pages(page, order); + + /* + * Check tail pages before head page information is cleared to + * avoid checking PageCompound for order-0 pages. + */ + if (unlikely(order)) { + bool compound = PageCompound(page); + int i; + + VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); + + for (i = 1; i < (1 << order); i++) { + if (compound) + bad += free_tail_pages_check(page, page + i); + if (unlikely(free_pages_check(page + i))) { + bad++; + continue; + } + (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; + } + } + if (PageAnonHead(page)) + page->mapping = NULL; + if (check_free) + bad += free_pages_check(page); + if (bad) + return false; + + page_cpupid_reset_last(page); + page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; + reset_page_owner(page, order); + + if (!PageHighMem(page)) { + debug_check_no_locks_freed(page_address(page), + PAGE_SIZE << order); + debug_check_no_obj_freed(page_address(page), + PAGE_SIZE << order); + } + arch_free_page(page, order); + kernel_poison_pages(page, 1 << order, 0); + kernel_map_pages(page, 1 << order, 0); + + return true; +} #ifdef CONFIG_DEBUG_VM static inline bool free_pcp_prepare(struct page *page) { - return free_pages_prepare(page, 0); + return free_pages_prepare(page, 0, true); } static inline bool bulkfree_pcp_prepare(struct page *page) @@ -1006,30 +1059,7 @@ static inline bool bulkfree_pcp_prepare(struct page *page) #else static bool free_pcp_prepare(struct page *page) { - VM_BUG_ON_PAGE(PageTail(page), page); - - trace_mm_page_free(page, 0); - kmemcheck_free_shadow(page, 0); - kasan_free_pages(page, 0); - - if (PageAnonHead(page)) - page->mapping = NULL; - - reset_page_owner(page, 0); - - if (!PageHighMem(page)) { - debug_check_no_locks_freed(page_address(page), - PAGE_SIZE); - debug_check_no_obj_freed(page_address(page), - PAGE_SIZE); - } - arch_free_page(page, 0); - kernel_poison_pages(page, 0, 0); - kernel_map_pages(page, 0, 0); - - page_cpupid_reset_last(page); - page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; - return true; + return free_pages_prepare(page, 0, false); } static bool bulkfree_pcp_prepare(struct page *page) @@ -1201,66 +1231,13 @@ void __meminit reserve_bootmem_region(unsigned long start, unsigned long end) } } -static bool free_pages_prepare(struct page *page, unsigned int order) -{ - int bad = 0; - - VM_BUG_ON_PAGE(PageTail(page), page); - - trace_mm_page_free(page, order); - kmemcheck_free_shadow(page, order); - kasan_free_pages(page, order); - - /* - * Check tail pages before head page information is cleared to - * avoid checking PageCompound for order-0 pages. - */ - if (unlikely(order)) { - bool compound = PageCompound(page); - int i; - - VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); - - for (i = 1; i < (1 << order); i++) { - if (compound) - bad += free_tail_pages_check(page, page + i); - if (unlikely(free_pages_check(page + i))) { - bad++; - continue; - } - (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; - } - } - if (PageAnonHead(page)) - page->mapping = NULL; - bad += free_pages_check(page); - if (bad) - return false; - - page_cpupid_reset_last(page); - page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; - reset_page_owner(page, order); - - if (!PageHighMem(page)) { - debug_check_no_locks_freed(page_address(page), - PAGE_SIZE << order); - debug_check_no_obj_freed(page_address(page), - PAGE_SIZE << order); - } - arch_free_page(page, order); - kernel_poison_pages(page, 1 << order, 0); - kernel_map_pages(page, 1 << order, 0); - - return true; -} - static void __free_pages_ok(struct page *page, unsigned int order) { unsigned long flags; int migratetype; unsigned long pfn = page_to_pfn(page); - if (!free_pages_prepare(page, order)) + if (!free_pages_prepare(page, order, true)) return; migratetype = get_pfnblock_migratetype(page, pfn);