forked from Minki/linux
198f0f4c58
This makes the default monitoring primitives for virtual address spaces and the physical address sapce to support memory regions prioritization for 'PAGEOUT' DAMOS action. It calculates hotness of each region as weighted sum of 'nr_accesses' and 'age' of the region and get the priority score as reverse of the hotness, so that cold regions can be paged out first. Link: https://lkml.kernel.org/r/20211019150731.16699-9-sj@kernel.org Signed-off-by: SeongJae Park <sj@kernel.org> Cc: Amit Shah <amit@kernel.org> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: David Hildenbrand <david@redhat.com> Cc: David Rientjes <rientjes@google.com> Cc: David Woodhouse <dwmw@amazon.com> Cc: Greg Thelen <gthelen@google.com> Cc: Jonathan Cameron <Jonathan.Cameron@huawei.com> Cc: Jonathan Corbet <corbet@lwn.net> Cc: Leonard Foerster <foersleo@amazon.de> Cc: Marco Elver <elver@google.com> Cc: Markus Boehme <markubo@amazon.de> Cc: Shakeel Butt <shakeelb@google.com> Cc: Shuah Khan <shuah@kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
134 lines
3.1 KiB
C
134 lines
3.1 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Common Primitives for Data Access Monitoring
|
|
*
|
|
* Author: SeongJae Park <sj@kernel.org>
|
|
*/
|
|
|
|
#include <linux/mmu_notifier.h>
|
|
#include <linux/page_idle.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/rmap.h>
|
|
|
|
#include "prmtv-common.h"
|
|
|
|
/*
|
|
* Get an online page for a pfn if it's in the LRU list. Otherwise, returns
|
|
* NULL.
|
|
*
|
|
* The body of this function is stolen from the 'page_idle_get_page()'. We
|
|
* steal rather than reuse it because the code is quite simple.
|
|
*/
|
|
struct page *damon_get_page(unsigned long pfn)
|
|
{
|
|
struct page *page = pfn_to_online_page(pfn);
|
|
|
|
if (!page || !PageLRU(page) || !get_page_unless_zero(page))
|
|
return NULL;
|
|
|
|
if (unlikely(!PageLRU(page))) {
|
|
put_page(page);
|
|
page = NULL;
|
|
}
|
|
return page;
|
|
}
|
|
|
|
void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr)
|
|
{
|
|
bool referenced = false;
|
|
struct page *page = damon_get_page(pte_pfn(*pte));
|
|
|
|
if (!page)
|
|
return;
|
|
|
|
if (pte_young(*pte)) {
|
|
referenced = true;
|
|
*pte = pte_mkold(*pte);
|
|
}
|
|
|
|
#ifdef CONFIG_MMU_NOTIFIER
|
|
if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE))
|
|
referenced = true;
|
|
#endif /* CONFIG_MMU_NOTIFIER */
|
|
|
|
if (referenced)
|
|
set_page_young(page);
|
|
|
|
set_page_idle(page);
|
|
put_page(page);
|
|
}
|
|
|
|
void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr)
|
|
{
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
bool referenced = false;
|
|
struct page *page = damon_get_page(pmd_pfn(*pmd));
|
|
|
|
if (!page)
|
|
return;
|
|
|
|
if (pmd_young(*pmd)) {
|
|
referenced = true;
|
|
*pmd = pmd_mkold(*pmd);
|
|
}
|
|
|
|
#ifdef CONFIG_MMU_NOTIFIER
|
|
if (mmu_notifier_clear_young(mm, addr,
|
|
addr + ((1UL) << HPAGE_PMD_SHIFT)))
|
|
referenced = true;
|
|
#endif /* CONFIG_MMU_NOTIFIER */
|
|
|
|
if (referenced)
|
|
set_page_young(page);
|
|
|
|
set_page_idle(page);
|
|
put_page(page);
|
|
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
|
}
|
|
|
|
#define DAMON_MAX_SUBSCORE (100)
|
|
#define DAMON_MAX_AGE_IN_LOG (32)
|
|
|
|
int damon_pageout_score(struct damon_ctx *c, struct damon_region *r,
|
|
struct damos *s)
|
|
{
|
|
unsigned int max_nr_accesses;
|
|
int freq_subscore;
|
|
unsigned int age_in_sec;
|
|
int age_in_log, age_subscore;
|
|
unsigned int freq_weight = s->quota.weight_nr_accesses;
|
|
unsigned int age_weight = s->quota.weight_age;
|
|
int hotness;
|
|
|
|
max_nr_accesses = c->aggr_interval / c->sample_interval;
|
|
freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
|
|
|
|
age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
|
|
for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
|
|
age_in_log++, age_in_sec >>= 1)
|
|
;
|
|
|
|
/* If frequency is 0, higher age means it's colder */
|
|
if (freq_subscore == 0)
|
|
age_in_log *= -1;
|
|
|
|
/*
|
|
* Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
|
|
* Scale it to be in [0, 100] and set it as age subscore.
|
|
*/
|
|
age_in_log += DAMON_MAX_AGE_IN_LOG;
|
|
age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
|
|
DAMON_MAX_AGE_IN_LOG / 2;
|
|
|
|
hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
|
|
if (freq_weight + age_weight)
|
|
hotness /= freq_weight + age_weight;
|
|
/*
|
|
* Transform it to fit in [0, DAMOS_MAX_SCORE]
|
|
*/
|
|
hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
|
|
|
|
/* Return coldness of the region */
|
|
return DAMOS_MAX_SCORE - hotness;
|
|
}
|