mirror of
https://github.com/torvalds/linux.git
synced 2024-12-11 13:41:55 +00:00
7d95f22798
Here is the warning converted as error and reported by GCC:
kernel/static_call.c: In function ‘__static_call_update’:
kernel/static_call.c:153:18: error: unused variable ‘mod’ [-Werror=unused-variable]
153 | struct module *mod = site_mod->mod;
| ^~~
cc1: all warnings being treated as errors
make[1]: *** [scripts/Makefile.build:271: kernel/static_call.o] Error 1
This is simply because since recently, we no longer use 'mod' variable
elsewhere if MODULE is unset.
When using 'make tinyconfig' to generate the default kconfig, MODULE is
unset.
There are different ways to fix this warning. Here I tried to minimised
the number of modified lines and not add more #ifdef. We could also move
the declaration of the 'mod' variable inside the if-statement or
directly use site_mod->mod.
Fixes: 698bacefe9
("static_call: Align static_call_is_init() patching condition")
Signed-off-by: Matthieu Baerts <matthieu.baerts@tessares.net>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20210326105023.2058860-1-matthieu.baerts@tessares.net
546 lines
12 KiB
C
546 lines
12 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#include <linux/init.h>
|
|
#include <linux/static_call.h>
|
|
#include <linux/bug.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/sort.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/module.h>
|
|
#include <linux/cpu.h>
|
|
#include <linux/processor.h>
|
|
#include <asm/sections.h>
|
|
|
|
extern struct static_call_site __start_static_call_sites[],
|
|
__stop_static_call_sites[];
|
|
extern struct static_call_tramp_key __start_static_call_tramp_key[],
|
|
__stop_static_call_tramp_key[];
|
|
|
|
static bool static_call_initialized;
|
|
|
|
/* mutex to protect key modules/sites */
|
|
static DEFINE_MUTEX(static_call_mutex);
|
|
|
|
static void static_call_lock(void)
|
|
{
|
|
mutex_lock(&static_call_mutex);
|
|
}
|
|
|
|
static void static_call_unlock(void)
|
|
{
|
|
mutex_unlock(&static_call_mutex);
|
|
}
|
|
|
|
static inline void *static_call_addr(struct static_call_site *site)
|
|
{
|
|
return (void *)((long)site->addr + (long)&site->addr);
|
|
}
|
|
|
|
static inline unsigned long __static_call_key(const struct static_call_site *site)
|
|
{
|
|
return (long)site->key + (long)&site->key;
|
|
}
|
|
|
|
static inline struct static_call_key *static_call_key(const struct static_call_site *site)
|
|
{
|
|
return (void *)(__static_call_key(site) & ~STATIC_CALL_SITE_FLAGS);
|
|
}
|
|
|
|
/* These assume the key is word-aligned. */
|
|
static inline bool static_call_is_init(struct static_call_site *site)
|
|
{
|
|
return __static_call_key(site) & STATIC_CALL_SITE_INIT;
|
|
}
|
|
|
|
static inline bool static_call_is_tail(struct static_call_site *site)
|
|
{
|
|
return __static_call_key(site) & STATIC_CALL_SITE_TAIL;
|
|
}
|
|
|
|
static inline void static_call_set_init(struct static_call_site *site)
|
|
{
|
|
site->key = (__static_call_key(site) | STATIC_CALL_SITE_INIT) -
|
|
(long)&site->key;
|
|
}
|
|
|
|
static int static_call_site_cmp(const void *_a, const void *_b)
|
|
{
|
|
const struct static_call_site *a = _a;
|
|
const struct static_call_site *b = _b;
|
|
const struct static_call_key *key_a = static_call_key(a);
|
|
const struct static_call_key *key_b = static_call_key(b);
|
|
|
|
if (key_a < key_b)
|
|
return -1;
|
|
|
|
if (key_a > key_b)
|
|
return 1;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void static_call_site_swap(void *_a, void *_b, int size)
|
|
{
|
|
long delta = (unsigned long)_a - (unsigned long)_b;
|
|
struct static_call_site *a = _a;
|
|
struct static_call_site *b = _b;
|
|
struct static_call_site tmp = *a;
|
|
|
|
a->addr = b->addr - delta;
|
|
a->key = b->key - delta;
|
|
|
|
b->addr = tmp.addr + delta;
|
|
b->key = tmp.key + delta;
|
|
}
|
|
|
|
static inline void static_call_sort_entries(struct static_call_site *start,
|
|
struct static_call_site *stop)
|
|
{
|
|
sort(start, stop - start, sizeof(struct static_call_site),
|
|
static_call_site_cmp, static_call_site_swap);
|
|
}
|
|
|
|
static inline bool static_call_key_has_mods(struct static_call_key *key)
|
|
{
|
|
return !(key->type & 1);
|
|
}
|
|
|
|
static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)
|
|
{
|
|
if (!static_call_key_has_mods(key))
|
|
return NULL;
|
|
|
|
return key->mods;
|
|
}
|
|
|
|
static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)
|
|
{
|
|
if (static_call_key_has_mods(key))
|
|
return NULL;
|
|
|
|
return (struct static_call_site *)(key->type & ~1);
|
|
}
|
|
|
|
void __static_call_update(struct static_call_key *key, void *tramp, void *func)
|
|
{
|
|
struct static_call_site *site, *stop;
|
|
struct static_call_mod *site_mod, first;
|
|
|
|
cpus_read_lock();
|
|
static_call_lock();
|
|
|
|
if (key->func == func)
|
|
goto done;
|
|
|
|
key->func = func;
|
|
|
|
arch_static_call_transform(NULL, tramp, func, false);
|
|
|
|
/*
|
|
* If uninitialized, we'll not update the callsites, but they still
|
|
* point to the trampoline and we just patched that.
|
|
*/
|
|
if (WARN_ON_ONCE(!static_call_initialized))
|
|
goto done;
|
|
|
|
first = (struct static_call_mod){
|
|
.next = static_call_key_next(key),
|
|
.mod = NULL,
|
|
.sites = static_call_key_sites(key),
|
|
};
|
|
|
|
for (site_mod = &first; site_mod; site_mod = site_mod->next) {
|
|
bool init = system_state < SYSTEM_RUNNING;
|
|
struct module *mod = site_mod->mod;
|
|
|
|
if (!site_mod->sites) {
|
|
/*
|
|
* This can happen if the static call key is defined in
|
|
* a module which doesn't use it.
|
|
*
|
|
* It also happens in the has_mods case, where the
|
|
* 'first' entry has no sites associated with it.
|
|
*/
|
|
continue;
|
|
}
|
|
|
|
stop = __stop_static_call_sites;
|
|
|
|
if (mod) {
|
|
#ifdef CONFIG_MODULES
|
|
stop = mod->static_call_sites +
|
|
mod->num_static_call_sites;
|
|
init = mod->state == MODULE_STATE_COMING;
|
|
#endif
|
|
}
|
|
|
|
for (site = site_mod->sites;
|
|
site < stop && static_call_key(site) == key; site++) {
|
|
void *site_addr = static_call_addr(site);
|
|
|
|
if (!init && static_call_is_init(site))
|
|
continue;
|
|
|
|
if (!kernel_text_address((unsigned long)site_addr)) {
|
|
/*
|
|
* This skips patching built-in __exit, which
|
|
* is part of init_section_contains() but is
|
|
* not part of kernel_text_address().
|
|
*
|
|
* Skipping built-in __exit is fine since it
|
|
* will never be executed.
|
|
*/
|
|
WARN_ONCE(!static_call_is_init(site),
|
|
"can't patch static call site at %pS",
|
|
site_addr);
|
|
continue;
|
|
}
|
|
|
|
arch_static_call_transform(site_addr, NULL, func,
|
|
static_call_is_tail(site));
|
|
}
|
|
}
|
|
|
|
done:
|
|
static_call_unlock();
|
|
cpus_read_unlock();
|
|
}
|
|
EXPORT_SYMBOL_GPL(__static_call_update);
|
|
|
|
static int __static_call_init(struct module *mod,
|
|
struct static_call_site *start,
|
|
struct static_call_site *stop)
|
|
{
|
|
struct static_call_site *site;
|
|
struct static_call_key *key, *prev_key = NULL;
|
|
struct static_call_mod *site_mod;
|
|
|
|
if (start == stop)
|
|
return 0;
|
|
|
|
static_call_sort_entries(start, stop);
|
|
|
|
for (site = start; site < stop; site++) {
|
|
void *site_addr = static_call_addr(site);
|
|
|
|
if ((mod && within_module_init((unsigned long)site_addr, mod)) ||
|
|
(!mod && init_section_contains(site_addr, 1)))
|
|
static_call_set_init(site);
|
|
|
|
key = static_call_key(site);
|
|
if (key != prev_key) {
|
|
prev_key = key;
|
|
|
|
/*
|
|
* For vmlinux (!mod) avoid the allocation by storing
|
|
* the sites pointer in the key itself. Also see
|
|
* __static_call_update()'s @first.
|
|
*
|
|
* This allows architectures (eg. x86) to call
|
|
* static_call_init() before memory allocation works.
|
|
*/
|
|
if (!mod) {
|
|
key->sites = site;
|
|
key->type |= 1;
|
|
goto do_transform;
|
|
}
|
|
|
|
site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
|
|
if (!site_mod)
|
|
return -ENOMEM;
|
|
|
|
/*
|
|
* When the key has a direct sites pointer, extract
|
|
* that into an explicit struct static_call_mod, so we
|
|
* can have a list of modules.
|
|
*/
|
|
if (static_call_key_sites(key)) {
|
|
site_mod->mod = NULL;
|
|
site_mod->next = NULL;
|
|
site_mod->sites = static_call_key_sites(key);
|
|
|
|
key->mods = site_mod;
|
|
|
|
site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
|
|
if (!site_mod)
|
|
return -ENOMEM;
|
|
}
|
|
|
|
site_mod->mod = mod;
|
|
site_mod->sites = site;
|
|
site_mod->next = static_call_key_next(key);
|
|
key->mods = site_mod;
|
|
}
|
|
|
|
do_transform:
|
|
arch_static_call_transform(site_addr, NULL, key->func,
|
|
static_call_is_tail(site));
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int addr_conflict(struct static_call_site *site, void *start, void *end)
|
|
{
|
|
unsigned long addr = (unsigned long)static_call_addr(site);
|
|
|
|
if (addr <= (unsigned long)end &&
|
|
addr + CALL_INSN_SIZE > (unsigned long)start)
|
|
return 1;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int __static_call_text_reserved(struct static_call_site *iter_start,
|
|
struct static_call_site *iter_stop,
|
|
void *start, void *end)
|
|
{
|
|
struct static_call_site *iter = iter_start;
|
|
|
|
while (iter < iter_stop) {
|
|
if (addr_conflict(iter, start, end))
|
|
return 1;
|
|
iter++;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
#ifdef CONFIG_MODULES
|
|
|
|
static int __static_call_mod_text_reserved(void *start, void *end)
|
|
{
|
|
struct module *mod;
|
|
int ret;
|
|
|
|
preempt_disable();
|
|
mod = __module_text_address((unsigned long)start);
|
|
WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
|
|
if (!try_module_get(mod))
|
|
mod = NULL;
|
|
preempt_enable();
|
|
|
|
if (!mod)
|
|
return 0;
|
|
|
|
ret = __static_call_text_reserved(mod->static_call_sites,
|
|
mod->static_call_sites + mod->num_static_call_sites,
|
|
start, end);
|
|
|
|
module_put(mod);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static unsigned long tramp_key_lookup(unsigned long addr)
|
|
{
|
|
struct static_call_tramp_key *start = __start_static_call_tramp_key;
|
|
struct static_call_tramp_key *stop = __stop_static_call_tramp_key;
|
|
struct static_call_tramp_key *tramp_key;
|
|
|
|
for (tramp_key = start; tramp_key != stop; tramp_key++) {
|
|
unsigned long tramp;
|
|
|
|
tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp;
|
|
if (tramp == addr)
|
|
return (long)tramp_key->key + (long)&tramp_key->key;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int static_call_add_module(struct module *mod)
|
|
{
|
|
struct static_call_site *start = mod->static_call_sites;
|
|
struct static_call_site *stop = start + mod->num_static_call_sites;
|
|
struct static_call_site *site;
|
|
|
|
for (site = start; site != stop; site++) {
|
|
unsigned long s_key = __static_call_key(site);
|
|
unsigned long addr = s_key & ~STATIC_CALL_SITE_FLAGS;
|
|
unsigned long key;
|
|
|
|
/*
|
|
* Is the key is exported, 'addr' points to the key, which
|
|
* means modules are allowed to call static_call_update() on
|
|
* it.
|
|
*
|
|
* Otherwise, the key isn't exported, and 'addr' points to the
|
|
* trampoline so we need to lookup the key.
|
|
*
|
|
* We go through this dance to prevent crazy modules from
|
|
* abusing sensitive static calls.
|
|
*/
|
|
if (!kernel_text_address(addr))
|
|
continue;
|
|
|
|
key = tramp_key_lookup(addr);
|
|
if (!key) {
|
|
pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n",
|
|
static_call_addr(site));
|
|
return -EINVAL;
|
|
}
|
|
|
|
key |= s_key & STATIC_CALL_SITE_FLAGS;
|
|
site->key = key - (long)&site->key;
|
|
}
|
|
|
|
return __static_call_init(mod, start, stop);
|
|
}
|
|
|
|
static void static_call_del_module(struct module *mod)
|
|
{
|
|
struct static_call_site *start = mod->static_call_sites;
|
|
struct static_call_site *stop = mod->static_call_sites +
|
|
mod->num_static_call_sites;
|
|
struct static_call_key *key, *prev_key = NULL;
|
|
struct static_call_mod *site_mod, **prev;
|
|
struct static_call_site *site;
|
|
|
|
for (site = start; site < stop; site++) {
|
|
key = static_call_key(site);
|
|
if (key == prev_key)
|
|
continue;
|
|
|
|
prev_key = key;
|
|
|
|
for (prev = &key->mods, site_mod = key->mods;
|
|
site_mod && site_mod->mod != mod;
|
|
prev = &site_mod->next, site_mod = site_mod->next)
|
|
;
|
|
|
|
if (!site_mod)
|
|
continue;
|
|
|
|
*prev = site_mod->next;
|
|
kfree(site_mod);
|
|
}
|
|
}
|
|
|
|
static int static_call_module_notify(struct notifier_block *nb,
|
|
unsigned long val, void *data)
|
|
{
|
|
struct module *mod = data;
|
|
int ret = 0;
|
|
|
|
cpus_read_lock();
|
|
static_call_lock();
|
|
|
|
switch (val) {
|
|
case MODULE_STATE_COMING:
|
|
ret = static_call_add_module(mod);
|
|
if (ret) {
|
|
WARN(1, "Failed to allocate memory for static calls");
|
|
static_call_del_module(mod);
|
|
}
|
|
break;
|
|
case MODULE_STATE_GOING:
|
|
static_call_del_module(mod);
|
|
break;
|
|
}
|
|
|
|
static_call_unlock();
|
|
cpus_read_unlock();
|
|
|
|
return notifier_from_errno(ret);
|
|
}
|
|
|
|
static struct notifier_block static_call_module_nb = {
|
|
.notifier_call = static_call_module_notify,
|
|
};
|
|
|
|
#else
|
|
|
|
static inline int __static_call_mod_text_reserved(void *start, void *end)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
#endif /* CONFIG_MODULES */
|
|
|
|
int static_call_text_reserved(void *start, void *end)
|
|
{
|
|
int ret = __static_call_text_reserved(__start_static_call_sites,
|
|
__stop_static_call_sites, start, end);
|
|
|
|
if (ret)
|
|
return ret;
|
|
|
|
return __static_call_mod_text_reserved(start, end);
|
|
}
|
|
|
|
int __init static_call_init(void)
|
|
{
|
|
int ret;
|
|
|
|
if (static_call_initialized)
|
|
return 0;
|
|
|
|
cpus_read_lock();
|
|
static_call_lock();
|
|
ret = __static_call_init(NULL, __start_static_call_sites,
|
|
__stop_static_call_sites);
|
|
static_call_unlock();
|
|
cpus_read_unlock();
|
|
|
|
if (ret) {
|
|
pr_err("Failed to allocate memory for static_call!\n");
|
|
BUG();
|
|
}
|
|
|
|
static_call_initialized = true;
|
|
|
|
#ifdef CONFIG_MODULES
|
|
register_module_notifier(&static_call_module_nb);
|
|
#endif
|
|
return 0;
|
|
}
|
|
early_initcall(static_call_init);
|
|
|
|
long __static_call_return0(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
#ifdef CONFIG_STATIC_CALL_SELFTEST
|
|
|
|
static int func_a(int x)
|
|
{
|
|
return x+1;
|
|
}
|
|
|
|
static int func_b(int x)
|
|
{
|
|
return x+2;
|
|
}
|
|
|
|
DEFINE_STATIC_CALL(sc_selftest, func_a);
|
|
|
|
static struct static_call_data {
|
|
int (*func)(int);
|
|
int val;
|
|
int expect;
|
|
} static_call_data [] __initdata = {
|
|
{ NULL, 2, 3 },
|
|
{ func_b, 2, 4 },
|
|
{ func_a, 2, 3 }
|
|
};
|
|
|
|
static int __init test_static_call_init(void)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < ARRAY_SIZE(static_call_data); i++ ) {
|
|
struct static_call_data *scd = &static_call_data[i];
|
|
|
|
if (scd->func)
|
|
static_call_update(sc_selftest, scd->func);
|
|
|
|
WARN_ON(static_call(sc_selftest)(scd->val) != scd->expect);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
early_initcall(test_static_call_init);
|
|
|
|
#endif /* CONFIG_STATIC_CALL_SELFTEST */
|