mirror of
https://github.com/torvalds/linux.git
synced 2024-11-25 21:51:40 +00:00
73ab05aa46
With KASAN and PREEMPT_RT enabled, calling task_work_add() in task_tick_mm_cid() may cause the following splat. [ 63.696416] BUG: sleeping function called from invalid context at kernel/locking/spinlock_rt.c:48 [ 63.696416] in_atomic(): 1, irqs_disabled(): 1, non_block: 0, pid: 610, name: modprobe [ 63.696416] preempt_count: 10001, expected: 0 [ 63.696416] RCU nest depth: 1, expected: 1 This problem is caused by the following call trace. sched_tick() [ acquire rq->__lock ] -> task_tick_mm_cid() -> task_work_add() -> __kasan_record_aux_stack() -> kasan_save_stack() -> stack_depot_save_flags() -> alloc_pages_mpol_noprof() -> __alloc_pages_noprof() -> get_page_from_freelist() -> rmqueue() -> rmqueue_pcplist() -> __rmqueue_pcplist() -> rmqueue_bulk() -> rt_spin_lock() The rq lock is a raw_spinlock_t. We can't sleep while holding it. IOW, we can't call alloc_pages() in stack_depot_save_flags(). The task_tick_mm_cid() function with its task_work_add() call was introduced by commit223baf9d17
("sched: Fix performance regression introduced by mm_cid") in v6.4 kernel. Fortunately, there is a kasan_record_aux_stack_noalloc() variant that calls stack_depot_save_flags() while not allowing it to allocate new pages. To allow task_tick_mm_cid() to use task_work without page allocation, a new TWAF_NO_ALLOC flag is added to enable calling kasan_record_aux_stack_noalloc() instead of kasan_record_aux_stack() if set. The task_tick_mm_cid() function is modified to add this new flag. The possible downside is the missing stack trace in a KASAN report due to new page allocation required when task_work_add_noallloc() is called which should be rare. Fixes:223baf9d17
("sched: Fix performance regression introduced by mm_cid") Signed-off-by: Waiman Long <longman@redhat.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lkml.kernel.org/r/20241010014432.194742-1-longman@redhat.com
245 lines
6.7 KiB
C
245 lines
6.7 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#include <linux/irq_work.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/task_work.h>
|
|
#include <linux/resume_user_mode.h>
|
|
|
|
static struct callback_head work_exited; /* all we need is ->next == NULL */
|
|
|
|
#ifdef CONFIG_IRQ_WORK
|
|
static void task_work_set_notify_irq(struct irq_work *entry)
|
|
{
|
|
test_and_set_tsk_thread_flag(current, TIF_NOTIFY_RESUME);
|
|
}
|
|
static DEFINE_PER_CPU(struct irq_work, irq_work_NMI_resume) =
|
|
IRQ_WORK_INIT_HARD(task_work_set_notify_irq);
|
|
#endif
|
|
|
|
/**
|
|
* task_work_add - ask the @task to execute @work->func()
|
|
* @task: the task which should run the callback
|
|
* @work: the callback to run
|
|
* @notify: how to notify the targeted task
|
|
*
|
|
* Queue @work for task_work_run() below and notify the @task if @notify
|
|
* is @TWA_RESUME, @TWA_SIGNAL, @TWA_SIGNAL_NO_IPI or @TWA_NMI_CURRENT.
|
|
*
|
|
* @TWA_SIGNAL works like signals, in that the it will interrupt the targeted
|
|
* task and run the task_work, regardless of whether the task is currently
|
|
* running in the kernel or userspace.
|
|
* @TWA_SIGNAL_NO_IPI works like @TWA_SIGNAL, except it doesn't send a
|
|
* reschedule IPI to force the targeted task to reschedule and run task_work.
|
|
* This can be advantageous if there's no strict requirement that the
|
|
* task_work be run as soon as possible, just whenever the task enters the
|
|
* kernel anyway.
|
|
* @TWA_RESUME work is run only when the task exits the kernel and returns to
|
|
* user mode, or before entering guest mode.
|
|
* @TWA_NMI_CURRENT works like @TWA_RESUME, except it can only be used for the
|
|
* current @task and if the current context is NMI.
|
|
*
|
|
* Fails if the @task is exiting/exited and thus it can't process this @work.
|
|
* Otherwise @work->func() will be called when the @task goes through one of
|
|
* the aforementioned transitions, or exits.
|
|
*
|
|
* If the targeted task is exiting, then an error is returned and the work item
|
|
* is not queued. It's up to the caller to arrange for an alternative mechanism
|
|
* in that case.
|
|
*
|
|
* Note: there is no ordering guarantee on works queued here. The task_work
|
|
* list is LIFO.
|
|
*
|
|
* RETURNS:
|
|
* 0 if succeeds or -ESRCH.
|
|
*/
|
|
int task_work_add(struct task_struct *task, struct callback_head *work,
|
|
enum task_work_notify_mode notify)
|
|
{
|
|
struct callback_head *head;
|
|
int flags = notify & TWA_FLAGS;
|
|
|
|
notify &= ~TWA_FLAGS;
|
|
if (notify == TWA_NMI_CURRENT) {
|
|
if (WARN_ON_ONCE(task != current))
|
|
return -EINVAL;
|
|
if (!IS_ENABLED(CONFIG_IRQ_WORK))
|
|
return -EINVAL;
|
|
} else {
|
|
/*
|
|
* Record the work call stack in order to print it in KASAN
|
|
* reports.
|
|
*
|
|
* Note that stack allocation can fail if TWAF_NO_ALLOC flag
|
|
* is set and new page is needed to expand the stack buffer.
|
|
*/
|
|
if (flags & TWAF_NO_ALLOC)
|
|
kasan_record_aux_stack_noalloc(work);
|
|
else
|
|
kasan_record_aux_stack(work);
|
|
}
|
|
|
|
head = READ_ONCE(task->task_works);
|
|
do {
|
|
if (unlikely(head == &work_exited))
|
|
return -ESRCH;
|
|
work->next = head;
|
|
} while (!try_cmpxchg(&task->task_works, &head, work));
|
|
|
|
switch (notify) {
|
|
case TWA_NONE:
|
|
break;
|
|
case TWA_RESUME:
|
|
set_notify_resume(task);
|
|
break;
|
|
case TWA_SIGNAL:
|
|
set_notify_signal(task);
|
|
break;
|
|
case TWA_SIGNAL_NO_IPI:
|
|
__set_notify_signal(task);
|
|
break;
|
|
#ifdef CONFIG_IRQ_WORK
|
|
case TWA_NMI_CURRENT:
|
|
irq_work_queue(this_cpu_ptr(&irq_work_NMI_resume));
|
|
break;
|
|
#endif
|
|
default:
|
|
WARN_ON_ONCE(1);
|
|
break;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* task_work_cancel_match - cancel a pending work added by task_work_add()
|
|
* @task: the task which should execute the work
|
|
* @match: match function to call
|
|
* @data: data to be passed in to match function
|
|
*
|
|
* RETURNS:
|
|
* The found work or NULL if not found.
|
|
*/
|
|
struct callback_head *
|
|
task_work_cancel_match(struct task_struct *task,
|
|
bool (*match)(struct callback_head *, void *data),
|
|
void *data)
|
|
{
|
|
struct callback_head **pprev = &task->task_works;
|
|
struct callback_head *work;
|
|
unsigned long flags;
|
|
|
|
if (likely(!task_work_pending(task)))
|
|
return NULL;
|
|
/*
|
|
* If cmpxchg() fails we continue without updating pprev.
|
|
* Either we raced with task_work_add() which added the
|
|
* new entry before this work, we will find it again. Or
|
|
* we raced with task_work_run(), *pprev == NULL/exited.
|
|
*/
|
|
raw_spin_lock_irqsave(&task->pi_lock, flags);
|
|
work = READ_ONCE(*pprev);
|
|
while (work) {
|
|
if (!match(work, data)) {
|
|
pprev = &work->next;
|
|
work = READ_ONCE(*pprev);
|
|
} else if (try_cmpxchg(pprev, &work, work->next))
|
|
break;
|
|
}
|
|
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
|
|
|
|
return work;
|
|
}
|
|
|
|
static bool task_work_func_match(struct callback_head *cb, void *data)
|
|
{
|
|
return cb->func == data;
|
|
}
|
|
|
|
/**
|
|
* task_work_cancel_func - cancel a pending work matching a function added by task_work_add()
|
|
* @task: the task which should execute the func's work
|
|
* @func: identifies the func to match with a work to remove
|
|
*
|
|
* Find the last queued pending work with ->func == @func and remove
|
|
* it from queue.
|
|
*
|
|
* RETURNS:
|
|
* The found work or NULL if not found.
|
|
*/
|
|
struct callback_head *
|
|
task_work_cancel_func(struct task_struct *task, task_work_func_t func)
|
|
{
|
|
return task_work_cancel_match(task, task_work_func_match, func);
|
|
}
|
|
|
|
static bool task_work_match(struct callback_head *cb, void *data)
|
|
{
|
|
return cb == data;
|
|
}
|
|
|
|
/**
|
|
* task_work_cancel - cancel a pending work added by task_work_add()
|
|
* @task: the task which should execute the work
|
|
* @cb: the callback to remove if queued
|
|
*
|
|
* Remove a callback from a task's queue if queued.
|
|
*
|
|
* RETURNS:
|
|
* True if the callback was queued and got cancelled, false otherwise.
|
|
*/
|
|
bool task_work_cancel(struct task_struct *task, struct callback_head *cb)
|
|
{
|
|
struct callback_head *ret;
|
|
|
|
ret = task_work_cancel_match(task, task_work_match, cb);
|
|
|
|
return ret == cb;
|
|
}
|
|
|
|
/**
|
|
* task_work_run - execute the works added by task_work_add()
|
|
*
|
|
* Flush the pending works. Should be used by the core kernel code.
|
|
* Called before the task returns to the user-mode or stops, or when
|
|
* it exits. In the latter case task_work_add() can no longer add the
|
|
* new work after task_work_run() returns.
|
|
*/
|
|
void task_work_run(void)
|
|
{
|
|
struct task_struct *task = current;
|
|
struct callback_head *work, *head, *next;
|
|
|
|
for (;;) {
|
|
/*
|
|
* work->func() can do task_work_add(), do not set
|
|
* work_exited unless the list is empty.
|
|
*/
|
|
work = READ_ONCE(task->task_works);
|
|
do {
|
|
head = NULL;
|
|
if (!work) {
|
|
if (task->flags & PF_EXITING)
|
|
head = &work_exited;
|
|
else
|
|
break;
|
|
}
|
|
} while (!try_cmpxchg(&task->task_works, &work, head));
|
|
|
|
if (!work)
|
|
break;
|
|
/*
|
|
* Synchronize with task_work_cancel_match(). It can not remove
|
|
* the first entry == work, cmpxchg(task_works) must fail.
|
|
* But it can remove another entry from the ->next list.
|
|
*/
|
|
raw_spin_lock_irq(&task->pi_lock);
|
|
raw_spin_unlock_irq(&task->pi_lock);
|
|
|
|
do {
|
|
next = work->next;
|
|
work->func(work);
|
|
work = next;
|
|
cond_resched();
|
|
} while (work);
|
|
}
|
|
}
|