forked from Minki/linux
8663effb24
I finally got around to creating trampolines for dynamically allocated ftrace_ops with using synchronize_rcu_tasks(). For users of the ftrace function hook callbacks, like perf, that allocate the ftrace_ops descriptor via kmalloc() and friends, ftrace was not able to optimize the functions being traced to use a trampoline because they would also need to be allocated dynamically. The problem is that they cannot be freed when CONFIG_PREEMPT is set, as there's no way to tell if a task was preempted on the trampoline. That was before Paul McKenney implemented synchronize_rcu_tasks() that would make sure all tasks (except idle) have scheduled out or have entered user space. While testing this, I triggered this bug: BUG: unable to handle kernel paging request at ffffffffa0230077 ... RIP: 0010:0xffffffffa0230077 ... Call Trace: schedule+0x5/0xe0 schedule_preempt_disabled+0x18/0x30 do_idle+0x172/0x220 What happened was that the idle task was preempted on the trampoline. As synchronize_rcu_tasks() ignores the idle thread, there's nothing that lets ftrace know that the idle task was preempted on a trampoline. The idle task shouldn't need to ever enable preemption. The idle task is simply a loop that calls schedule or places the cpu into idle mode. In fact, having preemption enabled is inefficient, because it can happen when idle is just about to call schedule anyway, which would cause schedule to be called twice. Once for when the interrupt came in and was returning back to normal context, and then again in the normal path that the idle loop is running in, which would be pointless, as it had already scheduled. The only reason schedule_preempt_disable() enables preemption is to be able to call sched_submit_work(), which requires preemption enabled. As this is a nop when the task is in the RUNNING state, and idle is always in the running state, there's no reason that idle needs to enable preemption. But that means it cannot use schedule_preempt_disable() as other callers of that function require calling sched_submit_work(). Adding a new function local to kernel/sched/ that allows idle to call the scheduler without enabling preemption, fixes the synchronize_rcu_tasks() issue, as well as removes the pointless spurious schedule calls caused by interrupts happening in the brief window where preemption is enabled just before it calls schedule. Reviewed: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Link: http://lkml.kernel.org/r/20170414084809.3dacde2a@gandalf.local.home Signed-off-by: Ingo Molnar <mingo@kernel.org>
7388 lines
181 KiB
C
7388 lines
181 KiB
C
/*
|
|
* kernel/sched/core.c
|
|
*
|
|
* Core kernel scheduler code and related syscalls
|
|
*
|
|
* Copyright (C) 1991-2002 Linus Torvalds
|
|
*/
|
|
#include <linux/sched.h>
|
|
#include <linux/sched/clock.h>
|
|
#include <uapi/linux/sched/types.h>
|
|
#include <linux/sched/loadavg.h>
|
|
#include <linux/sched/hotplug.h>
|
|
#include <linux/cpuset.h>
|
|
#include <linux/delayacct.h>
|
|
#include <linux/init_task.h>
|
|
#include <linux/context_tracking.h>
|
|
#include <linux/rcupdate_wait.h>
|
|
|
|
#include <linux/blkdev.h>
|
|
#include <linux/kprobes.h>
|
|
#include <linux/mmu_context.h>
|
|
#include <linux/module.h>
|
|
#include <linux/nmi.h>
|
|
#include <linux/prefetch.h>
|
|
#include <linux/profile.h>
|
|
#include <linux/security.h>
|
|
#include <linux/syscalls.h>
|
|
|
|
#include <asm/switch_to.h>
|
|
#include <asm/tlb.h>
|
|
#ifdef CONFIG_PARAVIRT
|
|
#include <asm/paravirt.h>
|
|
#endif
|
|
|
|
#include "sched.h"
|
|
#include "../workqueue_internal.h"
|
|
#include "../smpboot.h"
|
|
|
|
#define CREATE_TRACE_POINTS
|
|
#include <trace/events/sched.h>
|
|
|
|
DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
|
|
|
|
/*
|
|
* Debugging: various feature bits
|
|
*/
|
|
|
|
#define SCHED_FEAT(name, enabled) \
|
|
(1UL << __SCHED_FEAT_##name) * enabled |
|
|
|
|
const_debug unsigned int sysctl_sched_features =
|
|
#include "features.h"
|
|
0;
|
|
|
|
#undef SCHED_FEAT
|
|
|
|
/*
|
|
* Number of tasks to iterate in a single balance run.
|
|
* Limited because this is done with IRQs disabled.
|
|
*/
|
|
const_debug unsigned int sysctl_sched_nr_migrate = 32;
|
|
|
|
/*
|
|
* period over which we average the RT time consumption, measured
|
|
* in ms.
|
|
*
|
|
* default: 1s
|
|
*/
|
|
const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
|
|
|
|
/*
|
|
* period over which we measure -rt task CPU usage in us.
|
|
* default: 1s
|
|
*/
|
|
unsigned int sysctl_sched_rt_period = 1000000;
|
|
|
|
__read_mostly int scheduler_running;
|
|
|
|
/*
|
|
* part of the period that we allow rt tasks to run in us.
|
|
* default: 0.95s
|
|
*/
|
|
int sysctl_sched_rt_runtime = 950000;
|
|
|
|
/* CPUs with isolated domains */
|
|
cpumask_var_t cpu_isolated_map;
|
|
|
|
/*
|
|
* __task_rq_lock - lock the rq @p resides on.
|
|
*/
|
|
struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
|
|
__acquires(rq->lock)
|
|
{
|
|
struct rq *rq;
|
|
|
|
lockdep_assert_held(&p->pi_lock);
|
|
|
|
for (;;) {
|
|
rq = task_rq(p);
|
|
raw_spin_lock(&rq->lock);
|
|
if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
|
|
rq_pin_lock(rq, rf);
|
|
return rq;
|
|
}
|
|
raw_spin_unlock(&rq->lock);
|
|
|
|
while (unlikely(task_on_rq_migrating(p)))
|
|
cpu_relax();
|
|
}
|
|
}
|
|
|
|
/*
|
|
* task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
|
|
*/
|
|
struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
|
|
__acquires(p->pi_lock)
|
|
__acquires(rq->lock)
|
|
{
|
|
struct rq *rq;
|
|
|
|
for (;;) {
|
|
raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
|
|
rq = task_rq(p);
|
|
raw_spin_lock(&rq->lock);
|
|
/*
|
|
* move_queued_task() task_rq_lock()
|
|
*
|
|
* ACQUIRE (rq->lock)
|
|
* [S] ->on_rq = MIGRATING [L] rq = task_rq()
|
|
* WMB (__set_task_cpu()) ACQUIRE (rq->lock);
|
|
* [S] ->cpu = new_cpu [L] task_rq()
|
|
* [L] ->on_rq
|
|
* RELEASE (rq->lock)
|
|
*
|
|
* If we observe the old cpu in task_rq_lock, the acquire of
|
|
* the old rq->lock will fully serialize against the stores.
|
|
*
|
|
* If we observe the new CPU in task_rq_lock, the acquire will
|
|
* pair with the WMB to ensure we must then also see migrating.
|
|
*/
|
|
if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
|
|
rq_pin_lock(rq, rf);
|
|
return rq;
|
|
}
|
|
raw_spin_unlock(&rq->lock);
|
|
raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
|
|
|
|
while (unlikely(task_on_rq_migrating(p)))
|
|
cpu_relax();
|
|
}
|
|
}
|
|
|
|
/*
|
|
* RQ-clock updating methods:
|
|
*/
|
|
|
|
static void update_rq_clock_task(struct rq *rq, s64 delta)
|
|
{
|
|
/*
|
|
* In theory, the compile should just see 0 here, and optimize out the call
|
|
* to sched_rt_avg_update. But I don't trust it...
|
|
*/
|
|
#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
|
|
s64 steal = 0, irq_delta = 0;
|
|
#endif
|
|
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
|
|
irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
|
|
|
|
/*
|
|
* Since irq_time is only updated on {soft,}irq_exit, we might run into
|
|
* this case when a previous update_rq_clock() happened inside a
|
|
* {soft,}irq region.
|
|
*
|
|
* When this happens, we stop ->clock_task and only update the
|
|
* prev_irq_time stamp to account for the part that fit, so that a next
|
|
* update will consume the rest. This ensures ->clock_task is
|
|
* monotonic.
|
|
*
|
|
* It does however cause some slight miss-attribution of {soft,}irq
|
|
* time, a more accurate solution would be to update the irq_time using
|
|
* the current rq->clock timestamp, except that would require using
|
|
* atomic ops.
|
|
*/
|
|
if (irq_delta > delta)
|
|
irq_delta = delta;
|
|
|
|
rq->prev_irq_time += irq_delta;
|
|
delta -= irq_delta;
|
|
#endif
|
|
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
|
|
if (static_key_false((¶virt_steal_rq_enabled))) {
|
|
steal = paravirt_steal_clock(cpu_of(rq));
|
|
steal -= rq->prev_steal_time_rq;
|
|
|
|
if (unlikely(steal > delta))
|
|
steal = delta;
|
|
|
|
rq->prev_steal_time_rq += steal;
|
|
delta -= steal;
|
|
}
|
|
#endif
|
|
|
|
rq->clock_task += delta;
|
|
|
|
#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
|
|
if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
|
|
sched_rt_avg_update(rq, irq_delta + steal);
|
|
#endif
|
|
}
|
|
|
|
void update_rq_clock(struct rq *rq)
|
|
{
|
|
s64 delta;
|
|
|
|
lockdep_assert_held(&rq->lock);
|
|
|
|
if (rq->clock_update_flags & RQCF_ACT_SKIP)
|
|
return;
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
if (sched_feat(WARN_DOUBLE_CLOCK))
|
|
SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED);
|
|
rq->clock_update_flags |= RQCF_UPDATED;
|
|
#endif
|
|
|
|
delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
|
|
if (delta < 0)
|
|
return;
|
|
rq->clock += delta;
|
|
update_rq_clock_task(rq, delta);
|
|
}
|
|
|
|
|
|
#ifdef CONFIG_SCHED_HRTICK
|
|
/*
|
|
* Use HR-timers to deliver accurate preemption points.
|
|
*/
|
|
|
|
static void hrtick_clear(struct rq *rq)
|
|
{
|
|
if (hrtimer_active(&rq->hrtick_timer))
|
|
hrtimer_cancel(&rq->hrtick_timer);
|
|
}
|
|
|
|
/*
|
|
* High-resolution timer tick.
|
|
* Runs from hardirq context with interrupts disabled.
|
|
*/
|
|
static enum hrtimer_restart hrtick(struct hrtimer *timer)
|
|
{
|
|
struct rq *rq = container_of(timer, struct rq, hrtick_timer);
|
|
struct rq_flags rf;
|
|
|
|
WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
|
|
|
|
rq_lock(rq, &rf);
|
|
update_rq_clock(rq);
|
|
rq->curr->sched_class->task_tick(rq, rq->curr, 1);
|
|
rq_unlock(rq, &rf);
|
|
|
|
return HRTIMER_NORESTART;
|
|
}
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
static void __hrtick_restart(struct rq *rq)
|
|
{
|
|
struct hrtimer *timer = &rq->hrtick_timer;
|
|
|
|
hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
|
|
}
|
|
|
|
/*
|
|
* called from hardirq (IPI) context
|
|
*/
|
|
static void __hrtick_start(void *arg)
|
|
{
|
|
struct rq *rq = arg;
|
|
struct rq_flags rf;
|
|
|
|
rq_lock(rq, &rf);
|
|
__hrtick_restart(rq);
|
|
rq->hrtick_csd_pending = 0;
|
|
rq_unlock(rq, &rf);
|
|
}
|
|
|
|
/*
|
|
* Called to set the hrtick timer state.
|
|
*
|
|
* called with rq->lock held and irqs disabled
|
|
*/
|
|
void hrtick_start(struct rq *rq, u64 delay)
|
|
{
|
|
struct hrtimer *timer = &rq->hrtick_timer;
|
|
ktime_t time;
|
|
s64 delta;
|
|
|
|
/*
|
|
* Don't schedule slices shorter than 10000ns, that just
|
|
* doesn't make sense and can cause timer DoS.
|
|
*/
|
|
delta = max_t(s64, delay, 10000LL);
|
|
time = ktime_add_ns(timer->base->get_time(), delta);
|
|
|
|
hrtimer_set_expires(timer, time);
|
|
|
|
if (rq == this_rq()) {
|
|
__hrtick_restart(rq);
|
|
} else if (!rq->hrtick_csd_pending) {
|
|
smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
|
|
rq->hrtick_csd_pending = 1;
|
|
}
|
|
}
|
|
|
|
#else
|
|
/*
|
|
* Called to set the hrtick timer state.
|
|
*
|
|
* called with rq->lock held and irqs disabled
|
|
*/
|
|
void hrtick_start(struct rq *rq, u64 delay)
|
|
{
|
|
/*
|
|
* Don't schedule slices shorter than 10000ns, that just
|
|
* doesn't make sense. Rely on vruntime for fairness.
|
|
*/
|
|
delay = max_t(u64, delay, 10000LL);
|
|
hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
|
|
HRTIMER_MODE_REL_PINNED);
|
|
}
|
|
#endif /* CONFIG_SMP */
|
|
|
|
static void init_rq_hrtick(struct rq *rq)
|
|
{
|
|
#ifdef CONFIG_SMP
|
|
rq->hrtick_csd_pending = 0;
|
|
|
|
rq->hrtick_csd.flags = 0;
|
|
rq->hrtick_csd.func = __hrtick_start;
|
|
rq->hrtick_csd.info = rq;
|
|
#endif
|
|
|
|
hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
|
rq->hrtick_timer.function = hrtick;
|
|
}
|
|
#else /* CONFIG_SCHED_HRTICK */
|
|
static inline void hrtick_clear(struct rq *rq)
|
|
{
|
|
}
|
|
|
|
static inline void init_rq_hrtick(struct rq *rq)
|
|
{
|
|
}
|
|
#endif /* CONFIG_SCHED_HRTICK */
|
|
|
|
/*
|
|
* cmpxchg based fetch_or, macro so it works for different integer types
|
|
*/
|
|
#define fetch_or(ptr, mask) \
|
|
({ \
|
|
typeof(ptr) _ptr = (ptr); \
|
|
typeof(mask) _mask = (mask); \
|
|
typeof(*_ptr) _old, _val = *_ptr; \
|
|
\
|
|
for (;;) { \
|
|
_old = cmpxchg(_ptr, _val, _val | _mask); \
|
|
if (_old == _val) \
|
|
break; \
|
|
_val = _old; \
|
|
} \
|
|
_old; \
|
|
})
|
|
|
|
#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
|
|
/*
|
|
* Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
|
|
* this avoids any races wrt polling state changes and thereby avoids
|
|
* spurious IPIs.
|
|
*/
|
|
static bool set_nr_and_not_polling(struct task_struct *p)
|
|
{
|
|
struct thread_info *ti = task_thread_info(p);
|
|
return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
|
|
}
|
|
|
|
/*
|
|
* Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
|
|
*
|
|
* If this returns true, then the idle task promises to call
|
|
* sched_ttwu_pending() and reschedule soon.
|
|
*/
|
|
static bool set_nr_if_polling(struct task_struct *p)
|
|
{
|
|
struct thread_info *ti = task_thread_info(p);
|
|
typeof(ti->flags) old, val = READ_ONCE(ti->flags);
|
|
|
|
for (;;) {
|
|
if (!(val & _TIF_POLLING_NRFLAG))
|
|
return false;
|
|
if (val & _TIF_NEED_RESCHED)
|
|
return true;
|
|
old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
|
|
if (old == val)
|
|
break;
|
|
val = old;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
#else
|
|
static bool set_nr_and_not_polling(struct task_struct *p)
|
|
{
|
|
set_tsk_need_resched(p);
|
|
return true;
|
|
}
|
|
|
|
#ifdef CONFIG_SMP
|
|
static bool set_nr_if_polling(struct task_struct *p)
|
|
{
|
|
return false;
|
|
}
|
|
#endif
|
|
#endif
|
|
|
|
void wake_q_add(struct wake_q_head *head, struct task_struct *task)
|
|
{
|
|
struct wake_q_node *node = &task->wake_q;
|
|
|
|
/*
|
|
* Atomically grab the task, if ->wake_q is !nil already it means
|
|
* its already queued (either by us or someone else) and will get the
|
|
* wakeup due to that.
|
|
*
|
|
* This cmpxchg() implies a full barrier, which pairs with the write
|
|
* barrier implied by the wakeup in wake_up_q().
|
|
*/
|
|
if (cmpxchg(&node->next, NULL, WAKE_Q_TAIL))
|
|
return;
|
|
|
|
get_task_struct(task);
|
|
|
|
/*
|
|
* The head is context local, there can be no concurrency.
|
|
*/
|
|
*head->lastp = node;
|
|
head->lastp = &node->next;
|
|
}
|
|
|
|
void wake_up_q(struct wake_q_head *head)
|
|
{
|
|
struct wake_q_node *node = head->first;
|
|
|
|
while (node != WAKE_Q_TAIL) {
|
|
struct task_struct *task;
|
|
|
|
task = container_of(node, struct task_struct, wake_q);
|
|
BUG_ON(!task);
|
|
/* Task can safely be re-inserted now: */
|
|
node = node->next;
|
|
task->wake_q.next = NULL;
|
|
|
|
/*
|
|
* wake_up_process() implies a wmb() to pair with the queueing
|
|
* in wake_q_add() so as not to miss wakeups.
|
|
*/
|
|
wake_up_process(task);
|
|
put_task_struct(task);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* resched_curr - mark rq's current task 'to be rescheduled now'.
|
|
*
|
|
* On UP this means the setting of the need_resched flag, on SMP it
|
|
* might also involve a cross-CPU call to trigger the scheduler on
|
|
* the target CPU.
|
|
*/
|
|
void resched_curr(struct rq *rq)
|
|
{
|
|
struct task_struct *curr = rq->curr;
|
|
int cpu;
|
|
|
|
lockdep_assert_held(&rq->lock);
|
|
|
|
if (test_tsk_need_resched(curr))
|
|
return;
|
|
|
|
cpu = cpu_of(rq);
|
|
|
|
if (cpu == smp_processor_id()) {
|
|
set_tsk_need_resched(curr);
|
|
set_preempt_need_resched();
|
|
return;
|
|
}
|
|
|
|
if (set_nr_and_not_polling(curr))
|
|
smp_send_reschedule(cpu);
|
|
else
|
|
trace_sched_wake_idle_without_ipi(cpu);
|
|
}
|
|
|
|
void resched_cpu(int cpu)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
unsigned long flags;
|
|
|
|
if (!raw_spin_trylock_irqsave(&rq->lock, flags))
|
|
return;
|
|
resched_curr(rq);
|
|
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
|
}
|
|
|
|
#ifdef CONFIG_SMP
|
|
#ifdef CONFIG_NO_HZ_COMMON
|
|
/*
|
|
* In the semi idle case, use the nearest busy CPU for migrating timers
|
|
* from an idle CPU. This is good for power-savings.
|
|
*
|
|
* We don't do similar optimization for completely idle system, as
|
|
* selecting an idle CPU will add more delays to the timers than intended
|
|
* (as that CPU's timer base may not be uptodate wrt jiffies etc).
|
|
*/
|
|
int get_nohz_timer_target(void)
|
|
{
|
|
int i, cpu = smp_processor_id();
|
|
struct sched_domain *sd;
|
|
|
|
if (!idle_cpu(cpu) && is_housekeeping_cpu(cpu))
|
|
return cpu;
|
|
|
|
rcu_read_lock();
|
|
for_each_domain(cpu, sd) {
|
|
for_each_cpu(i, sched_domain_span(sd)) {
|
|
if (cpu == i)
|
|
continue;
|
|
|
|
if (!idle_cpu(i) && is_housekeeping_cpu(i)) {
|
|
cpu = i;
|
|
goto unlock;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!is_housekeeping_cpu(cpu))
|
|
cpu = housekeeping_any_cpu();
|
|
unlock:
|
|
rcu_read_unlock();
|
|
return cpu;
|
|
}
|
|
|
|
/*
|
|
* When add_timer_on() enqueues a timer into the timer wheel of an
|
|
* idle CPU then this timer might expire before the next timer event
|
|
* which is scheduled to wake up that CPU. In case of a completely
|
|
* idle system the next event might even be infinite time into the
|
|
* future. wake_up_idle_cpu() ensures that the CPU is woken up and
|
|
* leaves the inner idle loop so the newly added timer is taken into
|
|
* account when the CPU goes back to idle and evaluates the timer
|
|
* wheel for the next timer event.
|
|
*/
|
|
static void wake_up_idle_cpu(int cpu)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
if (cpu == smp_processor_id())
|
|
return;
|
|
|
|
if (set_nr_and_not_polling(rq->idle))
|
|
smp_send_reschedule(cpu);
|
|
else
|
|
trace_sched_wake_idle_without_ipi(cpu);
|
|
}
|
|
|
|
static bool wake_up_full_nohz_cpu(int cpu)
|
|
{
|
|
/*
|
|
* We just need the target to call irq_exit() and re-evaluate
|
|
* the next tick. The nohz full kick at least implies that.
|
|
* If needed we can still optimize that later with an
|
|
* empty IRQ.
|
|
*/
|
|
if (cpu_is_offline(cpu))
|
|
return true; /* Don't try to wake offline CPUs. */
|
|
if (tick_nohz_full_cpu(cpu)) {
|
|
if (cpu != smp_processor_id() ||
|
|
tick_nohz_tick_stopped())
|
|
tick_nohz_full_kick_cpu(cpu);
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/*
|
|
* Wake up the specified CPU. If the CPU is going offline, it is the
|
|
* caller's responsibility to deal with the lost wakeup, for example,
|
|
* by hooking into the CPU_DEAD notifier like timers and hrtimers do.
|
|
*/
|
|
void wake_up_nohz_cpu(int cpu)
|
|
{
|
|
if (!wake_up_full_nohz_cpu(cpu))
|
|
wake_up_idle_cpu(cpu);
|
|
}
|
|
|
|
static inline bool got_nohz_idle_kick(void)
|
|
{
|
|
int cpu = smp_processor_id();
|
|
|
|
if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)))
|
|
return false;
|
|
|
|
if (idle_cpu(cpu) && !need_resched())
|
|
return true;
|
|
|
|
/*
|
|
* We can't run Idle Load Balance on this CPU for this time so we
|
|
* cancel it and clear NOHZ_BALANCE_KICK
|
|
*/
|
|
clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu));
|
|
return false;
|
|
}
|
|
|
|
#else /* CONFIG_NO_HZ_COMMON */
|
|
|
|
static inline bool got_nohz_idle_kick(void)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
#endif /* CONFIG_NO_HZ_COMMON */
|
|
|
|
#ifdef CONFIG_NO_HZ_FULL
|
|
bool sched_can_stop_tick(struct rq *rq)
|
|
{
|
|
int fifo_nr_running;
|
|
|
|
/* Deadline tasks, even if single, need the tick */
|
|
if (rq->dl.dl_nr_running)
|
|
return false;
|
|
|
|
/*
|
|
* If there are more than one RR tasks, we need the tick to effect the
|
|
* actual RR behaviour.
|
|
*/
|
|
if (rq->rt.rr_nr_running) {
|
|
if (rq->rt.rr_nr_running == 1)
|
|
return true;
|
|
else
|
|
return false;
|
|
}
|
|
|
|
/*
|
|
* If there's no RR tasks, but FIFO tasks, we can skip the tick, no
|
|
* forced preemption between FIFO tasks.
|
|
*/
|
|
fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running;
|
|
if (fifo_nr_running)
|
|
return true;
|
|
|
|
/*
|
|
* If there are no DL,RR/FIFO tasks, there must only be CFS tasks left;
|
|
* if there's more than one we need the tick for involuntary
|
|
* preemption.
|
|
*/
|
|
if (rq->nr_running > 1)
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
#endif /* CONFIG_NO_HZ_FULL */
|
|
|
|
void sched_avg_update(struct rq *rq)
|
|
{
|
|
s64 period = sched_avg_period();
|
|
|
|
while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
|
|
/*
|
|
* Inline assembly required to prevent the compiler
|
|
* optimising this loop into a divmod call.
|
|
* See __iter_div_u64_rem() for another example of this.
|
|
*/
|
|
asm("" : "+rm" (rq->age_stamp));
|
|
rq->age_stamp += period;
|
|
rq->rt_avg /= 2;
|
|
}
|
|
}
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
|
|
(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
|
|
/*
|
|
* Iterate task_group tree rooted at *from, calling @down when first entering a
|
|
* node and @up when leaving it for the final time.
|
|
*
|
|
* Caller must hold rcu_lock or sufficient equivalent.
|
|
*/
|
|
int walk_tg_tree_from(struct task_group *from,
|
|
tg_visitor down, tg_visitor up, void *data)
|
|
{
|
|
struct task_group *parent, *child;
|
|
int ret;
|
|
|
|
parent = from;
|
|
|
|
down:
|
|
ret = (*down)(parent, data);
|
|
if (ret)
|
|
goto out;
|
|
list_for_each_entry_rcu(child, &parent->children, siblings) {
|
|
parent = child;
|
|
goto down;
|
|
|
|
up:
|
|
continue;
|
|
}
|
|
ret = (*up)(parent, data);
|
|
if (ret || parent == from)
|
|
goto out;
|
|
|
|
child = parent;
|
|
parent = parent->parent;
|
|
if (parent)
|
|
goto up;
|
|
out:
|
|
return ret;
|
|
}
|
|
|
|
int tg_nop(struct task_group *tg, void *data)
|
|
{
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
static void set_load_weight(struct task_struct *p)
|
|
{
|
|
int prio = p->static_prio - MAX_RT_PRIO;
|
|
struct load_weight *load = &p->se.load;
|
|
|
|
/*
|
|
* SCHED_IDLE tasks get minimal weight:
|
|
*/
|
|
if (idle_policy(p->policy)) {
|
|
load->weight = scale_load(WEIGHT_IDLEPRIO);
|
|
load->inv_weight = WMULT_IDLEPRIO;
|
|
return;
|
|
}
|
|
|
|
load->weight = scale_load(sched_prio_to_weight[prio]);
|
|
load->inv_weight = sched_prio_to_wmult[prio];
|
|
}
|
|
|
|
static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
|
|
{
|
|
if (!(flags & ENQUEUE_NOCLOCK))
|
|
update_rq_clock(rq);
|
|
|
|
if (!(flags & ENQUEUE_RESTORE))
|
|
sched_info_queued(rq, p);
|
|
|
|
p->sched_class->enqueue_task(rq, p, flags);
|
|
}
|
|
|
|
static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
|
|
{
|
|
if (!(flags & DEQUEUE_NOCLOCK))
|
|
update_rq_clock(rq);
|
|
|
|
if (!(flags & DEQUEUE_SAVE))
|
|
sched_info_dequeued(rq, p);
|
|
|
|
p->sched_class->dequeue_task(rq, p, flags);
|
|
}
|
|
|
|
void activate_task(struct rq *rq, struct task_struct *p, int flags)
|
|
{
|
|
if (task_contributes_to_load(p))
|
|
rq->nr_uninterruptible--;
|
|
|
|
enqueue_task(rq, p, flags);
|
|
}
|
|
|
|
void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
|
|
{
|
|
if (task_contributes_to_load(p))
|
|
rq->nr_uninterruptible++;
|
|
|
|
dequeue_task(rq, p, flags);
|
|
}
|
|
|
|
void sched_set_stop_task(int cpu, struct task_struct *stop)
|
|
{
|
|
struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
|
|
struct task_struct *old_stop = cpu_rq(cpu)->stop;
|
|
|
|
if (stop) {
|
|
/*
|
|
* Make it appear like a SCHED_FIFO task, its something
|
|
* userspace knows about and won't get confused about.
|
|
*
|
|
* Also, it will make PI more or less work without too
|
|
* much confusion -- but then, stop work should not
|
|
* rely on PI working anyway.
|
|
*/
|
|
sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m);
|
|
|
|
stop->sched_class = &stop_sched_class;
|
|
}
|
|
|
|
cpu_rq(cpu)->stop = stop;
|
|
|
|
if (old_stop) {
|
|
/*
|
|
* Reset it back to a normal scheduling class so that
|
|
* it can die in pieces.
|
|
*/
|
|
old_stop->sched_class = &rt_sched_class;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* __normal_prio - return the priority that is based on the static prio
|
|
*/
|
|
static inline int __normal_prio(struct task_struct *p)
|
|
{
|
|
return p->static_prio;
|
|
}
|
|
|
|
/*
|
|
* Calculate the expected normal priority: i.e. priority
|
|
* without taking RT-inheritance into account. Might be
|
|
* boosted by interactivity modifiers. Changes upon fork,
|
|
* setprio syscalls, and whenever the interactivity
|
|
* estimator recalculates.
|
|
*/
|
|
static inline int normal_prio(struct task_struct *p)
|
|
{
|
|
int prio;
|
|
|
|
if (task_has_dl_policy(p))
|
|
prio = MAX_DL_PRIO-1;
|
|
else if (task_has_rt_policy(p))
|
|
prio = MAX_RT_PRIO-1 - p->rt_priority;
|
|
else
|
|
prio = __normal_prio(p);
|
|
return prio;
|
|
}
|
|
|
|
/*
|
|
* Calculate the current priority, i.e. the priority
|
|
* taken into account by the scheduler. This value might
|
|
* be boosted by RT tasks, or might be boosted by
|
|
* interactivity modifiers. Will be RT if the task got
|
|
* RT-boosted. If not then it returns p->normal_prio.
|
|
*/
|
|
static int effective_prio(struct task_struct *p)
|
|
{
|
|
p->normal_prio = normal_prio(p);
|
|
/*
|
|
* If we are RT tasks or we were boosted to RT priority,
|
|
* keep the priority unchanged. Otherwise, update priority
|
|
* to the normal priority:
|
|
*/
|
|
if (!rt_prio(p->prio))
|
|
return p->normal_prio;
|
|
return p->prio;
|
|
}
|
|
|
|
/**
|
|
* task_curr - is this task currently executing on a CPU?
|
|
* @p: the task in question.
|
|
*
|
|
* Return: 1 if the task is currently executing. 0 otherwise.
|
|
*/
|
|
inline int task_curr(const struct task_struct *p)
|
|
{
|
|
return cpu_curr(task_cpu(p)) == p;
|
|
}
|
|
|
|
/*
|
|
* switched_from, switched_to and prio_changed must _NOT_ drop rq->lock,
|
|
* use the balance_callback list if you want balancing.
|
|
*
|
|
* this means any call to check_class_changed() must be followed by a call to
|
|
* balance_callback().
|
|
*/
|
|
static inline void check_class_changed(struct rq *rq, struct task_struct *p,
|
|
const struct sched_class *prev_class,
|
|
int oldprio)
|
|
{
|
|
if (prev_class != p->sched_class) {
|
|
if (prev_class->switched_from)
|
|
prev_class->switched_from(rq, p);
|
|
|
|
p->sched_class->switched_to(rq, p);
|
|
} else if (oldprio != p->prio || dl_task(p))
|
|
p->sched_class->prio_changed(rq, p, oldprio);
|
|
}
|
|
|
|
void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
|
|
{
|
|
const struct sched_class *class;
|
|
|
|
if (p->sched_class == rq->curr->sched_class) {
|
|
rq->curr->sched_class->check_preempt_curr(rq, p, flags);
|
|
} else {
|
|
for_each_class(class) {
|
|
if (class == rq->curr->sched_class)
|
|
break;
|
|
if (class == p->sched_class) {
|
|
resched_curr(rq);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* A queue event has occurred, and we're going to schedule. In
|
|
* this case, we can save a useless back to back clock update.
|
|
*/
|
|
if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
|
|
rq_clock_skip_update(rq, true);
|
|
}
|
|
|
|
#ifdef CONFIG_SMP
|
|
/*
|
|
* This is how migration works:
|
|
*
|
|
* 1) we invoke migration_cpu_stop() on the target CPU using
|
|
* stop_one_cpu().
|
|
* 2) stopper starts to run (implicitly forcing the migrated thread
|
|
* off the CPU)
|
|
* 3) it checks whether the migrated task is still in the wrong runqueue.
|
|
* 4) if it's in the wrong runqueue then the migration thread removes
|
|
* it and puts it into the right queue.
|
|
* 5) stopper completes and stop_one_cpu() returns and the migration
|
|
* is done.
|
|
*/
|
|
|
|
/*
|
|
* move_queued_task - move a queued task to new rq.
|
|
*
|
|
* Returns (locked) new rq. Old rq's lock is released.
|
|
*/
|
|
static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,
|
|
struct task_struct *p, int new_cpu)
|
|
{
|
|
lockdep_assert_held(&rq->lock);
|
|
|
|
p->on_rq = TASK_ON_RQ_MIGRATING;
|
|
dequeue_task(rq, p, DEQUEUE_NOCLOCK);
|
|
set_task_cpu(p, new_cpu);
|
|
rq_unlock(rq, rf);
|
|
|
|
rq = cpu_rq(new_cpu);
|
|
|
|
rq_lock(rq, rf);
|
|
BUG_ON(task_cpu(p) != new_cpu);
|
|
enqueue_task(rq, p, 0);
|
|
p->on_rq = TASK_ON_RQ_QUEUED;
|
|
check_preempt_curr(rq, p, 0);
|
|
|
|
return rq;
|
|
}
|
|
|
|
struct migration_arg {
|
|
struct task_struct *task;
|
|
int dest_cpu;
|
|
};
|
|
|
|
/*
|
|
* Move (not current) task off this CPU, onto the destination CPU. We're doing
|
|
* this because either it can't run here any more (set_cpus_allowed()
|
|
* away from this CPU, or CPU going down), or because we're
|
|
* attempting to rebalance this task on exec (sched_exec).
|
|
*
|
|
* So we race with normal scheduler movements, but that's OK, as long
|
|
* as the task is no longer on this CPU.
|
|
*/
|
|
static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
|
|
struct task_struct *p, int dest_cpu)
|
|
{
|
|
if (unlikely(!cpu_active(dest_cpu)))
|
|
return rq;
|
|
|
|
/* Affinity changed (again). */
|
|
if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
|
|
return rq;
|
|
|
|
update_rq_clock(rq);
|
|
rq = move_queued_task(rq, rf, p, dest_cpu);
|
|
|
|
return rq;
|
|
}
|
|
|
|
/*
|
|
* migration_cpu_stop - this will be executed by a highprio stopper thread
|
|
* and performs thread migration by bumping thread off CPU then
|
|
* 'pushing' onto another runqueue.
|
|
*/
|
|
static int migration_cpu_stop(void *data)
|
|
{
|
|
struct migration_arg *arg = data;
|
|
struct task_struct *p = arg->task;
|
|
struct rq *rq = this_rq();
|
|
struct rq_flags rf;
|
|
|
|
/*
|
|
* The original target CPU might have gone down and we might
|
|
* be on another CPU but it doesn't matter.
|
|
*/
|
|
local_irq_disable();
|
|
/*
|
|
* We need to explicitly wake pending tasks before running
|
|
* __migrate_task() such that we will not miss enforcing cpus_allowed
|
|
* during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
|
|
*/
|
|
sched_ttwu_pending();
|
|
|
|
raw_spin_lock(&p->pi_lock);
|
|
rq_lock(rq, &rf);
|
|
/*
|
|
* If task_rq(p) != rq, it cannot be migrated here, because we're
|
|
* holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
|
|
* we're holding p->pi_lock.
|
|
*/
|
|
if (task_rq(p) == rq) {
|
|
if (task_on_rq_queued(p))
|
|
rq = __migrate_task(rq, &rf, p, arg->dest_cpu);
|
|
else
|
|
p->wake_cpu = arg->dest_cpu;
|
|
}
|
|
rq_unlock(rq, &rf);
|
|
raw_spin_unlock(&p->pi_lock);
|
|
|
|
local_irq_enable();
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* sched_class::set_cpus_allowed must do the below, but is not required to
|
|
* actually call this function.
|
|
*/
|
|
void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask)
|
|
{
|
|
cpumask_copy(&p->cpus_allowed, new_mask);
|
|
p->nr_cpus_allowed = cpumask_weight(new_mask);
|
|
}
|
|
|
|
void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
|
|
{
|
|
struct rq *rq = task_rq(p);
|
|
bool queued, running;
|
|
|
|
lockdep_assert_held(&p->pi_lock);
|
|
|
|
queued = task_on_rq_queued(p);
|
|
running = task_current(rq, p);
|
|
|
|
if (queued) {
|
|
/*
|
|
* Because __kthread_bind() calls this on blocked tasks without
|
|
* holding rq->lock.
|
|
*/
|
|
lockdep_assert_held(&rq->lock);
|
|
dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
|
|
}
|
|
if (running)
|
|
put_prev_task(rq, p);
|
|
|
|
p->sched_class->set_cpus_allowed(p, new_mask);
|
|
|
|
if (queued)
|
|
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
|
|
if (running)
|
|
set_curr_task(rq, p);
|
|
}
|
|
|
|
/*
|
|
* Change a given task's CPU affinity. Migrate the thread to a
|
|
* proper CPU and schedule it away if the CPU it's executing on
|
|
* is removed from the allowed bitmask.
|
|
*
|
|
* NOTE: the caller must have a valid reference to the task, the
|
|
* task must not exit() & deallocate itself prematurely. The
|
|
* call is not atomic; no spinlocks may be held.
|
|
*/
|
|
static int __set_cpus_allowed_ptr(struct task_struct *p,
|
|
const struct cpumask *new_mask, bool check)
|
|
{
|
|
const struct cpumask *cpu_valid_mask = cpu_active_mask;
|
|
unsigned int dest_cpu;
|
|
struct rq_flags rf;
|
|
struct rq *rq;
|
|
int ret = 0;
|
|
|
|
rq = task_rq_lock(p, &rf);
|
|
update_rq_clock(rq);
|
|
|
|
if (p->flags & PF_KTHREAD) {
|
|
/*
|
|
* Kernel threads are allowed on online && !active CPUs
|
|
*/
|
|
cpu_valid_mask = cpu_online_mask;
|
|
}
|
|
|
|
/*
|
|
* Must re-check here, to close a race against __kthread_bind(),
|
|
* sched_setaffinity() is not guaranteed to observe the flag.
|
|
*/
|
|
if (check && (p->flags & PF_NO_SETAFFINITY)) {
|
|
ret = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
if (cpumask_equal(&p->cpus_allowed, new_mask))
|
|
goto out;
|
|
|
|
if (!cpumask_intersects(new_mask, cpu_valid_mask)) {
|
|
ret = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
do_set_cpus_allowed(p, new_mask);
|
|
|
|
if (p->flags & PF_KTHREAD) {
|
|
/*
|
|
* For kernel threads that do indeed end up on online &&
|
|
* !active we want to ensure they are strict per-CPU threads.
|
|
*/
|
|
WARN_ON(cpumask_intersects(new_mask, cpu_online_mask) &&
|
|
!cpumask_intersects(new_mask, cpu_active_mask) &&
|
|
p->nr_cpus_allowed != 1);
|
|
}
|
|
|
|
/* Can the task run on the task's current CPU? If so, we're done */
|
|
if (cpumask_test_cpu(task_cpu(p), new_mask))
|
|
goto out;
|
|
|
|
dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask);
|
|
if (task_running(rq, p) || p->state == TASK_WAKING) {
|
|
struct migration_arg arg = { p, dest_cpu };
|
|
/* Need help from migration thread: drop lock and wait. */
|
|
task_rq_unlock(rq, p, &rf);
|
|
stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
|
|
tlb_migrate_finish(p->mm);
|
|
return 0;
|
|
} else if (task_on_rq_queued(p)) {
|
|
/*
|
|
* OK, since we're going to drop the lock immediately
|
|
* afterwards anyway.
|
|
*/
|
|
rq = move_queued_task(rq, &rf, p, dest_cpu);
|
|
}
|
|
out:
|
|
task_rq_unlock(rq, p, &rf);
|
|
|
|
return ret;
|
|
}
|
|
|
|
int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
|
|
{
|
|
return __set_cpus_allowed_ptr(p, new_mask, false);
|
|
}
|
|
EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
|
|
|
|
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
|
|
{
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
/*
|
|
* We should never call set_task_cpu() on a blocked task,
|
|
* ttwu() will sort out the placement.
|
|
*/
|
|
WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
|
|
!p->on_rq);
|
|
|
|
/*
|
|
* Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING,
|
|
* because schedstat_wait_{start,end} rebase migrating task's wait_start
|
|
* time relying on p->on_rq.
|
|
*/
|
|
WARN_ON_ONCE(p->state == TASK_RUNNING &&
|
|
p->sched_class == &fair_sched_class &&
|
|
(p->on_rq && !task_on_rq_migrating(p)));
|
|
|
|
#ifdef CONFIG_LOCKDEP
|
|
/*
|
|
* The caller should hold either p->pi_lock or rq->lock, when changing
|
|
* a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
|
|
*
|
|
* sched_move_task() holds both and thus holding either pins the cgroup,
|
|
* see task_group().
|
|
*
|
|
* Furthermore, all task_rq users should acquire both locks, see
|
|
* task_rq_lock().
|
|
*/
|
|
WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
|
|
lockdep_is_held(&task_rq(p)->lock)));
|
|
#endif
|
|
#endif
|
|
|
|
trace_sched_migrate_task(p, new_cpu);
|
|
|
|
if (task_cpu(p) != new_cpu) {
|
|
if (p->sched_class->migrate_task_rq)
|
|
p->sched_class->migrate_task_rq(p);
|
|
p->se.nr_migrations++;
|
|
perf_event_task_migrate(p);
|
|
}
|
|
|
|
__set_task_cpu(p, new_cpu);
|
|
}
|
|
|
|
static void __migrate_swap_task(struct task_struct *p, int cpu)
|
|
{
|
|
if (task_on_rq_queued(p)) {
|
|
struct rq *src_rq, *dst_rq;
|
|
struct rq_flags srf, drf;
|
|
|
|
src_rq = task_rq(p);
|
|
dst_rq = cpu_rq(cpu);
|
|
|
|
rq_pin_lock(src_rq, &srf);
|
|
rq_pin_lock(dst_rq, &drf);
|
|
|
|
p->on_rq = TASK_ON_RQ_MIGRATING;
|
|
deactivate_task(src_rq, p, 0);
|
|
set_task_cpu(p, cpu);
|
|
activate_task(dst_rq, p, 0);
|
|
p->on_rq = TASK_ON_RQ_QUEUED;
|
|
check_preempt_curr(dst_rq, p, 0);
|
|
|
|
rq_unpin_lock(dst_rq, &drf);
|
|
rq_unpin_lock(src_rq, &srf);
|
|
|
|
} else {
|
|
/*
|
|
* Task isn't running anymore; make it appear like we migrated
|
|
* it before it went to sleep. This means on wakeup we make the
|
|
* previous CPU our target instead of where it really is.
|
|
*/
|
|
p->wake_cpu = cpu;
|
|
}
|
|
}
|
|
|
|
struct migration_swap_arg {
|
|
struct task_struct *src_task, *dst_task;
|
|
int src_cpu, dst_cpu;
|
|
};
|
|
|
|
static int migrate_swap_stop(void *data)
|
|
{
|
|
struct migration_swap_arg *arg = data;
|
|
struct rq *src_rq, *dst_rq;
|
|
int ret = -EAGAIN;
|
|
|
|
if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu))
|
|
return -EAGAIN;
|
|
|
|
src_rq = cpu_rq(arg->src_cpu);
|
|
dst_rq = cpu_rq(arg->dst_cpu);
|
|
|
|
double_raw_lock(&arg->src_task->pi_lock,
|
|
&arg->dst_task->pi_lock);
|
|
double_rq_lock(src_rq, dst_rq);
|
|
|
|
if (task_cpu(arg->dst_task) != arg->dst_cpu)
|
|
goto unlock;
|
|
|
|
if (task_cpu(arg->src_task) != arg->src_cpu)
|
|
goto unlock;
|
|
|
|
if (!cpumask_test_cpu(arg->dst_cpu, &arg->src_task->cpus_allowed))
|
|
goto unlock;
|
|
|
|
if (!cpumask_test_cpu(arg->src_cpu, &arg->dst_task->cpus_allowed))
|
|
goto unlock;
|
|
|
|
__migrate_swap_task(arg->src_task, arg->dst_cpu);
|
|
__migrate_swap_task(arg->dst_task, arg->src_cpu);
|
|
|
|
ret = 0;
|
|
|
|
unlock:
|
|
double_rq_unlock(src_rq, dst_rq);
|
|
raw_spin_unlock(&arg->dst_task->pi_lock);
|
|
raw_spin_unlock(&arg->src_task->pi_lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Cross migrate two tasks
|
|
*/
|
|
int migrate_swap(struct task_struct *cur, struct task_struct *p)
|
|
{
|
|
struct migration_swap_arg arg;
|
|
int ret = -EINVAL;
|
|
|
|
arg = (struct migration_swap_arg){
|
|
.src_task = cur,
|
|
.src_cpu = task_cpu(cur),
|
|
.dst_task = p,
|
|
.dst_cpu = task_cpu(p),
|
|
};
|
|
|
|
if (arg.src_cpu == arg.dst_cpu)
|
|
goto out;
|
|
|
|
/*
|
|
* These three tests are all lockless; this is OK since all of them
|
|
* will be re-checked with proper locks held further down the line.
|
|
*/
|
|
if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
|
|
goto out;
|
|
|
|
if (!cpumask_test_cpu(arg.dst_cpu, &arg.src_task->cpus_allowed))
|
|
goto out;
|
|
|
|
if (!cpumask_test_cpu(arg.src_cpu, &arg.dst_task->cpus_allowed))
|
|
goto out;
|
|
|
|
trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
|
|
ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
|
|
|
|
out:
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* wait_task_inactive - wait for a thread to unschedule.
|
|
*
|
|
* If @match_state is nonzero, it's the @p->state value just checked and
|
|
* not expected to change. If it changes, i.e. @p might have woken up,
|
|
* then return zero. When we succeed in waiting for @p to be off its CPU,
|
|
* we return a positive number (its total switch count). If a second call
|
|
* a short while later returns the same number, the caller can be sure that
|
|
* @p has remained unscheduled the whole time.
|
|
*
|
|
* The caller must ensure that the task *will* unschedule sometime soon,
|
|
* else this function might spin for a *long* time. This function can't
|
|
* be called with interrupts off, or it may introduce deadlock with
|
|
* smp_call_function() if an IPI is sent by the same process we are
|
|
* waiting to become inactive.
|
|
*/
|
|
unsigned long wait_task_inactive(struct task_struct *p, long match_state)
|
|
{
|
|
int running, queued;
|
|
struct rq_flags rf;
|
|
unsigned long ncsw;
|
|
struct rq *rq;
|
|
|
|
for (;;) {
|
|
/*
|
|
* We do the initial early heuristics without holding
|
|
* any task-queue locks at all. We'll only try to get
|
|
* the runqueue lock when things look like they will
|
|
* work out!
|
|
*/
|
|
rq = task_rq(p);
|
|
|
|
/*
|
|
* If the task is actively running on another CPU
|
|
* still, just relax and busy-wait without holding
|
|
* any locks.
|
|
*
|
|
* NOTE! Since we don't hold any locks, it's not
|
|
* even sure that "rq" stays as the right runqueue!
|
|
* But we don't care, since "task_running()" will
|
|
* return false if the runqueue has changed and p
|
|
* is actually now running somewhere else!
|
|
*/
|
|
while (task_running(rq, p)) {
|
|
if (match_state && unlikely(p->state != match_state))
|
|
return 0;
|
|
cpu_relax();
|
|
}
|
|
|
|
/*
|
|
* Ok, time to look more closely! We need the rq
|
|
* lock now, to be *sure*. If we're wrong, we'll
|
|
* just go back and repeat.
|
|
*/
|
|
rq = task_rq_lock(p, &rf);
|
|
trace_sched_wait_task(p);
|
|
running = task_running(rq, p);
|
|
queued = task_on_rq_queued(p);
|
|
ncsw = 0;
|
|
if (!match_state || p->state == match_state)
|
|
ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
|
|
task_rq_unlock(rq, p, &rf);
|
|
|
|
/*
|
|
* If it changed from the expected state, bail out now.
|
|
*/
|
|
if (unlikely(!ncsw))
|
|
break;
|
|
|
|
/*
|
|
* Was it really running after all now that we
|
|
* checked with the proper locks actually held?
|
|
*
|
|
* Oops. Go back and try again..
|
|
*/
|
|
if (unlikely(running)) {
|
|
cpu_relax();
|
|
continue;
|
|
}
|
|
|
|
/*
|
|
* It's not enough that it's not actively running,
|
|
* it must be off the runqueue _entirely_, and not
|
|
* preempted!
|
|
*
|
|
* So if it was still runnable (but just not actively
|
|
* running right now), it's preempted, and we should
|
|
* yield - it could be a while.
|
|
*/
|
|
if (unlikely(queued)) {
|
|
ktime_t to = NSEC_PER_SEC / HZ;
|
|
|
|
set_current_state(TASK_UNINTERRUPTIBLE);
|
|
schedule_hrtimeout(&to, HRTIMER_MODE_REL);
|
|
continue;
|
|
}
|
|
|
|
/*
|
|
* Ahh, all good. It wasn't running, and it wasn't
|
|
* runnable, which means that it will never become
|
|
* running in the future either. We're all done!
|
|
*/
|
|
break;
|
|
}
|
|
|
|
return ncsw;
|
|
}
|
|
|
|
/***
|
|
* kick_process - kick a running thread to enter/exit the kernel
|
|
* @p: the to-be-kicked thread
|
|
*
|
|
* Cause a process which is running on another CPU to enter
|
|
* kernel-mode, without any delay. (to get signals handled.)
|
|
*
|
|
* NOTE: this function doesn't have to take the runqueue lock,
|
|
* because all it wants to ensure is that the remote task enters
|
|
* the kernel. If the IPI races and the task has been migrated
|
|
* to another CPU then no harm is done and the purpose has been
|
|
* achieved as well.
|
|
*/
|
|
void kick_process(struct task_struct *p)
|
|
{
|
|
int cpu;
|
|
|
|
preempt_disable();
|
|
cpu = task_cpu(p);
|
|
if ((cpu != smp_processor_id()) && task_curr(p))
|
|
smp_send_reschedule(cpu);
|
|
preempt_enable();
|
|
}
|
|
EXPORT_SYMBOL_GPL(kick_process);
|
|
|
|
/*
|
|
* ->cpus_allowed is protected by both rq->lock and p->pi_lock
|
|
*
|
|
* A few notes on cpu_active vs cpu_online:
|
|
*
|
|
* - cpu_active must be a subset of cpu_online
|
|
*
|
|
* - on cpu-up we allow per-cpu kthreads on the online && !active cpu,
|
|
* see __set_cpus_allowed_ptr(). At this point the newly online
|
|
* CPU isn't yet part of the sched domains, and balancing will not
|
|
* see it.
|
|
*
|
|
* - on CPU-down we clear cpu_active() to mask the sched domains and
|
|
* avoid the load balancer to place new tasks on the to be removed
|
|
* CPU. Existing tasks will remain running there and will be taken
|
|
* off.
|
|
*
|
|
* This means that fallback selection must not select !active CPUs.
|
|
* And can assume that any active CPU must be online. Conversely
|
|
* select_task_rq() below may allow selection of !active CPUs in order
|
|
* to satisfy the above rules.
|
|
*/
|
|
static int select_fallback_rq(int cpu, struct task_struct *p)
|
|
{
|
|
int nid = cpu_to_node(cpu);
|
|
const struct cpumask *nodemask = NULL;
|
|
enum { cpuset, possible, fail } state = cpuset;
|
|
int dest_cpu;
|
|
|
|
/*
|
|
* If the node that the CPU is on has been offlined, cpu_to_node()
|
|
* will return -1. There is no CPU on the node, and we should
|
|
* select the CPU on the other node.
|
|
*/
|
|
if (nid != -1) {
|
|
nodemask = cpumask_of_node(nid);
|
|
|
|
/* Look for allowed, online CPU in same node. */
|
|
for_each_cpu(dest_cpu, nodemask) {
|
|
if (!cpu_active(dest_cpu))
|
|
continue;
|
|
if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
|
|
return dest_cpu;
|
|
}
|
|
}
|
|
|
|
for (;;) {
|
|
/* Any allowed, online CPU? */
|
|
for_each_cpu(dest_cpu, &p->cpus_allowed) {
|
|
if (!(p->flags & PF_KTHREAD) && !cpu_active(dest_cpu))
|
|
continue;
|
|
if (!cpu_online(dest_cpu))
|
|
continue;
|
|
goto out;
|
|
}
|
|
|
|
/* No more Mr. Nice Guy. */
|
|
switch (state) {
|
|
case cpuset:
|
|
if (IS_ENABLED(CONFIG_CPUSETS)) {
|
|
cpuset_cpus_allowed_fallback(p);
|
|
state = possible;
|
|
break;
|
|
}
|
|
/* Fall-through */
|
|
case possible:
|
|
do_set_cpus_allowed(p, cpu_possible_mask);
|
|
state = fail;
|
|
break;
|
|
|
|
case fail:
|
|
BUG();
|
|
break;
|
|
}
|
|
}
|
|
|
|
out:
|
|
if (state != cpuset) {
|
|
/*
|
|
* Don't tell them about moving exiting tasks or
|
|
* kernel threads (both mm NULL), since they never
|
|
* leave kernel.
|
|
*/
|
|
if (p->mm && printk_ratelimit()) {
|
|
printk_deferred("process %d (%s) no longer affine to cpu%d\n",
|
|
task_pid_nr(p), p->comm, cpu);
|
|
}
|
|
}
|
|
|
|
return dest_cpu;
|
|
}
|
|
|
|
/*
|
|
* The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
|
|
*/
|
|
static inline
|
|
int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
|
|
{
|
|
lockdep_assert_held(&p->pi_lock);
|
|
|
|
if (p->nr_cpus_allowed > 1)
|
|
cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
|
|
else
|
|
cpu = cpumask_any(&p->cpus_allowed);
|
|
|
|
/*
|
|
* In order not to call set_task_cpu() on a blocking task we need
|
|
* to rely on ttwu() to place the task on a valid ->cpus_allowed
|
|
* CPU.
|
|
*
|
|
* Since this is common to all placement strategies, this lives here.
|
|
*
|
|
* [ this allows ->select_task() to simply return task_cpu(p) and
|
|
* not worry about this generic constraint ]
|
|
*/
|
|
if (unlikely(!cpumask_test_cpu(cpu, &p->cpus_allowed) ||
|
|
!cpu_online(cpu)))
|
|
cpu = select_fallback_rq(task_cpu(p), p);
|
|
|
|
return cpu;
|
|
}
|
|
|
|
static void update_avg(u64 *avg, u64 sample)
|
|
{
|
|
s64 diff = sample - *avg;
|
|
*avg += diff >> 3;
|
|
}
|
|
|
|
#else
|
|
|
|
static inline int __set_cpus_allowed_ptr(struct task_struct *p,
|
|
const struct cpumask *new_mask, bool check)
|
|
{
|
|
return set_cpus_allowed_ptr(p, new_mask);
|
|
}
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
static void
|
|
ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
|
|
{
|
|
struct rq *rq;
|
|
|
|
if (!schedstat_enabled())
|
|
return;
|
|
|
|
rq = this_rq();
|
|
|
|
#ifdef CONFIG_SMP
|
|
if (cpu == rq->cpu) {
|
|
schedstat_inc(rq->ttwu_local);
|
|
schedstat_inc(p->se.statistics.nr_wakeups_local);
|
|
} else {
|
|
struct sched_domain *sd;
|
|
|
|
schedstat_inc(p->se.statistics.nr_wakeups_remote);
|
|
rcu_read_lock();
|
|
for_each_domain(rq->cpu, sd) {
|
|
if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
|
|
schedstat_inc(sd->ttwu_wake_remote);
|
|
break;
|
|
}
|
|
}
|
|
rcu_read_unlock();
|
|
}
|
|
|
|
if (wake_flags & WF_MIGRATED)
|
|
schedstat_inc(p->se.statistics.nr_wakeups_migrate);
|
|
#endif /* CONFIG_SMP */
|
|
|
|
schedstat_inc(rq->ttwu_count);
|
|
schedstat_inc(p->se.statistics.nr_wakeups);
|
|
|
|
if (wake_flags & WF_SYNC)
|
|
schedstat_inc(p->se.statistics.nr_wakeups_sync);
|
|
}
|
|
|
|
static inline void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
|
|
{
|
|
activate_task(rq, p, en_flags);
|
|
p->on_rq = TASK_ON_RQ_QUEUED;
|
|
|
|
/* If a worker is waking up, notify the workqueue: */
|
|
if (p->flags & PF_WQ_WORKER)
|
|
wq_worker_waking_up(p, cpu_of(rq));
|
|
}
|
|
|
|
/*
|
|
* Mark the task runnable and perform wakeup-preemption.
|
|
*/
|
|
static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
|
|
struct rq_flags *rf)
|
|
{
|
|
check_preempt_curr(rq, p, wake_flags);
|
|
p->state = TASK_RUNNING;
|
|
trace_sched_wakeup(p);
|
|
|
|
#ifdef CONFIG_SMP
|
|
if (p->sched_class->task_woken) {
|
|
/*
|
|
* Our task @p is fully woken up and running; so its safe to
|
|
* drop the rq->lock, hereafter rq is only used for statistics.
|
|
*/
|
|
rq_unpin_lock(rq, rf);
|
|
p->sched_class->task_woken(rq, p);
|
|
rq_repin_lock(rq, rf);
|
|
}
|
|
|
|
if (rq->idle_stamp) {
|
|
u64 delta = rq_clock(rq) - rq->idle_stamp;
|
|
u64 max = 2*rq->max_idle_balance_cost;
|
|
|
|
update_avg(&rq->avg_idle, delta);
|
|
|
|
if (rq->avg_idle > max)
|
|
rq->avg_idle = max;
|
|
|
|
rq->idle_stamp = 0;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
static void
|
|
ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
|
|
struct rq_flags *rf)
|
|
{
|
|
int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK;
|
|
|
|
lockdep_assert_held(&rq->lock);
|
|
|
|
#ifdef CONFIG_SMP
|
|
if (p->sched_contributes_to_load)
|
|
rq->nr_uninterruptible--;
|
|
|
|
if (wake_flags & WF_MIGRATED)
|
|
en_flags |= ENQUEUE_MIGRATED;
|
|
#endif
|
|
|
|
ttwu_activate(rq, p, en_flags);
|
|
ttwu_do_wakeup(rq, p, wake_flags, rf);
|
|
}
|
|
|
|
/*
|
|
* Called in case the task @p isn't fully descheduled from its runqueue,
|
|
* in this case we must do a remote wakeup. Its a 'light' wakeup though,
|
|
* since all we need to do is flip p->state to TASK_RUNNING, since
|
|
* the task is still ->on_rq.
|
|
*/
|
|
static int ttwu_remote(struct task_struct *p, int wake_flags)
|
|
{
|
|
struct rq_flags rf;
|
|
struct rq *rq;
|
|
int ret = 0;
|
|
|
|
rq = __task_rq_lock(p, &rf);
|
|
if (task_on_rq_queued(p)) {
|
|
/* check_preempt_curr() may use rq clock */
|
|
update_rq_clock(rq);
|
|
ttwu_do_wakeup(rq, p, wake_flags, &rf);
|
|
ret = 1;
|
|
}
|
|
__task_rq_unlock(rq, &rf);
|
|
|
|
return ret;
|
|
}
|
|
|
|
#ifdef CONFIG_SMP
|
|
void sched_ttwu_pending(void)
|
|
{
|
|
struct rq *rq = this_rq();
|
|
struct llist_node *llist = llist_del_all(&rq->wake_list);
|
|
struct task_struct *p;
|
|
struct rq_flags rf;
|
|
|
|
if (!llist)
|
|
return;
|
|
|
|
rq_lock_irqsave(rq, &rf);
|
|
update_rq_clock(rq);
|
|
|
|
while (llist) {
|
|
int wake_flags = 0;
|
|
|
|
p = llist_entry(llist, struct task_struct, wake_entry);
|
|
llist = llist_next(llist);
|
|
|
|
if (p->sched_remote_wakeup)
|
|
wake_flags = WF_MIGRATED;
|
|
|
|
ttwu_do_activate(rq, p, wake_flags, &rf);
|
|
}
|
|
|
|
rq_unlock_irqrestore(rq, &rf);
|
|
}
|
|
|
|
void scheduler_ipi(void)
|
|
{
|
|
/*
|
|
* Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
|
|
* TIF_NEED_RESCHED remotely (for the first time) will also send
|
|
* this IPI.
|
|
*/
|
|
preempt_fold_need_resched();
|
|
|
|
if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
|
|
return;
|
|
|
|
/*
|
|
* Not all reschedule IPI handlers call irq_enter/irq_exit, since
|
|
* traditionally all their work was done from the interrupt return
|
|
* path. Now that we actually do some work, we need to make sure
|
|
* we do call them.
|
|
*
|
|
* Some archs already do call them, luckily irq_enter/exit nest
|
|
* properly.
|
|
*
|
|
* Arguably we should visit all archs and update all handlers,
|
|
* however a fair share of IPIs are still resched only so this would
|
|
* somewhat pessimize the simple resched case.
|
|
*/
|
|
irq_enter();
|
|
sched_ttwu_pending();
|
|
|
|
/*
|
|
* Check if someone kicked us for doing the nohz idle load balance.
|
|
*/
|
|
if (unlikely(got_nohz_idle_kick())) {
|
|
this_rq()->idle_balance = 1;
|
|
raise_softirq_irqoff(SCHED_SOFTIRQ);
|
|
}
|
|
irq_exit();
|
|
}
|
|
|
|
static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
|
|
|
|
if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
|
|
if (!set_nr_if_polling(rq->idle))
|
|
smp_send_reschedule(cpu);
|
|
else
|
|
trace_sched_wake_idle_without_ipi(cpu);
|
|
}
|
|
}
|
|
|
|
void wake_up_if_idle(int cpu)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
struct rq_flags rf;
|
|
|
|
rcu_read_lock();
|
|
|
|
if (!is_idle_task(rcu_dereference(rq->curr)))
|
|
goto out;
|
|
|
|
if (set_nr_if_polling(rq->idle)) {
|
|
trace_sched_wake_idle_without_ipi(cpu);
|
|
} else {
|
|
rq_lock_irqsave(rq, &rf);
|
|
if (is_idle_task(rq->curr))
|
|
smp_send_reschedule(cpu);
|
|
/* Else CPU is not idle, do nothing here: */
|
|
rq_unlock_irqrestore(rq, &rf);
|
|
}
|
|
|
|
out:
|
|
rcu_read_unlock();
|
|
}
|
|
|
|
bool cpus_share_cache(int this_cpu, int that_cpu)
|
|
{
|
|
return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
|
|
}
|
|
#endif /* CONFIG_SMP */
|
|
|
|
static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
struct rq_flags rf;
|
|
|
|
#if defined(CONFIG_SMP)
|
|
if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
|
|
sched_clock_cpu(cpu); /* Sync clocks across CPUs */
|
|
ttwu_queue_remote(p, cpu, wake_flags);
|
|
return;
|
|
}
|
|
#endif
|
|
|
|
rq_lock(rq, &rf);
|
|
update_rq_clock(rq);
|
|
ttwu_do_activate(rq, p, wake_flags, &rf);
|
|
rq_unlock(rq, &rf);
|
|
}
|
|
|
|
/*
|
|
* Notes on Program-Order guarantees on SMP systems.
|
|
*
|
|
* MIGRATION
|
|
*
|
|
* The basic program-order guarantee on SMP systems is that when a task [t]
|
|
* migrates, all its activity on its old CPU [c0] happens-before any subsequent
|
|
* execution on its new CPU [c1].
|
|
*
|
|
* For migration (of runnable tasks) this is provided by the following means:
|
|
*
|
|
* A) UNLOCK of the rq(c0)->lock scheduling out task t
|
|
* B) migration for t is required to synchronize *both* rq(c0)->lock and
|
|
* rq(c1)->lock (if not at the same time, then in that order).
|
|
* C) LOCK of the rq(c1)->lock scheduling in task
|
|
*
|
|
* Transitivity guarantees that B happens after A and C after B.
|
|
* Note: we only require RCpc transitivity.
|
|
* Note: the CPU doing B need not be c0 or c1
|
|
*
|
|
* Example:
|
|
*
|
|
* CPU0 CPU1 CPU2
|
|
*
|
|
* LOCK rq(0)->lock
|
|
* sched-out X
|
|
* sched-in Y
|
|
* UNLOCK rq(0)->lock
|
|
*
|
|
* LOCK rq(0)->lock // orders against CPU0
|
|
* dequeue X
|
|
* UNLOCK rq(0)->lock
|
|
*
|
|
* LOCK rq(1)->lock
|
|
* enqueue X
|
|
* UNLOCK rq(1)->lock
|
|
*
|
|
* LOCK rq(1)->lock // orders against CPU2
|
|
* sched-out Z
|
|
* sched-in X
|
|
* UNLOCK rq(1)->lock
|
|
*
|
|
*
|
|
* BLOCKING -- aka. SLEEP + WAKEUP
|
|
*
|
|
* For blocking we (obviously) need to provide the same guarantee as for
|
|
* migration. However the means are completely different as there is no lock
|
|
* chain to provide order. Instead we do:
|
|
*
|
|
* 1) smp_store_release(X->on_cpu, 0)
|
|
* 2) smp_cond_load_acquire(!X->on_cpu)
|
|
*
|
|
* Example:
|
|
*
|
|
* CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule)
|
|
*
|
|
* LOCK rq(0)->lock LOCK X->pi_lock
|
|
* dequeue X
|
|
* sched-out X
|
|
* smp_store_release(X->on_cpu, 0);
|
|
*
|
|
* smp_cond_load_acquire(&X->on_cpu, !VAL);
|
|
* X->state = WAKING
|
|
* set_task_cpu(X,2)
|
|
*
|
|
* LOCK rq(2)->lock
|
|
* enqueue X
|
|
* X->state = RUNNING
|
|
* UNLOCK rq(2)->lock
|
|
*
|
|
* LOCK rq(2)->lock // orders against CPU1
|
|
* sched-out Z
|
|
* sched-in X
|
|
* UNLOCK rq(2)->lock
|
|
*
|
|
* UNLOCK X->pi_lock
|
|
* UNLOCK rq(0)->lock
|
|
*
|
|
*
|
|
* However; for wakeups there is a second guarantee we must provide, namely we
|
|
* must observe the state that lead to our wakeup. That is, not only must our
|
|
* task observe its own prior state, it must also observe the stores prior to
|
|
* its wakeup.
|
|
*
|
|
* This means that any means of doing remote wakeups must order the CPU doing
|
|
* the wakeup against the CPU the task is going to end up running on. This,
|
|
* however, is already required for the regular Program-Order guarantee above,
|
|
* since the waking CPU is the one issueing the ACQUIRE (smp_cond_load_acquire).
|
|
*
|
|
*/
|
|
|
|
/**
|
|
* try_to_wake_up - wake up a thread
|
|
* @p: the thread to be awakened
|
|
* @state: the mask of task states that can be woken
|
|
* @wake_flags: wake modifier flags (WF_*)
|
|
*
|
|
* If (@state & @p->state) @p->state = TASK_RUNNING.
|
|
*
|
|
* If the task was not queued/runnable, also place it back on a runqueue.
|
|
*
|
|
* Atomic against schedule() which would dequeue a task, also see
|
|
* set_current_state().
|
|
*
|
|
* Return: %true if @p->state changes (an actual wakeup was done),
|
|
* %false otherwise.
|
|
*/
|
|
static int
|
|
try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
|
|
{
|
|
unsigned long flags;
|
|
int cpu, success = 0;
|
|
|
|
/*
|
|
* If we are going to wake up a thread waiting for CONDITION we
|
|
* need to ensure that CONDITION=1 done by the caller can not be
|
|
* reordered with p->state check below. This pairs with mb() in
|
|
* set_current_state() the waiting thread does.
|
|
*/
|
|
smp_mb__before_spinlock();
|
|
raw_spin_lock_irqsave(&p->pi_lock, flags);
|
|
if (!(p->state & state))
|
|
goto out;
|
|
|
|
trace_sched_waking(p);
|
|
|
|
/* We're going to change ->state: */
|
|
success = 1;
|
|
cpu = task_cpu(p);
|
|
|
|
/*
|
|
* Ensure we load p->on_rq _after_ p->state, otherwise it would
|
|
* be possible to, falsely, observe p->on_rq == 0 and get stuck
|
|
* in smp_cond_load_acquire() below.
|
|
*
|
|
* sched_ttwu_pending() try_to_wake_up()
|
|
* [S] p->on_rq = 1; [L] P->state
|
|
* UNLOCK rq->lock -----.
|
|
* \
|
|
* +--- RMB
|
|
* schedule() /
|
|
* LOCK rq->lock -----'
|
|
* UNLOCK rq->lock
|
|
*
|
|
* [task p]
|
|
* [S] p->state = UNINTERRUPTIBLE [L] p->on_rq
|
|
*
|
|
* Pairs with the UNLOCK+LOCK on rq->lock from the
|
|
* last wakeup of our task and the schedule that got our task
|
|
* current.
|
|
*/
|
|
smp_rmb();
|
|
if (p->on_rq && ttwu_remote(p, wake_flags))
|
|
goto stat;
|
|
|
|
#ifdef CONFIG_SMP
|
|
/*
|
|
* Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
|
|
* possible to, falsely, observe p->on_cpu == 0.
|
|
*
|
|
* One must be running (->on_cpu == 1) in order to remove oneself
|
|
* from the runqueue.
|
|
*
|
|
* [S] ->on_cpu = 1; [L] ->on_rq
|
|
* UNLOCK rq->lock
|
|
* RMB
|
|
* LOCK rq->lock
|
|
* [S] ->on_rq = 0; [L] ->on_cpu
|
|
*
|
|
* Pairs with the full barrier implied in the UNLOCK+LOCK on rq->lock
|
|
* from the consecutive calls to schedule(); the first switching to our
|
|
* task, the second putting it to sleep.
|
|
*/
|
|
smp_rmb();
|
|
|
|
/*
|
|
* If the owning (remote) CPU is still in the middle of schedule() with
|
|
* this task as prev, wait until its done referencing the task.
|
|
*
|
|
* Pairs with the smp_store_release() in finish_lock_switch().
|
|
*
|
|
* This ensures that tasks getting woken will be fully ordered against
|
|
* their previous state and preserve Program Order.
|
|
*/
|
|
smp_cond_load_acquire(&p->on_cpu, !VAL);
|
|
|
|
p->sched_contributes_to_load = !!task_contributes_to_load(p);
|
|
p->state = TASK_WAKING;
|
|
|
|
if (p->in_iowait) {
|
|
delayacct_blkio_end();
|
|
atomic_dec(&task_rq(p)->nr_iowait);
|
|
}
|
|
|
|
cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
|
|
if (task_cpu(p) != cpu) {
|
|
wake_flags |= WF_MIGRATED;
|
|
set_task_cpu(p, cpu);
|
|
}
|
|
|
|
#else /* CONFIG_SMP */
|
|
|
|
if (p->in_iowait) {
|
|
delayacct_blkio_end();
|
|
atomic_dec(&task_rq(p)->nr_iowait);
|
|
}
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
ttwu_queue(p, cpu, wake_flags);
|
|
stat:
|
|
ttwu_stat(p, cpu, wake_flags);
|
|
out:
|
|
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
|
|
|
|
return success;
|
|
}
|
|
|
|
/**
|
|
* try_to_wake_up_local - try to wake up a local task with rq lock held
|
|
* @p: the thread to be awakened
|
|
* @cookie: context's cookie for pinning
|
|
*
|
|
* Put @p on the run-queue if it's not already there. The caller must
|
|
* ensure that this_rq() is locked, @p is bound to this_rq() and not
|
|
* the current task.
|
|
*/
|
|
static void try_to_wake_up_local(struct task_struct *p, struct rq_flags *rf)
|
|
{
|
|
struct rq *rq = task_rq(p);
|
|
|
|
if (WARN_ON_ONCE(rq != this_rq()) ||
|
|
WARN_ON_ONCE(p == current))
|
|
return;
|
|
|
|
lockdep_assert_held(&rq->lock);
|
|
|
|
if (!raw_spin_trylock(&p->pi_lock)) {
|
|
/*
|
|
* This is OK, because current is on_cpu, which avoids it being
|
|
* picked for load-balance and preemption/IRQs are still
|
|
* disabled avoiding further scheduler activity on it and we've
|
|
* not yet picked a replacement task.
|
|
*/
|
|
rq_unlock(rq, rf);
|
|
raw_spin_lock(&p->pi_lock);
|
|
rq_relock(rq, rf);
|
|
}
|
|
|
|
if (!(p->state & TASK_NORMAL))
|
|
goto out;
|
|
|
|
trace_sched_waking(p);
|
|
|
|
if (!task_on_rq_queued(p)) {
|
|
if (p->in_iowait) {
|
|
delayacct_blkio_end();
|
|
atomic_dec(&rq->nr_iowait);
|
|
}
|
|
ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK);
|
|
}
|
|
|
|
ttwu_do_wakeup(rq, p, 0, rf);
|
|
ttwu_stat(p, smp_processor_id(), 0);
|
|
out:
|
|
raw_spin_unlock(&p->pi_lock);
|
|
}
|
|
|
|
/**
|
|
* wake_up_process - Wake up a specific process
|
|
* @p: The process to be woken up.
|
|
*
|
|
* Attempt to wake up the nominated process and move it to the set of runnable
|
|
* processes.
|
|
*
|
|
* Return: 1 if the process was woken up, 0 if it was already running.
|
|
*
|
|
* It may be assumed that this function implies a write memory barrier before
|
|
* changing the task state if and only if any tasks are woken up.
|
|
*/
|
|
int wake_up_process(struct task_struct *p)
|
|
{
|
|
return try_to_wake_up(p, TASK_NORMAL, 0);
|
|
}
|
|
EXPORT_SYMBOL(wake_up_process);
|
|
|
|
int wake_up_state(struct task_struct *p, unsigned int state)
|
|
{
|
|
return try_to_wake_up(p, state, 0);
|
|
}
|
|
|
|
/*
|
|
* This function clears the sched_dl_entity static params.
|
|
*/
|
|
void __dl_clear_params(struct task_struct *p)
|
|
{
|
|
struct sched_dl_entity *dl_se = &p->dl;
|
|
|
|
dl_se->dl_runtime = 0;
|
|
dl_se->dl_deadline = 0;
|
|
dl_se->dl_period = 0;
|
|
dl_se->flags = 0;
|
|
dl_se->dl_bw = 0;
|
|
|
|
dl_se->dl_throttled = 0;
|
|
dl_se->dl_yielded = 0;
|
|
}
|
|
|
|
/*
|
|
* Perform scheduler related setup for a newly forked process p.
|
|
* p is forked by current.
|
|
*
|
|
* __sched_fork() is basic setup used by init_idle() too:
|
|
*/
|
|
static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
|
|
{
|
|
p->on_rq = 0;
|
|
|
|
p->se.on_rq = 0;
|
|
p->se.exec_start = 0;
|
|
p->se.sum_exec_runtime = 0;
|
|
p->se.prev_sum_exec_runtime = 0;
|
|
p->se.nr_migrations = 0;
|
|
p->se.vruntime = 0;
|
|
INIT_LIST_HEAD(&p->se.group_node);
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
p->se.cfs_rq = NULL;
|
|
#endif
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
/* Even if schedstat is disabled, there should not be garbage */
|
|
memset(&p->se.statistics, 0, sizeof(p->se.statistics));
|
|
#endif
|
|
|
|
RB_CLEAR_NODE(&p->dl.rb_node);
|
|
init_dl_task_timer(&p->dl);
|
|
__dl_clear_params(p);
|
|
|
|
INIT_LIST_HEAD(&p->rt.run_list);
|
|
p->rt.timeout = 0;
|
|
p->rt.time_slice = sched_rr_timeslice;
|
|
p->rt.on_rq = 0;
|
|
p->rt.on_list = 0;
|
|
|
|
#ifdef CONFIG_PREEMPT_NOTIFIERS
|
|
INIT_HLIST_HEAD(&p->preempt_notifiers);
|
|
#endif
|
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
|
|
p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
|
|
p->mm->numa_scan_seq = 0;
|
|
}
|
|
|
|
if (clone_flags & CLONE_VM)
|
|
p->numa_preferred_nid = current->numa_preferred_nid;
|
|
else
|
|
p->numa_preferred_nid = -1;
|
|
|
|
p->node_stamp = 0ULL;
|
|
p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
|
|
p->numa_scan_period = sysctl_numa_balancing_scan_delay;
|
|
p->numa_work.next = &p->numa_work;
|
|
p->numa_faults = NULL;
|
|
p->last_task_numa_placement = 0;
|
|
p->last_sum_exec_runtime = 0;
|
|
|
|
p->numa_group = NULL;
|
|
#endif /* CONFIG_NUMA_BALANCING */
|
|
}
|
|
|
|
DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
|
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
|
|
void set_numabalancing_state(bool enabled)
|
|
{
|
|
if (enabled)
|
|
static_branch_enable(&sched_numa_balancing);
|
|
else
|
|
static_branch_disable(&sched_numa_balancing);
|
|
}
|
|
|
|
#ifdef CONFIG_PROC_SYSCTL
|
|
int sysctl_numa_balancing(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *lenp, loff_t *ppos)
|
|
{
|
|
struct ctl_table t;
|
|
int err;
|
|
int state = static_branch_likely(&sched_numa_balancing);
|
|
|
|
if (write && !capable(CAP_SYS_ADMIN))
|
|
return -EPERM;
|
|
|
|
t = *table;
|
|
t.data = &state;
|
|
err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
|
|
if (err < 0)
|
|
return err;
|
|
if (write)
|
|
set_numabalancing_state(state);
|
|
return err;
|
|
}
|
|
#endif
|
|
#endif
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
|
DEFINE_STATIC_KEY_FALSE(sched_schedstats);
|
|
static bool __initdata __sched_schedstats = false;
|
|
|
|
static void set_schedstats(bool enabled)
|
|
{
|
|
if (enabled)
|
|
static_branch_enable(&sched_schedstats);
|
|
else
|
|
static_branch_disable(&sched_schedstats);
|
|
}
|
|
|
|
void force_schedstat_enabled(void)
|
|
{
|
|
if (!schedstat_enabled()) {
|
|
pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n");
|
|
static_branch_enable(&sched_schedstats);
|
|
}
|
|
}
|
|
|
|
static int __init setup_schedstats(char *str)
|
|
{
|
|
int ret = 0;
|
|
if (!str)
|
|
goto out;
|
|
|
|
/*
|
|
* This code is called before jump labels have been set up, so we can't
|
|
* change the static branch directly just yet. Instead set a temporary
|
|
* variable so init_schedstats() can do it later.
|
|
*/
|
|
if (!strcmp(str, "enable")) {
|
|
__sched_schedstats = true;
|
|
ret = 1;
|
|
} else if (!strcmp(str, "disable")) {
|
|
__sched_schedstats = false;
|
|
ret = 1;
|
|
}
|
|
out:
|
|
if (!ret)
|
|
pr_warn("Unable to parse schedstats=\n");
|
|
|
|
return ret;
|
|
}
|
|
__setup("schedstats=", setup_schedstats);
|
|
|
|
static void __init init_schedstats(void)
|
|
{
|
|
set_schedstats(__sched_schedstats);
|
|
}
|
|
|
|
#ifdef CONFIG_PROC_SYSCTL
|
|
int sysctl_schedstats(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *lenp, loff_t *ppos)
|
|
{
|
|
struct ctl_table t;
|
|
int err;
|
|
int state = static_branch_likely(&sched_schedstats);
|
|
|
|
if (write && !capable(CAP_SYS_ADMIN))
|
|
return -EPERM;
|
|
|
|
t = *table;
|
|
t.data = &state;
|
|
err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
|
|
if (err < 0)
|
|
return err;
|
|
if (write)
|
|
set_schedstats(state);
|
|
return err;
|
|
}
|
|
#endif /* CONFIG_PROC_SYSCTL */
|
|
#else /* !CONFIG_SCHEDSTATS */
|
|
static inline void init_schedstats(void) {}
|
|
#endif /* CONFIG_SCHEDSTATS */
|
|
|
|
/*
|
|
* fork()/clone()-time setup:
|
|
*/
|
|
int sched_fork(unsigned long clone_flags, struct task_struct *p)
|
|
{
|
|
unsigned long flags;
|
|
int cpu = get_cpu();
|
|
|
|
__sched_fork(clone_flags, p);
|
|
/*
|
|
* We mark the process as NEW here. This guarantees that
|
|
* nobody will actually run it, and a signal or other external
|
|
* event cannot wake it up and insert it on the runqueue either.
|
|
*/
|
|
p->state = TASK_NEW;
|
|
|
|
/*
|
|
* Make sure we do not leak PI boosting priority to the child.
|
|
*/
|
|
p->prio = current->normal_prio;
|
|
|
|
/*
|
|
* Revert to default priority/policy on fork if requested.
|
|
*/
|
|
if (unlikely(p->sched_reset_on_fork)) {
|
|
if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
|
|
p->policy = SCHED_NORMAL;
|
|
p->static_prio = NICE_TO_PRIO(0);
|
|
p->rt_priority = 0;
|
|
} else if (PRIO_TO_NICE(p->static_prio) < 0)
|
|
p->static_prio = NICE_TO_PRIO(0);
|
|
|
|
p->prio = p->normal_prio = __normal_prio(p);
|
|
set_load_weight(p);
|
|
|
|
/*
|
|
* We don't need the reset flag anymore after the fork. It has
|
|
* fulfilled its duty:
|
|
*/
|
|
p->sched_reset_on_fork = 0;
|
|
}
|
|
|
|
if (dl_prio(p->prio)) {
|
|
put_cpu();
|
|
return -EAGAIN;
|
|
} else if (rt_prio(p->prio)) {
|
|
p->sched_class = &rt_sched_class;
|
|
} else {
|
|
p->sched_class = &fair_sched_class;
|
|
}
|
|
|
|
init_entity_runnable_average(&p->se);
|
|
|
|
/*
|
|
* The child is not yet in the pid-hash so no cgroup attach races,
|
|
* and the cgroup is pinned to this child due to cgroup_fork()
|
|
* is ran before sched_fork().
|
|
*
|
|
* Silence PROVE_RCU.
|
|
*/
|
|
raw_spin_lock_irqsave(&p->pi_lock, flags);
|
|
/*
|
|
* We're setting the CPU for the first time, we don't migrate,
|
|
* so use __set_task_cpu().
|
|
*/
|
|
__set_task_cpu(p, cpu);
|
|
if (p->sched_class->task_fork)
|
|
p->sched_class->task_fork(p);
|
|
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
|
|
|
|
#ifdef CONFIG_SCHED_INFO
|
|
if (likely(sched_info_on()))
|
|
memset(&p->sched_info, 0, sizeof(p->sched_info));
|
|
#endif
|
|
#if defined(CONFIG_SMP)
|
|
p->on_cpu = 0;
|
|
#endif
|
|
init_task_preempt_count(p);
|
|
#ifdef CONFIG_SMP
|
|
plist_node_init(&p->pushable_tasks, MAX_PRIO);
|
|
RB_CLEAR_NODE(&p->pushable_dl_tasks);
|
|
#endif
|
|
|
|
put_cpu();
|
|
return 0;
|
|
}
|
|
|
|
unsigned long to_ratio(u64 period, u64 runtime)
|
|
{
|
|
if (runtime == RUNTIME_INF)
|
|
return 1ULL << 20;
|
|
|
|
/*
|
|
* Doing this here saves a lot of checks in all
|
|
* the calling paths, and returning zero seems
|
|
* safe for them anyway.
|
|
*/
|
|
if (period == 0)
|
|
return 0;
|
|
|
|
return div64_u64(runtime << 20, period);
|
|
}
|
|
|
|
#ifdef CONFIG_SMP
|
|
inline struct dl_bw *dl_bw_of(int i)
|
|
{
|
|
RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(),
|
|
"sched RCU must be held");
|
|
return &cpu_rq(i)->rd->dl_bw;
|
|
}
|
|
|
|
static inline int dl_bw_cpus(int i)
|
|
{
|
|
struct root_domain *rd = cpu_rq(i)->rd;
|
|
int cpus = 0;
|
|
|
|
RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(),
|
|
"sched RCU must be held");
|
|
for_each_cpu_and(i, rd->span, cpu_active_mask)
|
|
cpus++;
|
|
|
|
return cpus;
|
|
}
|
|
#else
|
|
inline struct dl_bw *dl_bw_of(int i)
|
|
{
|
|
return &cpu_rq(i)->dl.dl_bw;
|
|
}
|
|
|
|
static inline int dl_bw_cpus(int i)
|
|
{
|
|
return 1;
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* We must be sure that accepting a new task (or allowing changing the
|
|
* parameters of an existing one) is consistent with the bandwidth
|
|
* constraints. If yes, this function also accordingly updates the currently
|
|
* allocated bandwidth to reflect the new situation.
|
|
*
|
|
* This function is called while holding p's rq->lock.
|
|
*
|
|
* XXX we should delay bw change until the task's 0-lag point, see
|
|
* __setparam_dl().
|
|
*/
|
|
static int dl_overflow(struct task_struct *p, int policy,
|
|
const struct sched_attr *attr)
|
|
{
|
|
|
|
struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
|
|
u64 period = attr->sched_period ?: attr->sched_deadline;
|
|
u64 runtime = attr->sched_runtime;
|
|
u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0;
|
|
int cpus, err = -1;
|
|
|
|
/* !deadline task may carry old deadline bandwidth */
|
|
if (new_bw == p->dl.dl_bw && task_has_dl_policy(p))
|
|
return 0;
|
|
|
|
/*
|
|
* Either if a task, enters, leave, or stays -deadline but changes
|
|
* its parameters, we may need to update accordingly the total
|
|
* allocated bandwidth of the container.
|
|
*/
|
|
raw_spin_lock(&dl_b->lock);
|
|
cpus = dl_bw_cpus(task_cpu(p));
|
|
if (dl_policy(policy) && !task_has_dl_policy(p) &&
|
|
!__dl_overflow(dl_b, cpus, 0, new_bw)) {
|
|
__dl_add(dl_b, new_bw);
|
|
err = 0;
|
|
} else if (dl_policy(policy) && task_has_dl_policy(p) &&
|
|
!__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
|
|
__dl_clear(dl_b, p->dl.dl_bw);
|
|
__dl_add(dl_b, new_bw);
|
|
err = 0;
|
|
} else if (!dl_policy(policy) && task_has_dl_policy(p)) {
|
|
__dl_clear(dl_b, p->dl.dl_bw);
|
|
err = 0;
|
|
}
|
|
raw_spin_unlock(&dl_b->lock);
|
|
|
|
return err;
|
|
}
|
|
|
|
extern void init_dl_bw(struct dl_bw *dl_b);
|
|
|
|
/*
|
|
* wake_up_new_task - wake up a newly created task for the first time.
|
|
*
|
|
* This function will do some initial scheduler statistics housekeeping
|
|
* that must be done for every newly created context, then puts the task
|
|
* on the runqueue and wakes it.
|
|
*/
|
|
void wake_up_new_task(struct task_struct *p)
|
|
{
|
|
struct rq_flags rf;
|
|
struct rq *rq;
|
|
|
|
raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
|
|
p->state = TASK_RUNNING;
|
|
#ifdef CONFIG_SMP
|
|
/*
|
|
* Fork balancing, do it here and not earlier because:
|
|
* - cpus_allowed can change in the fork path
|
|
* - any previously selected CPU might disappear through hotplug
|
|
*
|
|
* Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq,
|
|
* as we're not fully set-up yet.
|
|
*/
|
|
__set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
|
|
#endif
|
|
rq = __task_rq_lock(p, &rf);
|
|
update_rq_clock(rq);
|
|
post_init_entity_util_avg(&p->se);
|
|
|
|
activate_task(rq, p, ENQUEUE_NOCLOCK);
|
|
p->on_rq = TASK_ON_RQ_QUEUED;
|
|
trace_sched_wakeup_new(p);
|
|
check_preempt_curr(rq, p, WF_FORK);
|
|
#ifdef CONFIG_SMP
|
|
if (p->sched_class->task_woken) {
|
|
/*
|
|
* Nothing relies on rq->lock after this, so its fine to
|
|
* drop it.
|
|
*/
|
|
rq_unpin_lock(rq, &rf);
|
|
p->sched_class->task_woken(rq, p);
|
|
rq_repin_lock(rq, &rf);
|
|
}
|
|
#endif
|
|
task_rq_unlock(rq, p, &rf);
|
|
}
|
|
|
|
#ifdef CONFIG_PREEMPT_NOTIFIERS
|
|
|
|
static struct static_key preempt_notifier_key = STATIC_KEY_INIT_FALSE;
|
|
|
|
void preempt_notifier_inc(void)
|
|
{
|
|
static_key_slow_inc(&preempt_notifier_key);
|
|
}
|
|
EXPORT_SYMBOL_GPL(preempt_notifier_inc);
|
|
|
|
void preempt_notifier_dec(void)
|
|
{
|
|
static_key_slow_dec(&preempt_notifier_key);
|
|
}
|
|
EXPORT_SYMBOL_GPL(preempt_notifier_dec);
|
|
|
|
/**
|
|
* preempt_notifier_register - tell me when current is being preempted & rescheduled
|
|
* @notifier: notifier struct to register
|
|
*/
|
|
void preempt_notifier_register(struct preempt_notifier *notifier)
|
|
{
|
|
if (!static_key_false(&preempt_notifier_key))
|
|
WARN(1, "registering preempt_notifier while notifiers disabled\n");
|
|
|
|
hlist_add_head(¬ifier->link, ¤t->preempt_notifiers);
|
|
}
|
|
EXPORT_SYMBOL_GPL(preempt_notifier_register);
|
|
|
|
/**
|
|
* preempt_notifier_unregister - no longer interested in preemption notifications
|
|
* @notifier: notifier struct to unregister
|
|
*
|
|
* This is *not* safe to call from within a preemption notifier.
|
|
*/
|
|
void preempt_notifier_unregister(struct preempt_notifier *notifier)
|
|
{
|
|
hlist_del(¬ifier->link);
|
|
}
|
|
EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
|
|
|
|
static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
|
|
{
|
|
struct preempt_notifier *notifier;
|
|
|
|
hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
|
|
notifier->ops->sched_in(notifier, raw_smp_processor_id());
|
|
}
|
|
|
|
static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
|
|
{
|
|
if (static_key_false(&preempt_notifier_key))
|
|
__fire_sched_in_preempt_notifiers(curr);
|
|
}
|
|
|
|
static void
|
|
__fire_sched_out_preempt_notifiers(struct task_struct *curr,
|
|
struct task_struct *next)
|
|
{
|
|
struct preempt_notifier *notifier;
|
|
|
|
hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
|
|
notifier->ops->sched_out(notifier, next);
|
|
}
|
|
|
|
static __always_inline void
|
|
fire_sched_out_preempt_notifiers(struct task_struct *curr,
|
|
struct task_struct *next)
|
|
{
|
|
if (static_key_false(&preempt_notifier_key))
|
|
__fire_sched_out_preempt_notifiers(curr, next);
|
|
}
|
|
|
|
#else /* !CONFIG_PREEMPT_NOTIFIERS */
|
|
|
|
static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
|
|
{
|
|
}
|
|
|
|
static inline void
|
|
fire_sched_out_preempt_notifiers(struct task_struct *curr,
|
|
struct task_struct *next)
|
|
{
|
|
}
|
|
|
|
#endif /* CONFIG_PREEMPT_NOTIFIERS */
|
|
|
|
/**
|
|
* prepare_task_switch - prepare to switch tasks
|
|
* @rq: the runqueue preparing to switch
|
|
* @prev: the current task that is being switched out
|
|
* @next: the task we are going to switch to.
|
|
*
|
|
* This is called with the rq lock held and interrupts off. It must
|
|
* be paired with a subsequent finish_task_switch after the context
|
|
* switch.
|
|
*
|
|
* prepare_task_switch sets up locking and calls architecture specific
|
|
* hooks.
|
|
*/
|
|
static inline void
|
|
prepare_task_switch(struct rq *rq, struct task_struct *prev,
|
|
struct task_struct *next)
|
|
{
|
|
sched_info_switch(rq, prev, next);
|
|
perf_event_task_sched_out(prev, next);
|
|
fire_sched_out_preempt_notifiers(prev, next);
|
|
prepare_lock_switch(rq, next);
|
|
prepare_arch_switch(next);
|
|
}
|
|
|
|
/**
|
|
* finish_task_switch - clean up after a task-switch
|
|
* @prev: the thread we just switched away from.
|
|
*
|
|
* finish_task_switch must be called after the context switch, paired
|
|
* with a prepare_task_switch call before the context switch.
|
|
* finish_task_switch will reconcile locking set up by prepare_task_switch,
|
|
* and do any other architecture-specific cleanup actions.
|
|
*
|
|
* Note that we may have delayed dropping an mm in context_switch(). If
|
|
* so, we finish that here outside of the runqueue lock. (Doing it
|
|
* with the lock held can cause deadlocks; see schedule() for
|
|
* details.)
|
|
*
|
|
* The context switch have flipped the stack from under us and restored the
|
|
* local variables which were saved when this task called schedule() in the
|
|
* past. prev == current is still correct but we need to recalculate this_rq
|
|
* because prev may have moved to another CPU.
|
|
*/
|
|
static struct rq *finish_task_switch(struct task_struct *prev)
|
|
__releases(rq->lock)
|
|
{
|
|
struct rq *rq = this_rq();
|
|
struct mm_struct *mm = rq->prev_mm;
|
|
long prev_state;
|
|
|
|
/*
|
|
* The previous task will have left us with a preempt_count of 2
|
|
* because it left us after:
|
|
*
|
|
* schedule()
|
|
* preempt_disable(); // 1
|
|
* __schedule()
|
|
* raw_spin_lock_irq(&rq->lock) // 2
|
|
*
|
|
* Also, see FORK_PREEMPT_COUNT.
|
|
*/
|
|
if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
|
|
"corrupted preempt_count: %s/%d/0x%x\n",
|
|
current->comm, current->pid, preempt_count()))
|
|
preempt_count_set(FORK_PREEMPT_COUNT);
|
|
|
|
rq->prev_mm = NULL;
|
|
|
|
/*
|
|
* A task struct has one reference for the use as "current".
|
|
* If a task dies, then it sets TASK_DEAD in tsk->state and calls
|
|
* schedule one last time. The schedule call will never return, and
|
|
* the scheduled task must drop that reference.
|
|
*
|
|
* We must observe prev->state before clearing prev->on_cpu (in
|
|
* finish_lock_switch), otherwise a concurrent wakeup can get prev
|
|
* running on another CPU and we could rave with its RUNNING -> DEAD
|
|
* transition, resulting in a double drop.
|
|
*/
|
|
prev_state = prev->state;
|
|
vtime_task_switch(prev);
|
|
perf_event_task_sched_in(prev, current);
|
|
finish_lock_switch(rq, prev);
|
|
finish_arch_post_lock_switch();
|
|
|
|
fire_sched_in_preempt_notifiers(current);
|
|
if (mm)
|
|
mmdrop(mm);
|
|
if (unlikely(prev_state == TASK_DEAD)) {
|
|
if (prev->sched_class->task_dead)
|
|
prev->sched_class->task_dead(prev);
|
|
|
|
/*
|
|
* Remove function-return probe instances associated with this
|
|
* task and put them back on the free list.
|
|
*/
|
|
kprobe_flush_task(prev);
|
|
|
|
/* Task is done with its stack. */
|
|
put_task_stack(prev);
|
|
|
|
put_task_struct(prev);
|
|
}
|
|
|
|
tick_nohz_task_switch();
|
|
return rq;
|
|
}
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
/* rq->lock is NOT held, but preemption is disabled */
|
|
static void __balance_callback(struct rq *rq)
|
|
{
|
|
struct callback_head *head, *next;
|
|
void (*func)(struct rq *rq);
|
|
unsigned long flags;
|
|
|
|
raw_spin_lock_irqsave(&rq->lock, flags);
|
|
head = rq->balance_callback;
|
|
rq->balance_callback = NULL;
|
|
while (head) {
|
|
func = (void (*)(struct rq *))head->func;
|
|
next = head->next;
|
|
head->next = NULL;
|
|
head = next;
|
|
|
|
func(rq);
|
|
}
|
|
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
|
}
|
|
|
|
static inline void balance_callback(struct rq *rq)
|
|
{
|
|
if (unlikely(rq->balance_callback))
|
|
__balance_callback(rq);
|
|
}
|
|
|
|
#else
|
|
|
|
static inline void balance_callback(struct rq *rq)
|
|
{
|
|
}
|
|
|
|
#endif
|
|
|
|
/**
|
|
* schedule_tail - first thing a freshly forked thread must call.
|
|
* @prev: the thread we just switched away from.
|
|
*/
|
|
asmlinkage __visible void schedule_tail(struct task_struct *prev)
|
|
__releases(rq->lock)
|
|
{
|
|
struct rq *rq;
|
|
|
|
/*
|
|
* New tasks start with FORK_PREEMPT_COUNT, see there and
|
|
* finish_task_switch() for details.
|
|
*
|
|
* finish_task_switch() will drop rq->lock() and lower preempt_count
|
|
* and the preempt_enable() will end up enabling preemption (on
|
|
* PREEMPT_COUNT kernels).
|
|
*/
|
|
|
|
rq = finish_task_switch(prev);
|
|
balance_callback(rq);
|
|
preempt_enable();
|
|
|
|
if (current->set_child_tid)
|
|
put_user(task_pid_vnr(current), current->set_child_tid);
|
|
}
|
|
|
|
/*
|
|
* context_switch - switch to the new MM and the new thread's register state.
|
|
*/
|
|
static __always_inline struct rq *
|
|
context_switch(struct rq *rq, struct task_struct *prev,
|
|
struct task_struct *next, struct rq_flags *rf)
|
|
{
|
|
struct mm_struct *mm, *oldmm;
|
|
|
|
prepare_task_switch(rq, prev, next);
|
|
|
|
mm = next->mm;
|
|
oldmm = prev->active_mm;
|
|
/*
|
|
* For paravirt, this is coupled with an exit in switch_to to
|
|
* combine the page table reload and the switch backend into
|
|
* one hypercall.
|
|
*/
|
|
arch_start_context_switch(prev);
|
|
|
|
if (!mm) {
|
|
next->active_mm = oldmm;
|
|
mmgrab(oldmm);
|
|
enter_lazy_tlb(oldmm, next);
|
|
} else
|
|
switch_mm_irqs_off(oldmm, mm, next);
|
|
|
|
if (!prev->mm) {
|
|
prev->active_mm = NULL;
|
|
rq->prev_mm = oldmm;
|
|
}
|
|
|
|
rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
|
|
|
|
/*
|
|
* Since the runqueue lock will be released by the next
|
|
* task (which is an invalid locking op but in the case
|
|
* of the scheduler it's an obvious special-case), so we
|
|
* do an early lockdep release here:
|
|
*/
|
|
rq_unpin_lock(rq, rf);
|
|
spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
|
|
|
|
/* Here we just switch the register state and the stack. */
|
|
switch_to(prev, next, prev);
|
|
barrier();
|
|
|
|
return finish_task_switch(prev);
|
|
}
|
|
|
|
/*
|
|
* nr_running and nr_context_switches:
|
|
*
|
|
* externally visible scheduler statistics: current number of runnable
|
|
* threads, total number of context switches performed since bootup.
|
|
*/
|
|
unsigned long nr_running(void)
|
|
{
|
|
unsigned long i, sum = 0;
|
|
|
|
for_each_online_cpu(i)
|
|
sum += cpu_rq(i)->nr_running;
|
|
|
|
return sum;
|
|
}
|
|
|
|
/*
|
|
* Check if only the current task is running on the CPU.
|
|
*
|
|
* Caution: this function does not check that the caller has disabled
|
|
* preemption, thus the result might have a time-of-check-to-time-of-use
|
|
* race. The caller is responsible to use it correctly, for example:
|
|
*
|
|
* - from a non-preemptable section (of course)
|
|
*
|
|
* - from a thread that is bound to a single CPU
|
|
*
|
|
* - in a loop with very short iterations (e.g. a polling loop)
|
|
*/
|
|
bool single_task_running(void)
|
|
{
|
|
return raw_rq()->nr_running == 1;
|
|
}
|
|
EXPORT_SYMBOL(single_task_running);
|
|
|
|
unsigned long long nr_context_switches(void)
|
|
{
|
|
int i;
|
|
unsigned long long sum = 0;
|
|
|
|
for_each_possible_cpu(i)
|
|
sum += cpu_rq(i)->nr_switches;
|
|
|
|
return sum;
|
|
}
|
|
|
|
/*
|
|
* IO-wait accounting, and how its mostly bollocks (on SMP).
|
|
*
|
|
* The idea behind IO-wait account is to account the idle time that we could
|
|
* have spend running if it were not for IO. That is, if we were to improve the
|
|
* storage performance, we'd have a proportional reduction in IO-wait time.
|
|
*
|
|
* This all works nicely on UP, where, when a task blocks on IO, we account
|
|
* idle time as IO-wait, because if the storage were faster, it could've been
|
|
* running and we'd not be idle.
|
|
*
|
|
* This has been extended to SMP, by doing the same for each CPU. This however
|
|
* is broken.
|
|
*
|
|
* Imagine for instance the case where two tasks block on one CPU, only the one
|
|
* CPU will have IO-wait accounted, while the other has regular idle. Even
|
|
* though, if the storage were faster, both could've ran at the same time,
|
|
* utilising both CPUs.
|
|
*
|
|
* This means, that when looking globally, the current IO-wait accounting on
|
|
* SMP is a lower bound, by reason of under accounting.
|
|
*
|
|
* Worse, since the numbers are provided per CPU, they are sometimes
|
|
* interpreted per CPU, and that is nonsensical. A blocked task isn't strictly
|
|
* associated with any one particular CPU, it can wake to another CPU than it
|
|
* blocked on. This means the per CPU IO-wait number is meaningless.
|
|
*
|
|
* Task CPU affinities can make all that even more 'interesting'.
|
|
*/
|
|
|
|
unsigned long nr_iowait(void)
|
|
{
|
|
unsigned long i, sum = 0;
|
|
|
|
for_each_possible_cpu(i)
|
|
sum += atomic_read(&cpu_rq(i)->nr_iowait);
|
|
|
|
return sum;
|
|
}
|
|
|
|
/*
|
|
* Consumers of these two interfaces, like for example the cpufreq menu
|
|
* governor are using nonsensical data. Boosting frequency for a CPU that has
|
|
* IO-wait which might not even end up running the task when it does become
|
|
* runnable.
|
|
*/
|
|
|
|
unsigned long nr_iowait_cpu(int cpu)
|
|
{
|
|
struct rq *this = cpu_rq(cpu);
|
|
return atomic_read(&this->nr_iowait);
|
|
}
|
|
|
|
void get_iowait_load(unsigned long *nr_waiters, unsigned long *load)
|
|
{
|
|
struct rq *rq = this_rq();
|
|
*nr_waiters = atomic_read(&rq->nr_iowait);
|
|
*load = rq->load.weight;
|
|
}
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
/*
|
|
* sched_exec - execve() is a valuable balancing opportunity, because at
|
|
* this point the task has the smallest effective memory and cache footprint.
|
|
*/
|
|
void sched_exec(void)
|
|
{
|
|
struct task_struct *p = current;
|
|
unsigned long flags;
|
|
int dest_cpu;
|
|
|
|
raw_spin_lock_irqsave(&p->pi_lock, flags);
|
|
dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
|
|
if (dest_cpu == smp_processor_id())
|
|
goto unlock;
|
|
|
|
if (likely(cpu_active(dest_cpu))) {
|
|
struct migration_arg arg = { p, dest_cpu };
|
|
|
|
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
|
|
stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
|
|
return;
|
|
}
|
|
unlock:
|
|
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
|
|
}
|
|
|
|
#endif
|
|
|
|
DEFINE_PER_CPU(struct kernel_stat, kstat);
|
|
DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
|
|
|
|
EXPORT_PER_CPU_SYMBOL(kstat);
|
|
EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
|
|
|
|
/*
|
|
* The function fair_sched_class.update_curr accesses the struct curr
|
|
* and its field curr->exec_start; when called from task_sched_runtime(),
|
|
* we observe a high rate of cache misses in practice.
|
|
* Prefetching this data results in improved performance.
|
|
*/
|
|
static inline void prefetch_curr_exec_start(struct task_struct *p)
|
|
{
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
struct sched_entity *curr = (&p->se)->cfs_rq->curr;
|
|
#else
|
|
struct sched_entity *curr = (&task_rq(p)->cfs)->curr;
|
|
#endif
|
|
prefetch(curr);
|
|
prefetch(&curr->exec_start);
|
|
}
|
|
|
|
/*
|
|
* Return accounted runtime for the task.
|
|
* In case the task is currently running, return the runtime plus current's
|
|
* pending runtime that have not been accounted yet.
|
|
*/
|
|
unsigned long long task_sched_runtime(struct task_struct *p)
|
|
{
|
|
struct rq_flags rf;
|
|
struct rq *rq;
|
|
u64 ns;
|
|
|
|
#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
|
|
/*
|
|
* 64-bit doesn't need locks to atomically read a 64bit value.
|
|
* So we have a optimization chance when the task's delta_exec is 0.
|
|
* Reading ->on_cpu is racy, but this is ok.
|
|
*
|
|
* If we race with it leaving CPU, we'll take a lock. So we're correct.
|
|
* If we race with it entering CPU, unaccounted time is 0. This is
|
|
* indistinguishable from the read occurring a few cycles earlier.
|
|
* If we see ->on_cpu without ->on_rq, the task is leaving, and has
|
|
* been accounted, so we're correct here as well.
|
|
*/
|
|
if (!p->on_cpu || !task_on_rq_queued(p))
|
|
return p->se.sum_exec_runtime;
|
|
#endif
|
|
|
|
rq = task_rq_lock(p, &rf);
|
|
/*
|
|
* Must be ->curr _and_ ->on_rq. If dequeued, we would
|
|
* project cycles that may never be accounted to this
|
|
* thread, breaking clock_gettime().
|
|
*/
|
|
if (task_current(rq, p) && task_on_rq_queued(p)) {
|
|
prefetch_curr_exec_start(p);
|
|
update_rq_clock(rq);
|
|
p->sched_class->update_curr(rq);
|
|
}
|
|
ns = p->se.sum_exec_runtime;
|
|
task_rq_unlock(rq, p, &rf);
|
|
|
|
return ns;
|
|
}
|
|
|
|
/*
|
|
* This function gets called by the timer code, with HZ frequency.
|
|
* We call it with interrupts disabled.
|
|
*/
|
|
void scheduler_tick(void)
|
|
{
|
|
int cpu = smp_processor_id();
|
|
struct rq *rq = cpu_rq(cpu);
|
|
struct task_struct *curr = rq->curr;
|
|
struct rq_flags rf;
|
|
|
|
sched_clock_tick();
|
|
|
|
rq_lock(rq, &rf);
|
|
|
|
update_rq_clock(rq);
|
|
curr->sched_class->task_tick(rq, curr, 0);
|
|
cpu_load_update_active(rq);
|
|
calc_global_load_tick(rq);
|
|
|
|
rq_unlock(rq, &rf);
|
|
|
|
perf_event_task_tick();
|
|
|
|
#ifdef CONFIG_SMP
|
|
rq->idle_balance = idle_cpu(cpu);
|
|
trigger_load_balance(rq);
|
|
#endif
|
|
rq_last_tick_reset(rq);
|
|
}
|
|
|
|
#ifdef CONFIG_NO_HZ_FULL
|
|
/**
|
|
* scheduler_tick_max_deferment
|
|
*
|
|
* Keep at least one tick per second when a single
|
|
* active task is running because the scheduler doesn't
|
|
* yet completely support full dynticks environment.
|
|
*
|
|
* This makes sure that uptime, CFS vruntime, load
|
|
* balancing, etc... continue to move forward, even
|
|
* with a very low granularity.
|
|
*
|
|
* Return: Maximum deferment in nanoseconds.
|
|
*/
|
|
u64 scheduler_tick_max_deferment(void)
|
|
{
|
|
struct rq *rq = this_rq();
|
|
unsigned long next, now = READ_ONCE(jiffies);
|
|
|
|
next = rq->last_sched_tick + HZ;
|
|
|
|
if (time_before_eq(next, now))
|
|
return 0;
|
|
|
|
return jiffies_to_nsecs(next - now);
|
|
}
|
|
#endif
|
|
|
|
#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
|
|
defined(CONFIG_PREEMPT_TRACER))
|
|
/*
|
|
* If the value passed in is equal to the current preempt count
|
|
* then we just disabled preemption. Start timing the latency.
|
|
*/
|
|
static inline void preempt_latency_start(int val)
|
|
{
|
|
if (preempt_count() == val) {
|
|
unsigned long ip = get_lock_parent_ip();
|
|
#ifdef CONFIG_DEBUG_PREEMPT
|
|
current->preempt_disable_ip = ip;
|
|
#endif
|
|
trace_preempt_off(CALLER_ADDR0, ip);
|
|
}
|
|
}
|
|
|
|
void preempt_count_add(int val)
|
|
{
|
|
#ifdef CONFIG_DEBUG_PREEMPT
|
|
/*
|
|
* Underflow?
|
|
*/
|
|
if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
|
|
return;
|
|
#endif
|
|
__preempt_count_add(val);
|
|
#ifdef CONFIG_DEBUG_PREEMPT
|
|
/*
|
|
* Spinlock count overflowing soon?
|
|
*/
|
|
DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
|
|
PREEMPT_MASK - 10);
|
|
#endif
|
|
preempt_latency_start(val);
|
|
}
|
|
EXPORT_SYMBOL(preempt_count_add);
|
|
NOKPROBE_SYMBOL(preempt_count_add);
|
|
|
|
/*
|
|
* If the value passed in equals to the current preempt count
|
|
* then we just enabled preemption. Stop timing the latency.
|
|
*/
|
|
static inline void preempt_latency_stop(int val)
|
|
{
|
|
if (preempt_count() == val)
|
|
trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
|
|
}
|
|
|
|
void preempt_count_sub(int val)
|
|
{
|
|
#ifdef CONFIG_DEBUG_PREEMPT
|
|
/*
|
|
* Underflow?
|
|
*/
|
|
if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
|
|
return;
|
|
/*
|
|
* Is the spinlock portion underflowing?
|
|
*/
|
|
if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
|
|
!(preempt_count() & PREEMPT_MASK)))
|
|
return;
|
|
#endif
|
|
|
|
preempt_latency_stop(val);
|
|
__preempt_count_sub(val);
|
|
}
|
|
EXPORT_SYMBOL(preempt_count_sub);
|
|
NOKPROBE_SYMBOL(preempt_count_sub);
|
|
|
|
#else
|
|
static inline void preempt_latency_start(int val) { }
|
|
static inline void preempt_latency_stop(int val) { }
|
|
#endif
|
|
|
|
static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
|
|
{
|
|
#ifdef CONFIG_DEBUG_PREEMPT
|
|
return p->preempt_disable_ip;
|
|
#else
|
|
return 0;
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Print scheduling while atomic bug:
|
|
*/
|
|
static noinline void __schedule_bug(struct task_struct *prev)
|
|
{
|
|
/* Save this before calling printk(), since that will clobber it */
|
|
unsigned long preempt_disable_ip = get_preempt_disable_ip(current);
|
|
|
|
if (oops_in_progress)
|
|
return;
|
|
|
|
printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
|
|
prev->comm, prev->pid, preempt_count());
|
|
|
|
debug_show_held_locks(prev);
|
|
print_modules();
|
|
if (irqs_disabled())
|
|
print_irqtrace_events(prev);
|
|
if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
|
|
&& in_atomic_preempt_off()) {
|
|
pr_err("Preemption disabled at:");
|
|
print_ip_sym(preempt_disable_ip);
|
|
pr_cont("\n");
|
|
}
|
|
if (panic_on_warn)
|
|
panic("scheduling while atomic\n");
|
|
|
|
dump_stack();
|
|
add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
|
|
}
|
|
|
|
/*
|
|
* Various schedule()-time debugging checks and statistics:
|
|
*/
|
|
static inline void schedule_debug(struct task_struct *prev)
|
|
{
|
|
#ifdef CONFIG_SCHED_STACK_END_CHECK
|
|
if (task_stack_end_corrupted(prev))
|
|
panic("corrupted stack end detected inside scheduler\n");
|
|
#endif
|
|
|
|
if (unlikely(in_atomic_preempt_off())) {
|
|
__schedule_bug(prev);
|
|
preempt_count_set(PREEMPT_DISABLED);
|
|
}
|
|
rcu_sleep_check();
|
|
|
|
profile_hit(SCHED_PROFILING, __builtin_return_address(0));
|
|
|
|
schedstat_inc(this_rq()->sched_count);
|
|
}
|
|
|
|
/*
|
|
* Pick up the highest-prio task:
|
|
*/
|
|
static inline struct task_struct *
|
|
pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
|
|
{
|
|
const struct sched_class *class;
|
|
struct task_struct *p;
|
|
|
|
/*
|
|
* Optimization: we know that if all tasks are in the fair class we can
|
|
* call that function directly, but only if the @prev task wasn't of a
|
|
* higher scheduling class, because otherwise those loose the
|
|
* opportunity to pull in more work from other CPUs.
|
|
*/
|
|
if (likely((prev->sched_class == &idle_sched_class ||
|
|
prev->sched_class == &fair_sched_class) &&
|
|
rq->nr_running == rq->cfs.h_nr_running)) {
|
|
|
|
p = fair_sched_class.pick_next_task(rq, prev, rf);
|
|
if (unlikely(p == RETRY_TASK))
|
|
goto again;
|
|
|
|
/* Assumes fair_sched_class->next == idle_sched_class */
|
|
if (unlikely(!p))
|
|
p = idle_sched_class.pick_next_task(rq, prev, rf);
|
|
|
|
return p;
|
|
}
|
|
|
|
again:
|
|
for_each_class(class) {
|
|
p = class->pick_next_task(rq, prev, rf);
|
|
if (p) {
|
|
if (unlikely(p == RETRY_TASK))
|
|
goto again;
|
|
return p;
|
|
}
|
|
}
|
|
|
|
/* The idle class should always have a runnable task: */
|
|
BUG();
|
|
}
|
|
|
|
/*
|
|
* __schedule() is the main scheduler function.
|
|
*
|
|
* The main means of driving the scheduler and thus entering this function are:
|
|
*
|
|
* 1. Explicit blocking: mutex, semaphore, waitqueue, etc.
|
|
*
|
|
* 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
|
|
* paths. For example, see arch/x86/entry_64.S.
|
|
*
|
|
* To drive preemption between tasks, the scheduler sets the flag in timer
|
|
* interrupt handler scheduler_tick().
|
|
*
|
|
* 3. Wakeups don't really cause entry into schedule(). They add a
|
|
* task to the run-queue and that's it.
|
|
*
|
|
* Now, if the new task added to the run-queue preempts the current
|
|
* task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
|
|
* called on the nearest possible occasion:
|
|
*
|
|
* - If the kernel is preemptible (CONFIG_PREEMPT=y):
|
|
*
|
|
* - in syscall or exception context, at the next outmost
|
|
* preempt_enable(). (this might be as soon as the wake_up()'s
|
|
* spin_unlock()!)
|
|
*
|
|
* - in IRQ context, return from interrupt-handler to
|
|
* preemptible context
|
|
*
|
|
* - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
|
|
* then at the next:
|
|
*
|
|
* - cond_resched() call
|
|
* - explicit schedule() call
|
|
* - return from syscall or exception to user-space
|
|
* - return from interrupt-handler to user-space
|
|
*
|
|
* WARNING: must be called with preemption disabled!
|
|
*/
|
|
static void __sched notrace __schedule(bool preempt)
|
|
{
|
|
struct task_struct *prev, *next;
|
|
unsigned long *switch_count;
|
|
struct rq_flags rf;
|
|
struct rq *rq;
|
|
int cpu;
|
|
|
|
cpu = smp_processor_id();
|
|
rq = cpu_rq(cpu);
|
|
prev = rq->curr;
|
|
|
|
schedule_debug(prev);
|
|
|
|
if (sched_feat(HRTICK))
|
|
hrtick_clear(rq);
|
|
|
|
local_irq_disable();
|
|
rcu_note_context_switch(preempt);
|
|
|
|
/*
|
|
* Make sure that signal_pending_state()->signal_pending() below
|
|
* can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
|
|
* done by the caller to avoid the race with signal_wake_up().
|
|
*/
|
|
smp_mb__before_spinlock();
|
|
rq_lock(rq, &rf);
|
|
|
|
/* Promote REQ to ACT */
|
|
rq->clock_update_flags <<= 1;
|
|
update_rq_clock(rq);
|
|
|
|
switch_count = &prev->nivcsw;
|
|
if (!preempt && prev->state) {
|
|
if (unlikely(signal_pending_state(prev->state, prev))) {
|
|
prev->state = TASK_RUNNING;
|
|
} else {
|
|
deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK);
|
|
prev->on_rq = 0;
|
|
|
|
if (prev->in_iowait) {
|
|
atomic_inc(&rq->nr_iowait);
|
|
delayacct_blkio_start();
|
|
}
|
|
|
|
/*
|
|
* If a worker went to sleep, notify and ask workqueue
|
|
* whether it wants to wake up a task to maintain
|
|
* concurrency.
|
|
*/
|
|
if (prev->flags & PF_WQ_WORKER) {
|
|
struct task_struct *to_wakeup;
|
|
|
|
to_wakeup = wq_worker_sleeping(prev);
|
|
if (to_wakeup)
|
|
try_to_wake_up_local(to_wakeup, &rf);
|
|
}
|
|
}
|
|
switch_count = &prev->nvcsw;
|
|
}
|
|
|
|
next = pick_next_task(rq, prev, &rf);
|
|
clear_tsk_need_resched(prev);
|
|
clear_preempt_need_resched();
|
|
|
|
if (likely(prev != next)) {
|
|
rq->nr_switches++;
|
|
rq->curr = next;
|
|
++*switch_count;
|
|
|
|
trace_sched_switch(preempt, prev, next);
|
|
|
|
/* Also unlocks the rq: */
|
|
rq = context_switch(rq, prev, next, &rf);
|
|
} else {
|
|
rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
|
|
rq_unlock_irq(rq, &rf);
|
|
}
|
|
|
|
balance_callback(rq);
|
|
}
|
|
|
|
void __noreturn do_task_dead(void)
|
|
{
|
|
/*
|
|
* The setting of TASK_RUNNING by try_to_wake_up() may be delayed
|
|
* when the following two conditions become true.
|
|
* - There is race condition of mmap_sem (It is acquired by
|
|
* exit_mm()), and
|
|
* - SMI occurs before setting TASK_RUNINNG.
|
|
* (or hypervisor of virtual machine switches to other guest)
|
|
* As a result, we may become TASK_RUNNING after becoming TASK_DEAD
|
|
*
|
|
* To avoid it, we have to wait for releasing tsk->pi_lock which
|
|
* is held by try_to_wake_up()
|
|
*/
|
|
smp_mb();
|
|
raw_spin_unlock_wait(¤t->pi_lock);
|
|
|
|
/* Causes final put_task_struct in finish_task_switch(): */
|
|
__set_current_state(TASK_DEAD);
|
|
|
|
/* Tell freezer to ignore us: */
|
|
current->flags |= PF_NOFREEZE;
|
|
|
|
__schedule(false);
|
|
BUG();
|
|
|
|
/* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */
|
|
for (;;)
|
|
cpu_relax();
|
|
}
|
|
|
|
static inline void sched_submit_work(struct task_struct *tsk)
|
|
{
|
|
if (!tsk->state || tsk_is_pi_blocked(tsk))
|
|
return;
|
|
/*
|
|
* If we are going to sleep and we have plugged IO queued,
|
|
* make sure to submit it to avoid deadlocks.
|
|
*/
|
|
if (blk_needs_flush_plug(tsk))
|
|
blk_schedule_flush_plug(tsk);
|
|
}
|
|
|
|
asmlinkage __visible void __sched schedule(void)
|
|
{
|
|
struct task_struct *tsk = current;
|
|
|
|
sched_submit_work(tsk);
|
|
do {
|
|
preempt_disable();
|
|
__schedule(false);
|
|
sched_preempt_enable_no_resched();
|
|
} while (need_resched());
|
|
}
|
|
EXPORT_SYMBOL(schedule);
|
|
|
|
/*
|
|
* synchronize_rcu_tasks() makes sure that no task is stuck in preempted
|
|
* state (have scheduled out non-voluntarily) by making sure that all
|
|
* tasks have either left the run queue or have gone into user space.
|
|
* As idle tasks do not do either, they must not ever be preempted
|
|
* (schedule out non-voluntarily).
|
|
*
|
|
* schedule_idle() is similar to schedule_preempt_disable() except that it
|
|
* never enables preemption because it does not call sched_submit_work().
|
|
*/
|
|
void __sched schedule_idle(void)
|
|
{
|
|
/*
|
|
* As this skips calling sched_submit_work(), which the idle task does
|
|
* regardless because that function is a nop when the task is in a
|
|
* TASK_RUNNING state, make sure this isn't used someplace that the
|
|
* current task can be in any other state. Note, idle is always in the
|
|
* TASK_RUNNING state.
|
|
*/
|
|
WARN_ON_ONCE(current->state);
|
|
do {
|
|
__schedule(false);
|
|
} while (need_resched());
|
|
}
|
|
|
|
#ifdef CONFIG_CONTEXT_TRACKING
|
|
asmlinkage __visible void __sched schedule_user(void)
|
|
{
|
|
/*
|
|
* If we come here after a random call to set_need_resched(),
|
|
* or we have been woken up remotely but the IPI has not yet arrived,
|
|
* we haven't yet exited the RCU idle mode. Do it here manually until
|
|
* we find a better solution.
|
|
*
|
|
* NB: There are buggy callers of this function. Ideally we
|
|
* should warn if prev_state != CONTEXT_USER, but that will trigger
|
|
* too frequently to make sense yet.
|
|
*/
|
|
enum ctx_state prev_state = exception_enter();
|
|
schedule();
|
|
exception_exit(prev_state);
|
|
}
|
|
#endif
|
|
|
|
/**
|
|
* schedule_preempt_disabled - called with preemption disabled
|
|
*
|
|
* Returns with preemption disabled. Note: preempt_count must be 1
|
|
*/
|
|
void __sched schedule_preempt_disabled(void)
|
|
{
|
|
sched_preempt_enable_no_resched();
|
|
schedule();
|
|
preempt_disable();
|
|
}
|
|
|
|
static void __sched notrace preempt_schedule_common(void)
|
|
{
|
|
do {
|
|
/*
|
|
* Because the function tracer can trace preempt_count_sub()
|
|
* and it also uses preempt_enable/disable_notrace(), if
|
|
* NEED_RESCHED is set, the preempt_enable_notrace() called
|
|
* by the function tracer will call this function again and
|
|
* cause infinite recursion.
|
|
*
|
|
* Preemption must be disabled here before the function
|
|
* tracer can trace. Break up preempt_disable() into two
|
|
* calls. One to disable preemption without fear of being
|
|
* traced. The other to still record the preemption latency,
|
|
* which can also be traced by the function tracer.
|
|
*/
|
|
preempt_disable_notrace();
|
|
preempt_latency_start(1);
|
|
__schedule(true);
|
|
preempt_latency_stop(1);
|
|
preempt_enable_no_resched_notrace();
|
|
|
|
/*
|
|
* Check again in case we missed a preemption opportunity
|
|
* between schedule and now.
|
|
*/
|
|
} while (need_resched());
|
|
}
|
|
|
|
#ifdef CONFIG_PREEMPT
|
|
/*
|
|
* this is the entry point to schedule() from in-kernel preemption
|
|
* off of preempt_enable. Kernel preemptions off return from interrupt
|
|
* occur there and call schedule directly.
|
|
*/
|
|
asmlinkage __visible void __sched notrace preempt_schedule(void)
|
|
{
|
|
/*
|
|
* If there is a non-zero preempt_count or interrupts are disabled,
|
|
* we do not want to preempt the current task. Just return..
|
|
*/
|
|
if (likely(!preemptible()))
|
|
return;
|
|
|
|
preempt_schedule_common();
|
|
}
|
|
NOKPROBE_SYMBOL(preempt_schedule);
|
|
EXPORT_SYMBOL(preempt_schedule);
|
|
|
|
/**
|
|
* preempt_schedule_notrace - preempt_schedule called by tracing
|
|
*
|
|
* The tracing infrastructure uses preempt_enable_notrace to prevent
|
|
* recursion and tracing preempt enabling caused by the tracing
|
|
* infrastructure itself. But as tracing can happen in areas coming
|
|
* from userspace or just about to enter userspace, a preempt enable
|
|
* can occur before user_exit() is called. This will cause the scheduler
|
|
* to be called when the system is still in usermode.
|
|
*
|
|
* To prevent this, the preempt_enable_notrace will use this function
|
|
* instead of preempt_schedule() to exit user context if needed before
|
|
* calling the scheduler.
|
|
*/
|
|
asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
|
|
{
|
|
enum ctx_state prev_ctx;
|
|
|
|
if (likely(!preemptible()))
|
|
return;
|
|
|
|
do {
|
|
/*
|
|
* Because the function tracer can trace preempt_count_sub()
|
|
* and it also uses preempt_enable/disable_notrace(), if
|
|
* NEED_RESCHED is set, the preempt_enable_notrace() called
|
|
* by the function tracer will call this function again and
|
|
* cause infinite recursion.
|
|
*
|
|
* Preemption must be disabled here before the function
|
|
* tracer can trace. Break up preempt_disable() into two
|
|
* calls. One to disable preemption without fear of being
|
|
* traced. The other to still record the preemption latency,
|
|
* which can also be traced by the function tracer.
|
|
*/
|
|
preempt_disable_notrace();
|
|
preempt_latency_start(1);
|
|
/*
|
|
* Needs preempt disabled in case user_exit() is traced
|
|
* and the tracer calls preempt_enable_notrace() causing
|
|
* an infinite recursion.
|
|
*/
|
|
prev_ctx = exception_enter();
|
|
__schedule(true);
|
|
exception_exit(prev_ctx);
|
|
|
|
preempt_latency_stop(1);
|
|
preempt_enable_no_resched_notrace();
|
|
} while (need_resched());
|
|
}
|
|
EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
|
|
|
|
#endif /* CONFIG_PREEMPT */
|
|
|
|
/*
|
|
* this is the entry point to schedule() from kernel preemption
|
|
* off of irq context.
|
|
* Note, that this is called and return with irqs disabled. This will
|
|
* protect us against recursive calling from irq.
|
|
*/
|
|
asmlinkage __visible void __sched preempt_schedule_irq(void)
|
|
{
|
|
enum ctx_state prev_state;
|
|
|
|
/* Catch callers which need to be fixed */
|
|
BUG_ON(preempt_count() || !irqs_disabled());
|
|
|
|
prev_state = exception_enter();
|
|
|
|
do {
|
|
preempt_disable();
|
|
local_irq_enable();
|
|
__schedule(true);
|
|
local_irq_disable();
|
|
sched_preempt_enable_no_resched();
|
|
} while (need_resched());
|
|
|
|
exception_exit(prev_state);
|
|
}
|
|
|
|
int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
|
|
void *key)
|
|
{
|
|
return try_to_wake_up(curr->private, mode, wake_flags);
|
|
}
|
|
EXPORT_SYMBOL(default_wake_function);
|
|
|
|
#ifdef CONFIG_RT_MUTEXES
|
|
|
|
static inline int __rt_effective_prio(struct task_struct *pi_task, int prio)
|
|
{
|
|
if (pi_task)
|
|
prio = min(prio, pi_task->prio);
|
|
|
|
return prio;
|
|
}
|
|
|
|
static inline int rt_effective_prio(struct task_struct *p, int prio)
|
|
{
|
|
struct task_struct *pi_task = rt_mutex_get_top_task(p);
|
|
|
|
return __rt_effective_prio(pi_task, prio);
|
|
}
|
|
|
|
/*
|
|
* rt_mutex_setprio - set the current priority of a task
|
|
* @p: task to boost
|
|
* @pi_task: donor task
|
|
*
|
|
* This function changes the 'effective' priority of a task. It does
|
|
* not touch ->normal_prio like __setscheduler().
|
|
*
|
|
* Used by the rt_mutex code to implement priority inheritance
|
|
* logic. Call site only calls if the priority of the task changed.
|
|
*/
|
|
void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
|
|
{
|
|
int prio, oldprio, queued, running, queue_flag =
|
|
DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
|
|
const struct sched_class *prev_class;
|
|
struct rq_flags rf;
|
|
struct rq *rq;
|
|
|
|
/* XXX used to be waiter->prio, not waiter->task->prio */
|
|
prio = __rt_effective_prio(pi_task, p->normal_prio);
|
|
|
|
/*
|
|
* If nothing changed; bail early.
|
|
*/
|
|
if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio))
|
|
return;
|
|
|
|
rq = __task_rq_lock(p, &rf);
|
|
update_rq_clock(rq);
|
|
/*
|
|
* Set under pi_lock && rq->lock, such that the value can be used under
|
|
* either lock.
|
|
*
|
|
* Note that there is loads of tricky to make this pointer cache work
|
|
* right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to
|
|
* ensure a task is de-boosted (pi_task is set to NULL) before the
|
|
* task is allowed to run again (and can exit). This ensures the pointer
|
|
* points to a blocked task -- which guaratees the task is present.
|
|
*/
|
|
p->pi_top_task = pi_task;
|
|
|
|
/*
|
|
* For FIFO/RR we only need to set prio, if that matches we're done.
|
|
*/
|
|
if (prio == p->prio && !dl_prio(prio))
|
|
goto out_unlock;
|
|
|
|
/*
|
|
* Idle task boosting is a nono in general. There is one
|
|
* exception, when PREEMPT_RT and NOHZ is active:
|
|
*
|
|
* The idle task calls get_next_timer_interrupt() and holds
|
|
* the timer wheel base->lock on the CPU and another CPU wants
|
|
* to access the timer (probably to cancel it). We can safely
|
|
* ignore the boosting request, as the idle CPU runs this code
|
|
* with interrupts disabled and will complete the lock
|
|
* protected section without being interrupted. So there is no
|
|
* real need to boost.
|
|
*/
|
|
if (unlikely(p == rq->idle)) {
|
|
WARN_ON(p != rq->curr);
|
|
WARN_ON(p->pi_blocked_on);
|
|
goto out_unlock;
|
|
}
|
|
|
|
trace_sched_pi_setprio(p, pi_task);
|
|
oldprio = p->prio;
|
|
|
|
if (oldprio == prio)
|
|
queue_flag &= ~DEQUEUE_MOVE;
|
|
|
|
prev_class = p->sched_class;
|
|
queued = task_on_rq_queued(p);
|
|
running = task_current(rq, p);
|
|
if (queued)
|
|
dequeue_task(rq, p, queue_flag);
|
|
if (running)
|
|
put_prev_task(rq, p);
|
|
|
|
/*
|
|
* Boosting condition are:
|
|
* 1. -rt task is running and holds mutex A
|
|
* --> -dl task blocks on mutex A
|
|
*
|
|
* 2. -dl task is running and holds mutex A
|
|
* --> -dl task blocks on mutex A and could preempt the
|
|
* running task
|
|
*/
|
|
if (dl_prio(prio)) {
|
|
if (!dl_prio(p->normal_prio) ||
|
|
(pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
|
|
p->dl.dl_boosted = 1;
|
|
queue_flag |= ENQUEUE_REPLENISH;
|
|
} else
|
|
p->dl.dl_boosted = 0;
|
|
p->sched_class = &dl_sched_class;
|
|
} else if (rt_prio(prio)) {
|
|
if (dl_prio(oldprio))
|
|
p->dl.dl_boosted = 0;
|
|
if (oldprio < prio)
|
|
queue_flag |= ENQUEUE_HEAD;
|
|
p->sched_class = &rt_sched_class;
|
|
} else {
|
|
if (dl_prio(oldprio))
|
|
p->dl.dl_boosted = 0;
|
|
if (rt_prio(oldprio))
|
|
p->rt.timeout = 0;
|
|
p->sched_class = &fair_sched_class;
|
|
}
|
|
|
|
p->prio = prio;
|
|
|
|
if (queued)
|
|
enqueue_task(rq, p, queue_flag);
|
|
if (running)
|
|
set_curr_task(rq, p);
|
|
|
|
check_class_changed(rq, p, prev_class, oldprio);
|
|
out_unlock:
|
|
/* Avoid rq from going away on us: */
|
|
preempt_disable();
|
|
__task_rq_unlock(rq, &rf);
|
|
|
|
balance_callback(rq);
|
|
preempt_enable();
|
|
}
|
|
#else
|
|
static inline int rt_effective_prio(struct task_struct *p, int prio)
|
|
{
|
|
return prio;
|
|
}
|
|
#endif
|
|
|
|
void set_user_nice(struct task_struct *p, long nice)
|
|
{
|
|
bool queued, running;
|
|
int old_prio, delta;
|
|
struct rq_flags rf;
|
|
struct rq *rq;
|
|
|
|
if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
|
|
return;
|
|
/*
|
|
* We have to be careful, if called from sys_setpriority(),
|
|
* the task might be in the middle of scheduling on another CPU.
|
|
*/
|
|
rq = task_rq_lock(p, &rf);
|
|
update_rq_clock(rq);
|
|
|
|
/*
|
|
* The RT priorities are set via sched_setscheduler(), but we still
|
|
* allow the 'normal' nice value to be set - but as expected
|
|
* it wont have any effect on scheduling until the task is
|
|
* SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
|
|
*/
|
|
if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
|
|
p->static_prio = NICE_TO_PRIO(nice);
|
|
goto out_unlock;
|
|
}
|
|
queued = task_on_rq_queued(p);
|
|
running = task_current(rq, p);
|
|
if (queued)
|
|
dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
|
|
if (running)
|
|
put_prev_task(rq, p);
|
|
|
|
p->static_prio = NICE_TO_PRIO(nice);
|
|
set_load_weight(p);
|
|
old_prio = p->prio;
|
|
p->prio = effective_prio(p);
|
|
delta = p->prio - old_prio;
|
|
|
|
if (queued) {
|
|
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
|
|
/*
|
|
* If the task increased its priority or is running and
|
|
* lowered its priority, then reschedule its CPU:
|
|
*/
|
|
if (delta < 0 || (delta > 0 && task_running(rq, p)))
|
|
resched_curr(rq);
|
|
}
|
|
if (running)
|
|
set_curr_task(rq, p);
|
|
out_unlock:
|
|
task_rq_unlock(rq, p, &rf);
|
|
}
|
|
EXPORT_SYMBOL(set_user_nice);
|
|
|
|
/*
|
|
* can_nice - check if a task can reduce its nice value
|
|
* @p: task
|
|
* @nice: nice value
|
|
*/
|
|
int can_nice(const struct task_struct *p, const int nice)
|
|
{
|
|
/* Convert nice value [19,-20] to rlimit style value [1,40]: */
|
|
int nice_rlim = nice_to_rlimit(nice);
|
|
|
|
return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
|
|
capable(CAP_SYS_NICE));
|
|
}
|
|
|
|
#ifdef __ARCH_WANT_SYS_NICE
|
|
|
|
/*
|
|
* sys_nice - change the priority of the current process.
|
|
* @increment: priority increment
|
|
*
|
|
* sys_setpriority is a more generic, but much slower function that
|
|
* does similar things.
|
|
*/
|
|
SYSCALL_DEFINE1(nice, int, increment)
|
|
{
|
|
long nice, retval;
|
|
|
|
/*
|
|
* Setpriority might change our priority at the same moment.
|
|
* We don't have to worry. Conceptually one call occurs first
|
|
* and we have a single winner.
|
|
*/
|
|
increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
|
|
nice = task_nice(current) + increment;
|
|
|
|
nice = clamp_val(nice, MIN_NICE, MAX_NICE);
|
|
if (increment < 0 && !can_nice(current, nice))
|
|
return -EPERM;
|
|
|
|
retval = security_task_setnice(current, nice);
|
|
if (retval)
|
|
return retval;
|
|
|
|
set_user_nice(current, nice);
|
|
return 0;
|
|
}
|
|
|
|
#endif
|
|
|
|
/**
|
|
* task_prio - return the priority value of a given task.
|
|
* @p: the task in question.
|
|
*
|
|
* Return: The priority value as seen by users in /proc.
|
|
* RT tasks are offset by -200. Normal tasks are centered
|
|
* around 0, value goes from -16 to +15.
|
|
*/
|
|
int task_prio(const struct task_struct *p)
|
|
{
|
|
return p->prio - MAX_RT_PRIO;
|
|
}
|
|
|
|
/**
|
|
* idle_cpu - is a given CPU idle currently?
|
|
* @cpu: the processor in question.
|
|
*
|
|
* Return: 1 if the CPU is currently idle. 0 otherwise.
|
|
*/
|
|
int idle_cpu(int cpu)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
if (rq->curr != rq->idle)
|
|
return 0;
|
|
|
|
if (rq->nr_running)
|
|
return 0;
|
|
|
|
#ifdef CONFIG_SMP
|
|
if (!llist_empty(&rq->wake_list))
|
|
return 0;
|
|
#endif
|
|
|
|
return 1;
|
|
}
|
|
|
|
/**
|
|
* idle_task - return the idle task for a given CPU.
|
|
* @cpu: the processor in question.
|
|
*
|
|
* Return: The idle task for the CPU @cpu.
|
|
*/
|
|
struct task_struct *idle_task(int cpu)
|
|
{
|
|
return cpu_rq(cpu)->idle;
|
|
}
|
|
|
|
/**
|
|
* find_process_by_pid - find a process with a matching PID value.
|
|
* @pid: the pid in question.
|
|
*
|
|
* The task of @pid, if found. %NULL otherwise.
|
|
*/
|
|
static struct task_struct *find_process_by_pid(pid_t pid)
|
|
{
|
|
return pid ? find_task_by_vpid(pid) : current;
|
|
}
|
|
|
|
/*
|
|
* This function initializes the sched_dl_entity of a newly becoming
|
|
* SCHED_DEADLINE task.
|
|
*
|
|
* Only the static values are considered here, the actual runtime and the
|
|
* absolute deadline will be properly calculated when the task is enqueued
|
|
* for the first time with its new policy.
|
|
*/
|
|
static void
|
|
__setparam_dl(struct task_struct *p, const struct sched_attr *attr)
|
|
{
|
|
struct sched_dl_entity *dl_se = &p->dl;
|
|
|
|
dl_se->dl_runtime = attr->sched_runtime;
|
|
dl_se->dl_deadline = attr->sched_deadline;
|
|
dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
|
|
dl_se->flags = attr->sched_flags;
|
|
dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
|
|
|
|
/*
|
|
* Changing the parameters of a task is 'tricky' and we're not doing
|
|
* the correct thing -- also see task_dead_dl() and switched_from_dl().
|
|
*
|
|
* What we SHOULD do is delay the bandwidth release until the 0-lag
|
|
* point. This would include retaining the task_struct until that time
|
|
* and change dl_overflow() to not immediately decrement the current
|
|
* amount.
|
|
*
|
|
* Instead we retain the current runtime/deadline and let the new
|
|
* parameters take effect after the current reservation period lapses.
|
|
* This is safe (albeit pessimistic) because the 0-lag point is always
|
|
* before the current scheduling deadline.
|
|
*
|
|
* We can still have temporary overloads because we do not delay the
|
|
* change in bandwidth until that time; so admission control is
|
|
* not on the safe side. It does however guarantee tasks will never
|
|
* consume more than promised.
|
|
*/
|
|
}
|
|
|
|
/*
|
|
* sched_setparam() passes in -1 for its policy, to let the functions
|
|
* it calls know not to change it.
|
|
*/
|
|
#define SETPARAM_POLICY -1
|
|
|
|
static void __setscheduler_params(struct task_struct *p,
|
|
const struct sched_attr *attr)
|
|
{
|
|
int policy = attr->sched_policy;
|
|
|
|
if (policy == SETPARAM_POLICY)
|
|
policy = p->policy;
|
|
|
|
p->policy = policy;
|
|
|
|
if (dl_policy(policy))
|
|
__setparam_dl(p, attr);
|
|
else if (fair_policy(policy))
|
|
p->static_prio = NICE_TO_PRIO(attr->sched_nice);
|
|
|
|
/*
|
|
* __sched_setscheduler() ensures attr->sched_priority == 0 when
|
|
* !rt_policy. Always setting this ensures that things like
|
|
* getparam()/getattr() don't report silly values for !rt tasks.
|
|
*/
|
|
p->rt_priority = attr->sched_priority;
|
|
p->normal_prio = normal_prio(p);
|
|
set_load_weight(p);
|
|
}
|
|
|
|
/* Actually do priority change: must hold pi & rq lock. */
|
|
static void __setscheduler(struct rq *rq, struct task_struct *p,
|
|
const struct sched_attr *attr, bool keep_boost)
|
|
{
|
|
__setscheduler_params(p, attr);
|
|
|
|
/*
|
|
* Keep a potential priority boosting if called from
|
|
* sched_setscheduler().
|
|
*/
|
|
p->prio = normal_prio(p);
|
|
if (keep_boost)
|
|
p->prio = rt_effective_prio(p, p->prio);
|
|
|
|
if (dl_prio(p->prio))
|
|
p->sched_class = &dl_sched_class;
|
|
else if (rt_prio(p->prio))
|
|
p->sched_class = &rt_sched_class;
|
|
else
|
|
p->sched_class = &fair_sched_class;
|
|
}
|
|
|
|
static void
|
|
__getparam_dl(struct task_struct *p, struct sched_attr *attr)
|
|
{
|
|
struct sched_dl_entity *dl_se = &p->dl;
|
|
|
|
attr->sched_priority = p->rt_priority;
|
|
attr->sched_runtime = dl_se->dl_runtime;
|
|
attr->sched_deadline = dl_se->dl_deadline;
|
|
attr->sched_period = dl_se->dl_period;
|
|
attr->sched_flags = dl_se->flags;
|
|
}
|
|
|
|
/*
|
|
* This function validates the new parameters of a -deadline task.
|
|
* We ask for the deadline not being zero, and greater or equal
|
|
* than the runtime, as well as the period of being zero or
|
|
* greater than deadline. Furthermore, we have to be sure that
|
|
* user parameters are above the internal resolution of 1us (we
|
|
* check sched_runtime only since it is always the smaller one) and
|
|
* below 2^63 ns (we have to check both sched_deadline and
|
|
* sched_period, as the latter can be zero).
|
|
*/
|
|
static bool
|
|
__checkparam_dl(const struct sched_attr *attr)
|
|
{
|
|
/* deadline != 0 */
|
|
if (attr->sched_deadline == 0)
|
|
return false;
|
|
|
|
/*
|
|
* Since we truncate DL_SCALE bits, make sure we're at least
|
|
* that big.
|
|
*/
|
|
if (attr->sched_runtime < (1ULL << DL_SCALE))
|
|
return false;
|
|
|
|
/*
|
|
* Since we use the MSB for wrap-around and sign issues, make
|
|
* sure it's not set (mind that period can be equal to zero).
|
|
*/
|
|
if (attr->sched_deadline & (1ULL << 63) ||
|
|
attr->sched_period & (1ULL << 63))
|
|
return false;
|
|
|
|
/* runtime <= deadline <= period (if period != 0) */
|
|
if ((attr->sched_period != 0 &&
|
|
attr->sched_period < attr->sched_deadline) ||
|
|
attr->sched_deadline < attr->sched_runtime)
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
|
|
/*
|
|
* Check the target process has a UID that matches the current process's:
|
|
*/
|
|
static bool check_same_owner(struct task_struct *p)
|
|
{
|
|
const struct cred *cred = current_cred(), *pcred;
|
|
bool match;
|
|
|
|
rcu_read_lock();
|
|
pcred = __task_cred(p);
|
|
match = (uid_eq(cred->euid, pcred->euid) ||
|
|
uid_eq(cred->euid, pcred->uid));
|
|
rcu_read_unlock();
|
|
return match;
|
|
}
|
|
|
|
static bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr)
|
|
{
|
|
struct sched_dl_entity *dl_se = &p->dl;
|
|
|
|
if (dl_se->dl_runtime != attr->sched_runtime ||
|
|
dl_se->dl_deadline != attr->sched_deadline ||
|
|
dl_se->dl_period != attr->sched_period ||
|
|
dl_se->flags != attr->sched_flags)
|
|
return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
static int __sched_setscheduler(struct task_struct *p,
|
|
const struct sched_attr *attr,
|
|
bool user, bool pi)
|
|
{
|
|
int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
|
|
MAX_RT_PRIO - 1 - attr->sched_priority;
|
|
int retval, oldprio, oldpolicy = -1, queued, running;
|
|
int new_effective_prio, policy = attr->sched_policy;
|
|
const struct sched_class *prev_class;
|
|
struct rq_flags rf;
|
|
int reset_on_fork;
|
|
int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
|
|
struct rq *rq;
|
|
|
|
/* May grab non-irq protected spin_locks: */
|
|
BUG_ON(in_interrupt());
|
|
recheck:
|
|
/* Double check policy once rq lock held: */
|
|
if (policy < 0) {
|
|
reset_on_fork = p->sched_reset_on_fork;
|
|
policy = oldpolicy = p->policy;
|
|
} else {
|
|
reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
|
|
|
|
if (!valid_policy(policy))
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK))
|
|
return -EINVAL;
|
|
|
|
/*
|
|
* Valid priorities for SCHED_FIFO and SCHED_RR are
|
|
* 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
|
|
* SCHED_BATCH and SCHED_IDLE is 0.
|
|
*/
|
|
if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
|
|
(!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
|
|
return -EINVAL;
|
|
if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
|
|
(rt_policy(policy) != (attr->sched_priority != 0)))
|
|
return -EINVAL;
|
|
|
|
/*
|
|
* Allow unprivileged RT tasks to decrease priority:
|
|
*/
|
|
if (user && !capable(CAP_SYS_NICE)) {
|
|
if (fair_policy(policy)) {
|
|
if (attr->sched_nice < task_nice(p) &&
|
|
!can_nice(p, attr->sched_nice))
|
|
return -EPERM;
|
|
}
|
|
|
|
if (rt_policy(policy)) {
|
|
unsigned long rlim_rtprio =
|
|
task_rlimit(p, RLIMIT_RTPRIO);
|
|
|
|
/* Can't set/change the rt policy: */
|
|
if (policy != p->policy && !rlim_rtprio)
|
|
return -EPERM;
|
|
|
|
/* Can't increase priority: */
|
|
if (attr->sched_priority > p->rt_priority &&
|
|
attr->sched_priority > rlim_rtprio)
|
|
return -EPERM;
|
|
}
|
|
|
|
/*
|
|
* Can't set/change SCHED_DEADLINE policy at all for now
|
|
* (safest behavior); in the future we would like to allow
|
|
* unprivileged DL tasks to increase their relative deadline
|
|
* or reduce their runtime (both ways reducing utilization)
|
|
*/
|
|
if (dl_policy(policy))
|
|
return -EPERM;
|
|
|
|
/*
|
|
* Treat SCHED_IDLE as nice 20. Only allow a switch to
|
|
* SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
|
|
*/
|
|
if (idle_policy(p->policy) && !idle_policy(policy)) {
|
|
if (!can_nice(p, task_nice(p)))
|
|
return -EPERM;
|
|
}
|
|
|
|
/* Can't change other user's priorities: */
|
|
if (!check_same_owner(p))
|
|
return -EPERM;
|
|
|
|
/* Normal users shall not reset the sched_reset_on_fork flag: */
|
|
if (p->sched_reset_on_fork && !reset_on_fork)
|
|
return -EPERM;
|
|
}
|
|
|
|
if (user) {
|
|
retval = security_task_setscheduler(p);
|
|
if (retval)
|
|
return retval;
|
|
}
|
|
|
|
/*
|
|
* Make sure no PI-waiters arrive (or leave) while we are
|
|
* changing the priority of the task:
|
|
*
|
|
* To be able to change p->policy safely, the appropriate
|
|
* runqueue lock must be held.
|
|
*/
|
|
rq = task_rq_lock(p, &rf);
|
|
update_rq_clock(rq);
|
|
|
|
/*
|
|
* Changing the policy of the stop threads its a very bad idea:
|
|
*/
|
|
if (p == rq->stop) {
|
|
task_rq_unlock(rq, p, &rf);
|
|
return -EINVAL;
|
|
}
|
|
|
|
/*
|
|
* If not changing anything there's no need to proceed further,
|
|
* but store a possible modification of reset_on_fork.
|
|
*/
|
|
if (unlikely(policy == p->policy)) {
|
|
if (fair_policy(policy) && attr->sched_nice != task_nice(p))
|
|
goto change;
|
|
if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
|
|
goto change;
|
|
if (dl_policy(policy) && dl_param_changed(p, attr))
|
|
goto change;
|
|
|
|
p->sched_reset_on_fork = reset_on_fork;
|
|
task_rq_unlock(rq, p, &rf);
|
|
return 0;
|
|
}
|
|
change:
|
|
|
|
if (user) {
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
/*
|
|
* Do not allow realtime tasks into groups that have no runtime
|
|
* assigned.
|
|
*/
|
|
if (rt_bandwidth_enabled() && rt_policy(policy) &&
|
|
task_group(p)->rt_bandwidth.rt_runtime == 0 &&
|
|
!task_group_is_autogroup(task_group(p))) {
|
|
task_rq_unlock(rq, p, &rf);
|
|
return -EPERM;
|
|
}
|
|
#endif
|
|
#ifdef CONFIG_SMP
|
|
if (dl_bandwidth_enabled() && dl_policy(policy)) {
|
|
cpumask_t *span = rq->rd->span;
|
|
|
|
/*
|
|
* Don't allow tasks with an affinity mask smaller than
|
|
* the entire root_domain to become SCHED_DEADLINE. We
|
|
* will also fail if there's no bandwidth available.
|
|
*/
|
|
if (!cpumask_subset(span, &p->cpus_allowed) ||
|
|
rq->rd->dl_bw.bw == 0) {
|
|
task_rq_unlock(rq, p, &rf);
|
|
return -EPERM;
|
|
}
|
|
}
|
|
#endif
|
|
}
|
|
|
|
/* Re-check policy now with rq lock held: */
|
|
if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
|
|
policy = oldpolicy = -1;
|
|
task_rq_unlock(rq, p, &rf);
|
|
goto recheck;
|
|
}
|
|
|
|
/*
|
|
* If setscheduling to SCHED_DEADLINE (or changing the parameters
|
|
* of a SCHED_DEADLINE task) we need to check if enough bandwidth
|
|
* is available.
|
|
*/
|
|
if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) {
|
|
task_rq_unlock(rq, p, &rf);
|
|
return -EBUSY;
|
|
}
|
|
|
|
p->sched_reset_on_fork = reset_on_fork;
|
|
oldprio = p->prio;
|
|
|
|
if (pi) {
|
|
/*
|
|
* Take priority boosted tasks into account. If the new
|
|
* effective priority is unchanged, we just store the new
|
|
* normal parameters and do not touch the scheduler class and
|
|
* the runqueue. This will be done when the task deboost
|
|
* itself.
|
|
*/
|
|
new_effective_prio = rt_effective_prio(p, newprio);
|
|
if (new_effective_prio == oldprio)
|
|
queue_flags &= ~DEQUEUE_MOVE;
|
|
}
|
|
|
|
queued = task_on_rq_queued(p);
|
|
running = task_current(rq, p);
|
|
if (queued)
|
|
dequeue_task(rq, p, queue_flags);
|
|
if (running)
|
|
put_prev_task(rq, p);
|
|
|
|
prev_class = p->sched_class;
|
|
__setscheduler(rq, p, attr, pi);
|
|
|
|
if (queued) {
|
|
/*
|
|
* We enqueue to tail when the priority of a task is
|
|
* increased (user space view).
|
|
*/
|
|
if (oldprio < p->prio)
|
|
queue_flags |= ENQUEUE_HEAD;
|
|
|
|
enqueue_task(rq, p, queue_flags);
|
|
}
|
|
if (running)
|
|
set_curr_task(rq, p);
|
|
|
|
check_class_changed(rq, p, prev_class, oldprio);
|
|
|
|
/* Avoid rq from going away on us: */
|
|
preempt_disable();
|
|
task_rq_unlock(rq, p, &rf);
|
|
|
|
if (pi)
|
|
rt_mutex_adjust_pi(p);
|
|
|
|
/* Run balance callbacks after we've adjusted the PI chain: */
|
|
balance_callback(rq);
|
|
preempt_enable();
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int _sched_setscheduler(struct task_struct *p, int policy,
|
|
const struct sched_param *param, bool check)
|
|
{
|
|
struct sched_attr attr = {
|
|
.sched_policy = policy,
|
|
.sched_priority = param->sched_priority,
|
|
.sched_nice = PRIO_TO_NICE(p->static_prio),
|
|
};
|
|
|
|
/* Fixup the legacy SCHED_RESET_ON_FORK hack. */
|
|
if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
|
|
attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
|
|
policy &= ~SCHED_RESET_ON_FORK;
|
|
attr.sched_policy = policy;
|
|
}
|
|
|
|
return __sched_setscheduler(p, &attr, check, true);
|
|
}
|
|
/**
|
|
* sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
|
|
* @p: the task in question.
|
|
* @policy: new policy.
|
|
* @param: structure containing the new RT priority.
|
|
*
|
|
* Return: 0 on success. An error code otherwise.
|
|
*
|
|
* NOTE that the task may be already dead.
|
|
*/
|
|
int sched_setscheduler(struct task_struct *p, int policy,
|
|
const struct sched_param *param)
|
|
{
|
|
return _sched_setscheduler(p, policy, param, true);
|
|
}
|
|
EXPORT_SYMBOL_GPL(sched_setscheduler);
|
|
|
|
int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
|
|
{
|
|
return __sched_setscheduler(p, attr, true, true);
|
|
}
|
|
EXPORT_SYMBOL_GPL(sched_setattr);
|
|
|
|
/**
|
|
* sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
|
|
* @p: the task in question.
|
|
* @policy: new policy.
|
|
* @param: structure containing the new RT priority.
|
|
*
|
|
* Just like sched_setscheduler, only don't bother checking if the
|
|
* current context has permission. For example, this is needed in
|
|
* stop_machine(): we create temporary high priority worker threads,
|
|
* but our caller might not have that capability.
|
|
*
|
|
* Return: 0 on success. An error code otherwise.
|
|
*/
|
|
int sched_setscheduler_nocheck(struct task_struct *p, int policy,
|
|
const struct sched_param *param)
|
|
{
|
|
return _sched_setscheduler(p, policy, param, false);
|
|
}
|
|
EXPORT_SYMBOL_GPL(sched_setscheduler_nocheck);
|
|
|
|
static int
|
|
do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
|
|
{
|
|
struct sched_param lparam;
|
|
struct task_struct *p;
|
|
int retval;
|
|
|
|
if (!param || pid < 0)
|
|
return -EINVAL;
|
|
if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
|
|
return -EFAULT;
|
|
|
|
rcu_read_lock();
|
|
retval = -ESRCH;
|
|
p = find_process_by_pid(pid);
|
|
if (p != NULL)
|
|
retval = sched_setscheduler(p, policy, &lparam);
|
|
rcu_read_unlock();
|
|
|
|
return retval;
|
|
}
|
|
|
|
/*
|
|
* Mimics kernel/events/core.c perf_copy_attr().
|
|
*/
|
|
static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *attr)
|
|
{
|
|
u32 size;
|
|
int ret;
|
|
|
|
if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0))
|
|
return -EFAULT;
|
|
|
|
/* Zero the full structure, so that a short copy will be nice: */
|
|
memset(attr, 0, sizeof(*attr));
|
|
|
|
ret = get_user(size, &uattr->size);
|
|
if (ret)
|
|
return ret;
|
|
|
|
/* Bail out on silly large: */
|
|
if (size > PAGE_SIZE)
|
|
goto err_size;
|
|
|
|
/* ABI compatibility quirk: */
|
|
if (!size)
|
|
size = SCHED_ATTR_SIZE_VER0;
|
|
|
|
if (size < SCHED_ATTR_SIZE_VER0)
|
|
goto err_size;
|
|
|
|
/*
|
|
* If we're handed a bigger struct than we know of,
|
|
* ensure all the unknown bits are 0 - i.e. new
|
|
* user-space does not rely on any kernel feature
|
|
* extensions we dont know about yet.
|
|
*/
|
|
if (size > sizeof(*attr)) {
|
|
unsigned char __user *addr;
|
|
unsigned char __user *end;
|
|
unsigned char val;
|
|
|
|
addr = (void __user *)uattr + sizeof(*attr);
|
|
end = (void __user *)uattr + size;
|
|
|
|
for (; addr < end; addr++) {
|
|
ret = get_user(val, addr);
|
|
if (ret)
|
|
return ret;
|
|
if (val)
|
|
goto err_size;
|
|
}
|
|
size = sizeof(*attr);
|
|
}
|
|
|
|
ret = copy_from_user(attr, uattr, size);
|
|
if (ret)
|
|
return -EFAULT;
|
|
|
|
/*
|
|
* XXX: Do we want to be lenient like existing syscalls; or do we want
|
|
* to be strict and return an error on out-of-bounds values?
|
|
*/
|
|
attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
|
|
|
|
return 0;
|
|
|
|
err_size:
|
|
put_user(sizeof(*attr), &uattr->size);
|
|
return -E2BIG;
|
|
}
|
|
|
|
/**
|
|
* sys_sched_setscheduler - set/change the scheduler policy and RT priority
|
|
* @pid: the pid in question.
|
|
* @policy: new policy.
|
|
* @param: structure containing the new RT priority.
|
|
*
|
|
* Return: 0 on success. An error code otherwise.
|
|
*/
|
|
SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param)
|
|
{
|
|
if (policy < 0)
|
|
return -EINVAL;
|
|
|
|
return do_sched_setscheduler(pid, policy, param);
|
|
}
|
|
|
|
/**
|
|
* sys_sched_setparam - set/change the RT priority of a thread
|
|
* @pid: the pid in question.
|
|
* @param: structure containing the new RT priority.
|
|
*
|
|
* Return: 0 on success. An error code otherwise.
|
|
*/
|
|
SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
|
|
{
|
|
return do_sched_setscheduler(pid, SETPARAM_POLICY, param);
|
|
}
|
|
|
|
/**
|
|
* sys_sched_setattr - same as above, but with extended sched_attr
|
|
* @pid: the pid in question.
|
|
* @uattr: structure containing the extended parameters.
|
|
* @flags: for future extension.
|
|
*/
|
|
SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
|
|
unsigned int, flags)
|
|
{
|
|
struct sched_attr attr;
|
|
struct task_struct *p;
|
|
int retval;
|
|
|
|
if (!uattr || pid < 0 || flags)
|
|
return -EINVAL;
|
|
|
|
retval = sched_copy_attr(uattr, &attr);
|
|
if (retval)
|
|
return retval;
|
|
|
|
if ((int)attr.sched_policy < 0)
|
|
return -EINVAL;
|
|
|
|
rcu_read_lock();
|
|
retval = -ESRCH;
|
|
p = find_process_by_pid(pid);
|
|
if (p != NULL)
|
|
retval = sched_setattr(p, &attr);
|
|
rcu_read_unlock();
|
|
|
|
return retval;
|
|
}
|
|
|
|
/**
|
|
* sys_sched_getscheduler - get the policy (scheduling class) of a thread
|
|
* @pid: the pid in question.
|
|
*
|
|
* Return: On success, the policy of the thread. Otherwise, a negative error
|
|
* code.
|
|
*/
|
|
SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
|
|
{
|
|
struct task_struct *p;
|
|
int retval;
|
|
|
|
if (pid < 0)
|
|
return -EINVAL;
|
|
|
|
retval = -ESRCH;
|
|
rcu_read_lock();
|
|
p = find_process_by_pid(pid);
|
|
if (p) {
|
|
retval = security_task_getscheduler(p);
|
|
if (!retval)
|
|
retval = p->policy
|
|
| (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
|
|
}
|
|
rcu_read_unlock();
|
|
return retval;
|
|
}
|
|
|
|
/**
|
|
* sys_sched_getparam - get the RT priority of a thread
|
|
* @pid: the pid in question.
|
|
* @param: structure containing the RT priority.
|
|
*
|
|
* Return: On success, 0 and the RT priority is in @param. Otherwise, an error
|
|
* code.
|
|
*/
|
|
SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
|
|
{
|
|
struct sched_param lp = { .sched_priority = 0 };
|
|
struct task_struct *p;
|
|
int retval;
|
|
|
|
if (!param || pid < 0)
|
|
return -EINVAL;
|
|
|
|
rcu_read_lock();
|
|
p = find_process_by_pid(pid);
|
|
retval = -ESRCH;
|
|
if (!p)
|
|
goto out_unlock;
|
|
|
|
retval = security_task_getscheduler(p);
|
|
if (retval)
|
|
goto out_unlock;
|
|
|
|
if (task_has_rt_policy(p))
|
|
lp.sched_priority = p->rt_priority;
|
|
rcu_read_unlock();
|
|
|
|
/*
|
|
* This one might sleep, we cannot do it with a spinlock held ...
|
|
*/
|
|
retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
|
|
|
|
return retval;
|
|
|
|
out_unlock:
|
|
rcu_read_unlock();
|
|
return retval;
|
|
}
|
|
|
|
static int sched_read_attr(struct sched_attr __user *uattr,
|
|
struct sched_attr *attr,
|
|
unsigned int usize)
|
|
{
|
|
int ret;
|
|
|
|
if (!access_ok(VERIFY_WRITE, uattr, usize))
|
|
return -EFAULT;
|
|
|
|
/*
|
|
* If we're handed a smaller struct than we know of,
|
|
* ensure all the unknown bits are 0 - i.e. old
|
|
* user-space does not get uncomplete information.
|
|
*/
|
|
if (usize < sizeof(*attr)) {
|
|
unsigned char *addr;
|
|
unsigned char *end;
|
|
|
|
addr = (void *)attr + usize;
|
|
end = (void *)attr + sizeof(*attr);
|
|
|
|
for (; addr < end; addr++) {
|
|
if (*addr)
|
|
return -EFBIG;
|
|
}
|
|
|
|
attr->size = usize;
|
|
}
|
|
|
|
ret = copy_to_user(uattr, attr, attr->size);
|
|
if (ret)
|
|
return -EFAULT;
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* sys_sched_getattr - similar to sched_getparam, but with sched_attr
|
|
* @pid: the pid in question.
|
|
* @uattr: structure containing the extended parameters.
|
|
* @size: sizeof(attr) for fwd/bwd comp.
|
|
* @flags: for future extension.
|
|
*/
|
|
SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
|
|
unsigned int, size, unsigned int, flags)
|
|
{
|
|
struct sched_attr attr = {
|
|
.size = sizeof(struct sched_attr),
|
|
};
|
|
struct task_struct *p;
|
|
int retval;
|
|
|
|
if (!uattr || pid < 0 || size > PAGE_SIZE ||
|
|
size < SCHED_ATTR_SIZE_VER0 || flags)
|
|
return -EINVAL;
|
|
|
|
rcu_read_lock();
|
|
p = find_process_by_pid(pid);
|
|
retval = -ESRCH;
|
|
if (!p)
|
|
goto out_unlock;
|
|
|
|
retval = security_task_getscheduler(p);
|
|
if (retval)
|
|
goto out_unlock;
|
|
|
|
attr.sched_policy = p->policy;
|
|
if (p->sched_reset_on_fork)
|
|
attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
|
|
if (task_has_dl_policy(p))
|
|
__getparam_dl(p, &attr);
|
|
else if (task_has_rt_policy(p))
|
|
attr.sched_priority = p->rt_priority;
|
|
else
|
|
attr.sched_nice = task_nice(p);
|
|
|
|
rcu_read_unlock();
|
|
|
|
retval = sched_read_attr(uattr, &attr, size);
|
|
return retval;
|
|
|
|
out_unlock:
|
|
rcu_read_unlock();
|
|
return retval;
|
|
}
|
|
|
|
long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
|
|
{
|
|
cpumask_var_t cpus_allowed, new_mask;
|
|
struct task_struct *p;
|
|
int retval;
|
|
|
|
rcu_read_lock();
|
|
|
|
p = find_process_by_pid(pid);
|
|
if (!p) {
|
|
rcu_read_unlock();
|
|
return -ESRCH;
|
|
}
|
|
|
|
/* Prevent p going away */
|
|
get_task_struct(p);
|
|
rcu_read_unlock();
|
|
|
|
if (p->flags & PF_NO_SETAFFINITY) {
|
|
retval = -EINVAL;
|
|
goto out_put_task;
|
|
}
|
|
if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
|
|
retval = -ENOMEM;
|
|
goto out_put_task;
|
|
}
|
|
if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
|
|
retval = -ENOMEM;
|
|
goto out_free_cpus_allowed;
|
|
}
|
|
retval = -EPERM;
|
|
if (!check_same_owner(p)) {
|
|
rcu_read_lock();
|
|
if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
|
|
rcu_read_unlock();
|
|
goto out_free_new_mask;
|
|
}
|
|
rcu_read_unlock();
|
|
}
|
|
|
|
retval = security_task_setscheduler(p);
|
|
if (retval)
|
|
goto out_free_new_mask;
|
|
|
|
|
|
cpuset_cpus_allowed(p, cpus_allowed);
|
|
cpumask_and(new_mask, in_mask, cpus_allowed);
|
|
|
|
/*
|
|
* Since bandwidth control happens on root_domain basis,
|
|
* if admission test is enabled, we only admit -deadline
|
|
* tasks allowed to run on all the CPUs in the task's
|
|
* root_domain.
|
|
*/
|
|
#ifdef CONFIG_SMP
|
|
if (task_has_dl_policy(p) && dl_bandwidth_enabled()) {
|
|
rcu_read_lock();
|
|
if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) {
|
|
retval = -EBUSY;
|
|
rcu_read_unlock();
|
|
goto out_free_new_mask;
|
|
}
|
|
rcu_read_unlock();
|
|
}
|
|
#endif
|
|
again:
|
|
retval = __set_cpus_allowed_ptr(p, new_mask, true);
|
|
|
|
if (!retval) {
|
|
cpuset_cpus_allowed(p, cpus_allowed);
|
|
if (!cpumask_subset(new_mask, cpus_allowed)) {
|
|
/*
|
|
* We must have raced with a concurrent cpuset
|
|
* update. Just reset the cpus_allowed to the
|
|
* cpuset's cpus_allowed
|
|
*/
|
|
cpumask_copy(new_mask, cpus_allowed);
|
|
goto again;
|
|
}
|
|
}
|
|
out_free_new_mask:
|
|
free_cpumask_var(new_mask);
|
|
out_free_cpus_allowed:
|
|
free_cpumask_var(cpus_allowed);
|
|
out_put_task:
|
|
put_task_struct(p);
|
|
return retval;
|
|
}
|
|
|
|
static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
|
|
struct cpumask *new_mask)
|
|
{
|
|
if (len < cpumask_size())
|
|
cpumask_clear(new_mask);
|
|
else if (len > cpumask_size())
|
|
len = cpumask_size();
|
|
|
|
return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
|
|
}
|
|
|
|
/**
|
|
* sys_sched_setaffinity - set the CPU affinity of a process
|
|
* @pid: pid of the process
|
|
* @len: length in bytes of the bitmask pointed to by user_mask_ptr
|
|
* @user_mask_ptr: user-space pointer to the new CPU mask
|
|
*
|
|
* Return: 0 on success. An error code otherwise.
|
|
*/
|
|
SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
|
|
unsigned long __user *, user_mask_ptr)
|
|
{
|
|
cpumask_var_t new_mask;
|
|
int retval;
|
|
|
|
if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
|
|
return -ENOMEM;
|
|
|
|
retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
|
|
if (retval == 0)
|
|
retval = sched_setaffinity(pid, new_mask);
|
|
free_cpumask_var(new_mask);
|
|
return retval;
|
|
}
|
|
|
|
long sched_getaffinity(pid_t pid, struct cpumask *mask)
|
|
{
|
|
struct task_struct *p;
|
|
unsigned long flags;
|
|
int retval;
|
|
|
|
rcu_read_lock();
|
|
|
|
retval = -ESRCH;
|
|
p = find_process_by_pid(pid);
|
|
if (!p)
|
|
goto out_unlock;
|
|
|
|
retval = security_task_getscheduler(p);
|
|
if (retval)
|
|
goto out_unlock;
|
|
|
|
raw_spin_lock_irqsave(&p->pi_lock, flags);
|
|
cpumask_and(mask, &p->cpus_allowed, cpu_active_mask);
|
|
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
|
|
|
|
out_unlock:
|
|
rcu_read_unlock();
|
|
|
|
return retval;
|
|
}
|
|
|
|
/**
|
|
* sys_sched_getaffinity - get the CPU affinity of a process
|
|
* @pid: pid of the process
|
|
* @len: length in bytes of the bitmask pointed to by user_mask_ptr
|
|
* @user_mask_ptr: user-space pointer to hold the current CPU mask
|
|
*
|
|
* Return: size of CPU mask copied to user_mask_ptr on success. An
|
|
* error code otherwise.
|
|
*/
|
|
SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
|
|
unsigned long __user *, user_mask_ptr)
|
|
{
|
|
int ret;
|
|
cpumask_var_t mask;
|
|
|
|
if ((len * BITS_PER_BYTE) < nr_cpu_ids)
|
|
return -EINVAL;
|
|
if (len & (sizeof(unsigned long)-1))
|
|
return -EINVAL;
|
|
|
|
if (!alloc_cpumask_var(&mask, GFP_KERNEL))
|
|
return -ENOMEM;
|
|
|
|
ret = sched_getaffinity(pid, mask);
|
|
if (ret == 0) {
|
|
size_t retlen = min_t(size_t, len, cpumask_size());
|
|
|
|
if (copy_to_user(user_mask_ptr, mask, retlen))
|
|
ret = -EFAULT;
|
|
else
|
|
ret = retlen;
|
|
}
|
|
free_cpumask_var(mask);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* sys_sched_yield - yield the current processor to other threads.
|
|
*
|
|
* This function yields the current CPU to other tasks. If there are no
|
|
* other threads running on this CPU then this function will return.
|
|
*
|
|
* Return: 0.
|
|
*/
|
|
SYSCALL_DEFINE0(sched_yield)
|
|
{
|
|
struct rq_flags rf;
|
|
struct rq *rq;
|
|
|
|
local_irq_disable();
|
|
rq = this_rq();
|
|
rq_lock(rq, &rf);
|
|
|
|
schedstat_inc(rq->yld_count);
|
|
current->sched_class->yield_task(rq);
|
|
|
|
/*
|
|
* Since we are going to call schedule() anyway, there's
|
|
* no need to preempt or enable interrupts:
|
|
*/
|
|
preempt_disable();
|
|
rq_unlock(rq, &rf);
|
|
sched_preempt_enable_no_resched();
|
|
|
|
schedule();
|
|
|
|
return 0;
|
|
}
|
|
|
|
#ifndef CONFIG_PREEMPT
|
|
int __sched _cond_resched(void)
|
|
{
|
|
if (should_resched(0)) {
|
|
preempt_schedule_common();
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(_cond_resched);
|
|
#endif
|
|
|
|
/*
|
|
* __cond_resched_lock() - if a reschedule is pending, drop the given lock,
|
|
* call schedule, and on return reacquire the lock.
|
|
*
|
|
* This works OK both with and without CONFIG_PREEMPT. We do strange low-level
|
|
* operations here to prevent schedule() from being called twice (once via
|
|
* spin_unlock(), once by hand).
|
|
*/
|
|
int __cond_resched_lock(spinlock_t *lock)
|
|
{
|
|
int resched = should_resched(PREEMPT_LOCK_OFFSET);
|
|
int ret = 0;
|
|
|
|
lockdep_assert_held(lock);
|
|
|
|
if (spin_needbreak(lock) || resched) {
|
|
spin_unlock(lock);
|
|
if (resched)
|
|
preempt_schedule_common();
|
|
else
|
|
cpu_relax();
|
|
ret = 1;
|
|
spin_lock(lock);
|
|
}
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(__cond_resched_lock);
|
|
|
|
int __sched __cond_resched_softirq(void)
|
|
{
|
|
BUG_ON(!in_softirq());
|
|
|
|
if (should_resched(SOFTIRQ_DISABLE_OFFSET)) {
|
|
local_bh_enable();
|
|
preempt_schedule_common();
|
|
local_bh_disable();
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(__cond_resched_softirq);
|
|
|
|
/**
|
|
* yield - yield the current processor to other threads.
|
|
*
|
|
* Do not ever use this function, there's a 99% chance you're doing it wrong.
|
|
*
|
|
* The scheduler is at all times free to pick the calling task as the most
|
|
* eligible task to run, if removing the yield() call from your code breaks
|
|
* it, its already broken.
|
|
*
|
|
* Typical broken usage is:
|
|
*
|
|
* while (!event)
|
|
* yield();
|
|
*
|
|
* where one assumes that yield() will let 'the other' process run that will
|
|
* make event true. If the current task is a SCHED_FIFO task that will never
|
|
* happen. Never use yield() as a progress guarantee!!
|
|
*
|
|
* If you want to use yield() to wait for something, use wait_event().
|
|
* If you want to use yield() to be 'nice' for others, use cond_resched().
|
|
* If you still want to use yield(), do not!
|
|
*/
|
|
void __sched yield(void)
|
|
{
|
|
set_current_state(TASK_RUNNING);
|
|
sys_sched_yield();
|
|
}
|
|
EXPORT_SYMBOL(yield);
|
|
|
|
/**
|
|
* yield_to - yield the current processor to another thread in
|
|
* your thread group, or accelerate that thread toward the
|
|
* processor it's on.
|
|
* @p: target task
|
|
* @preempt: whether task preemption is allowed or not
|
|
*
|
|
* It's the caller's job to ensure that the target task struct
|
|
* can't go away on us before we can do any checks.
|
|
*
|
|
* Return:
|
|
* true (>0) if we indeed boosted the target task.
|
|
* false (0) if we failed to boost the target.
|
|
* -ESRCH if there's no task to yield to.
|
|
*/
|
|
int __sched yield_to(struct task_struct *p, bool preempt)
|
|
{
|
|
struct task_struct *curr = current;
|
|
struct rq *rq, *p_rq;
|
|
unsigned long flags;
|
|
int yielded = 0;
|
|
|
|
local_irq_save(flags);
|
|
rq = this_rq();
|
|
|
|
again:
|
|
p_rq = task_rq(p);
|
|
/*
|
|
* If we're the only runnable task on the rq and target rq also
|
|
* has only one task, there's absolutely no point in yielding.
|
|
*/
|
|
if (rq->nr_running == 1 && p_rq->nr_running == 1) {
|
|
yielded = -ESRCH;
|
|
goto out_irq;
|
|
}
|
|
|
|
double_rq_lock(rq, p_rq);
|
|
if (task_rq(p) != p_rq) {
|
|
double_rq_unlock(rq, p_rq);
|
|
goto again;
|
|
}
|
|
|
|
if (!curr->sched_class->yield_to_task)
|
|
goto out_unlock;
|
|
|
|
if (curr->sched_class != p->sched_class)
|
|
goto out_unlock;
|
|
|
|
if (task_running(p_rq, p) || p->state)
|
|
goto out_unlock;
|
|
|
|
yielded = curr->sched_class->yield_to_task(rq, p, preempt);
|
|
if (yielded) {
|
|
schedstat_inc(rq->yld_count);
|
|
/*
|
|
* Make p's CPU reschedule; pick_next_entity takes care of
|
|
* fairness.
|
|
*/
|
|
if (preempt && rq != p_rq)
|
|
resched_curr(p_rq);
|
|
}
|
|
|
|
out_unlock:
|
|
double_rq_unlock(rq, p_rq);
|
|
out_irq:
|
|
local_irq_restore(flags);
|
|
|
|
if (yielded > 0)
|
|
schedule();
|
|
|
|
return yielded;
|
|
}
|
|
EXPORT_SYMBOL_GPL(yield_to);
|
|
|
|
int io_schedule_prepare(void)
|
|
{
|
|
int old_iowait = current->in_iowait;
|
|
|
|
current->in_iowait = 1;
|
|
blk_schedule_flush_plug(current);
|
|
|
|
return old_iowait;
|
|
}
|
|
|
|
void io_schedule_finish(int token)
|
|
{
|
|
current->in_iowait = token;
|
|
}
|
|
|
|
/*
|
|
* This task is about to go to sleep on IO. Increment rq->nr_iowait so
|
|
* that process accounting knows that this is a task in IO wait state.
|
|
*/
|
|
long __sched io_schedule_timeout(long timeout)
|
|
{
|
|
int token;
|
|
long ret;
|
|
|
|
token = io_schedule_prepare();
|
|
ret = schedule_timeout(timeout);
|
|
io_schedule_finish(token);
|
|
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(io_schedule_timeout);
|
|
|
|
void io_schedule(void)
|
|
{
|
|
int token;
|
|
|
|
token = io_schedule_prepare();
|
|
schedule();
|
|
io_schedule_finish(token);
|
|
}
|
|
EXPORT_SYMBOL(io_schedule);
|
|
|
|
/**
|
|
* sys_sched_get_priority_max - return maximum RT priority.
|
|
* @policy: scheduling class.
|
|
*
|
|
* Return: On success, this syscall returns the maximum
|
|
* rt_priority that can be used by a given scheduling class.
|
|
* On failure, a negative error code is returned.
|
|
*/
|
|
SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
|
|
{
|
|
int ret = -EINVAL;
|
|
|
|
switch (policy) {
|
|
case SCHED_FIFO:
|
|
case SCHED_RR:
|
|
ret = MAX_USER_RT_PRIO-1;
|
|
break;
|
|
case SCHED_DEADLINE:
|
|
case SCHED_NORMAL:
|
|
case SCHED_BATCH:
|
|
case SCHED_IDLE:
|
|
ret = 0;
|
|
break;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* sys_sched_get_priority_min - return minimum RT priority.
|
|
* @policy: scheduling class.
|
|
*
|
|
* Return: On success, this syscall returns the minimum
|
|
* rt_priority that can be used by a given scheduling class.
|
|
* On failure, a negative error code is returned.
|
|
*/
|
|
SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
|
|
{
|
|
int ret = -EINVAL;
|
|
|
|
switch (policy) {
|
|
case SCHED_FIFO:
|
|
case SCHED_RR:
|
|
ret = 1;
|
|
break;
|
|
case SCHED_DEADLINE:
|
|
case SCHED_NORMAL:
|
|
case SCHED_BATCH:
|
|
case SCHED_IDLE:
|
|
ret = 0;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* sys_sched_rr_get_interval - return the default timeslice of a process.
|
|
* @pid: pid of the process.
|
|
* @interval: userspace pointer to the timeslice value.
|
|
*
|
|
* this syscall writes the default timeslice value of a given process
|
|
* into the user-space timespec buffer. A value of '0' means infinity.
|
|
*
|
|
* Return: On success, 0 and the timeslice is in @interval. Otherwise,
|
|
* an error code.
|
|
*/
|
|
SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
|
|
struct timespec __user *, interval)
|
|
{
|
|
struct task_struct *p;
|
|
unsigned int time_slice;
|
|
struct rq_flags rf;
|
|
struct timespec t;
|
|
struct rq *rq;
|
|
int retval;
|
|
|
|
if (pid < 0)
|
|
return -EINVAL;
|
|
|
|
retval = -ESRCH;
|
|
rcu_read_lock();
|
|
p = find_process_by_pid(pid);
|
|
if (!p)
|
|
goto out_unlock;
|
|
|
|
retval = security_task_getscheduler(p);
|
|
if (retval)
|
|
goto out_unlock;
|
|
|
|
rq = task_rq_lock(p, &rf);
|
|
time_slice = 0;
|
|
if (p->sched_class->get_rr_interval)
|
|
time_slice = p->sched_class->get_rr_interval(rq, p);
|
|
task_rq_unlock(rq, p, &rf);
|
|
|
|
rcu_read_unlock();
|
|
jiffies_to_timespec(time_slice, &t);
|
|
retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
|
|
return retval;
|
|
|
|
out_unlock:
|
|
rcu_read_unlock();
|
|
return retval;
|
|
}
|
|
|
|
static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
|
|
|
|
void sched_show_task(struct task_struct *p)
|
|
{
|
|
unsigned long free = 0;
|
|
int ppid;
|
|
unsigned long state = p->state;
|
|
|
|
/* Make sure the string lines up properly with the number of task states: */
|
|
BUILD_BUG_ON(sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1);
|
|
|
|
if (!try_get_task_stack(p))
|
|
return;
|
|
if (state)
|
|
state = __ffs(state) + 1;
|
|
printk(KERN_INFO "%-15.15s %c", p->comm,
|
|
state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
|
|
if (state == TASK_RUNNING)
|
|
printk(KERN_CONT " running task ");
|
|
#ifdef CONFIG_DEBUG_STACK_USAGE
|
|
free = stack_not_used(p);
|
|
#endif
|
|
ppid = 0;
|
|
rcu_read_lock();
|
|
if (pid_alive(p))
|
|
ppid = task_pid_nr(rcu_dereference(p->real_parent));
|
|
rcu_read_unlock();
|
|
printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
|
|
task_pid_nr(p), ppid,
|
|
(unsigned long)task_thread_info(p)->flags);
|
|
|
|
print_worker_info(KERN_INFO, p);
|
|
show_stack(p, NULL);
|
|
put_task_stack(p);
|
|
}
|
|
|
|
void show_state_filter(unsigned long state_filter)
|
|
{
|
|
struct task_struct *g, *p;
|
|
|
|
#if BITS_PER_LONG == 32
|
|
printk(KERN_INFO
|
|
" task PC stack pid father\n");
|
|
#else
|
|
printk(KERN_INFO
|
|
" task PC stack pid father\n");
|
|
#endif
|
|
rcu_read_lock();
|
|
for_each_process_thread(g, p) {
|
|
/*
|
|
* reset the NMI-timeout, listing all files on a slow
|
|
* console might take a lot of time:
|
|
* Also, reset softlockup watchdogs on all CPUs, because
|
|
* another CPU might be blocked waiting for us to process
|
|
* an IPI.
|
|
*/
|
|
touch_nmi_watchdog();
|
|
touch_all_softlockup_watchdogs();
|
|
if (!state_filter || (p->state & state_filter))
|
|
sched_show_task(p);
|
|
}
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
if (!state_filter)
|
|
sysrq_sched_debug_show();
|
|
#endif
|
|
rcu_read_unlock();
|
|
/*
|
|
* Only show locks if all tasks are dumped:
|
|
*/
|
|
if (!state_filter)
|
|
debug_show_all_locks();
|
|
}
|
|
|
|
void init_idle_bootup_task(struct task_struct *idle)
|
|
{
|
|
idle->sched_class = &idle_sched_class;
|
|
}
|
|
|
|
/**
|
|
* init_idle - set up an idle thread for a given CPU
|
|
* @idle: task in question
|
|
* @cpu: CPU the idle task belongs to
|
|
*
|
|
* NOTE: this function does not set the idle thread's NEED_RESCHED
|
|
* flag, to make booting more robust.
|
|
*/
|
|
void init_idle(struct task_struct *idle, int cpu)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
unsigned long flags;
|
|
|
|
raw_spin_lock_irqsave(&idle->pi_lock, flags);
|
|
raw_spin_lock(&rq->lock);
|
|
|
|
__sched_fork(0, idle);
|
|
idle->state = TASK_RUNNING;
|
|
idle->se.exec_start = sched_clock();
|
|
idle->flags |= PF_IDLE;
|
|
|
|
kasan_unpoison_task_stack(idle);
|
|
|
|
#ifdef CONFIG_SMP
|
|
/*
|
|
* Its possible that init_idle() gets called multiple times on a task,
|
|
* in that case do_set_cpus_allowed() will not do the right thing.
|
|
*
|
|
* And since this is boot we can forgo the serialization.
|
|
*/
|
|
set_cpus_allowed_common(idle, cpumask_of(cpu));
|
|
#endif
|
|
/*
|
|
* We're having a chicken and egg problem, even though we are
|
|
* holding rq->lock, the CPU isn't yet set to this CPU so the
|
|
* lockdep check in task_group() will fail.
|
|
*
|
|
* Similar case to sched_fork(). / Alternatively we could
|
|
* use task_rq_lock() here and obtain the other rq->lock.
|
|
*
|
|
* Silence PROVE_RCU
|
|
*/
|
|
rcu_read_lock();
|
|
__set_task_cpu(idle, cpu);
|
|
rcu_read_unlock();
|
|
|
|
rq->curr = rq->idle = idle;
|
|
idle->on_rq = TASK_ON_RQ_QUEUED;
|
|
#ifdef CONFIG_SMP
|
|
idle->on_cpu = 1;
|
|
#endif
|
|
raw_spin_unlock(&rq->lock);
|
|
raw_spin_unlock_irqrestore(&idle->pi_lock, flags);
|
|
|
|
/* Set the preempt count _outside_ the spinlocks! */
|
|
init_idle_preempt_count(idle, cpu);
|
|
|
|
/*
|
|
* The idle tasks have their own, simple scheduling class:
|
|
*/
|
|
idle->sched_class = &idle_sched_class;
|
|
ftrace_graph_init_idle_task(idle, cpu);
|
|
vtime_init_idle(idle, cpu);
|
|
#ifdef CONFIG_SMP
|
|
sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
|
|
#endif
|
|
}
|
|
|
|
int cpuset_cpumask_can_shrink(const struct cpumask *cur,
|
|
const struct cpumask *trial)
|
|
{
|
|
int ret = 1, trial_cpus;
|
|
struct dl_bw *cur_dl_b;
|
|
unsigned long flags;
|
|
|
|
if (!cpumask_weight(cur))
|
|
return ret;
|
|
|
|
rcu_read_lock_sched();
|
|
cur_dl_b = dl_bw_of(cpumask_any(cur));
|
|
trial_cpus = cpumask_weight(trial);
|
|
|
|
raw_spin_lock_irqsave(&cur_dl_b->lock, flags);
|
|
if (cur_dl_b->bw != -1 &&
|
|
cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw)
|
|
ret = 0;
|
|
raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags);
|
|
rcu_read_unlock_sched();
|
|
|
|
return ret;
|
|
}
|
|
|
|
int task_can_attach(struct task_struct *p,
|
|
const struct cpumask *cs_cpus_allowed)
|
|
{
|
|
int ret = 0;
|
|
|
|
/*
|
|
* Kthreads which disallow setaffinity shouldn't be moved
|
|
* to a new cpuset; we don't want to change their CPU
|
|
* affinity and isolating such threads by their set of
|
|
* allowed nodes is unnecessary. Thus, cpusets are not
|
|
* applicable for such threads. This prevents checking for
|
|
* success of set_cpus_allowed_ptr() on all attached tasks
|
|
* before cpus_allowed may be changed.
|
|
*/
|
|
if (p->flags & PF_NO_SETAFFINITY) {
|
|
ret = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
#ifdef CONFIG_SMP
|
|
if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span,
|
|
cs_cpus_allowed)) {
|
|
unsigned int dest_cpu = cpumask_any_and(cpu_active_mask,
|
|
cs_cpus_allowed);
|
|
struct dl_bw *dl_b;
|
|
bool overflow;
|
|
int cpus;
|
|
unsigned long flags;
|
|
|
|
rcu_read_lock_sched();
|
|
dl_b = dl_bw_of(dest_cpu);
|
|
raw_spin_lock_irqsave(&dl_b->lock, flags);
|
|
cpus = dl_bw_cpus(dest_cpu);
|
|
overflow = __dl_overflow(dl_b, cpus, 0, p->dl.dl_bw);
|
|
if (overflow)
|
|
ret = -EBUSY;
|
|
else {
|
|
/*
|
|
* We reserve space for this task in the destination
|
|
* root_domain, as we can't fail after this point.
|
|
* We will free resources in the source root_domain
|
|
* later on (see set_cpus_allowed_dl()).
|
|
*/
|
|
__dl_add(dl_b, p->dl.dl_bw);
|
|
}
|
|
raw_spin_unlock_irqrestore(&dl_b->lock, flags);
|
|
rcu_read_unlock_sched();
|
|
|
|
}
|
|
#endif
|
|
out:
|
|
return ret;
|
|
}
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
bool sched_smp_initialized __read_mostly;
|
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
/* Migrate current task p to target_cpu */
|
|
int migrate_task_to(struct task_struct *p, int target_cpu)
|
|
{
|
|
struct migration_arg arg = { p, target_cpu };
|
|
int curr_cpu = task_cpu(p);
|
|
|
|
if (curr_cpu == target_cpu)
|
|
return 0;
|
|
|
|
if (!cpumask_test_cpu(target_cpu, &p->cpus_allowed))
|
|
return -EINVAL;
|
|
|
|
/* TODO: This is not properly updating schedstats */
|
|
|
|
trace_sched_move_numa(p, curr_cpu, target_cpu);
|
|
return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
|
|
}
|
|
|
|
/*
|
|
* Requeue a task on a given node and accurately track the number of NUMA
|
|
* tasks on the runqueues
|
|
*/
|
|
void sched_setnuma(struct task_struct *p, int nid)
|
|
{
|
|
bool queued, running;
|
|
struct rq_flags rf;
|
|
struct rq *rq;
|
|
|
|
rq = task_rq_lock(p, &rf);
|
|
queued = task_on_rq_queued(p);
|
|
running = task_current(rq, p);
|
|
|
|
if (queued)
|
|
dequeue_task(rq, p, DEQUEUE_SAVE);
|
|
if (running)
|
|
put_prev_task(rq, p);
|
|
|
|
p->numa_preferred_nid = nid;
|
|
|
|
if (queued)
|
|
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
|
|
if (running)
|
|
set_curr_task(rq, p);
|
|
task_rq_unlock(rq, p, &rf);
|
|
}
|
|
#endif /* CONFIG_NUMA_BALANCING */
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
/*
|
|
* Ensure that the idle task is using init_mm right before its CPU goes
|
|
* offline.
|
|
*/
|
|
void idle_task_exit(void)
|
|
{
|
|
struct mm_struct *mm = current->active_mm;
|
|
|
|
BUG_ON(cpu_online(smp_processor_id()));
|
|
|
|
if (mm != &init_mm) {
|
|
switch_mm_irqs_off(mm, &init_mm, current);
|
|
finish_arch_post_lock_switch();
|
|
}
|
|
mmdrop(mm);
|
|
}
|
|
|
|
/*
|
|
* Since this CPU is going 'away' for a while, fold any nr_active delta
|
|
* we might have. Assumes we're called after migrate_tasks() so that the
|
|
* nr_active count is stable. We need to take the teardown thread which
|
|
* is calling this into account, so we hand in adjust = 1 to the load
|
|
* calculation.
|
|
*
|
|
* Also see the comment "Global load-average calculations".
|
|
*/
|
|
static void calc_load_migrate(struct rq *rq)
|
|
{
|
|
long delta = calc_load_fold_active(rq, 1);
|
|
if (delta)
|
|
atomic_long_add(delta, &calc_load_tasks);
|
|
}
|
|
|
|
static void put_prev_task_fake(struct rq *rq, struct task_struct *prev)
|
|
{
|
|
}
|
|
|
|
static const struct sched_class fake_sched_class = {
|
|
.put_prev_task = put_prev_task_fake,
|
|
};
|
|
|
|
static struct task_struct fake_task = {
|
|
/*
|
|
* Avoid pull_{rt,dl}_task()
|
|
*/
|
|
.prio = MAX_PRIO + 1,
|
|
.sched_class = &fake_sched_class,
|
|
};
|
|
|
|
/*
|
|
* Migrate all tasks from the rq, sleeping tasks will be migrated by
|
|
* try_to_wake_up()->select_task_rq().
|
|
*
|
|
* Called with rq->lock held even though we'er in stop_machine() and
|
|
* there's no concurrency possible, we hold the required locks anyway
|
|
* because of lock validation efforts.
|
|
*/
|
|
static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf)
|
|
{
|
|
struct rq *rq = dead_rq;
|
|
struct task_struct *next, *stop = rq->stop;
|
|
struct rq_flags orf = *rf;
|
|
int dest_cpu;
|
|
|
|
/*
|
|
* Fudge the rq selection such that the below task selection loop
|
|
* doesn't get stuck on the currently eligible stop task.
|
|
*
|
|
* We're currently inside stop_machine() and the rq is either stuck
|
|
* in the stop_machine_cpu_stop() loop, or we're executing this code,
|
|
* either way we should never end up calling schedule() until we're
|
|
* done here.
|
|
*/
|
|
rq->stop = NULL;
|
|
|
|
/*
|
|
* put_prev_task() and pick_next_task() sched
|
|
* class method both need to have an up-to-date
|
|
* value of rq->clock[_task]
|
|
*/
|
|
update_rq_clock(rq);
|
|
|
|
for (;;) {
|
|
/*
|
|
* There's this thread running, bail when that's the only
|
|
* remaining thread:
|
|
*/
|
|
if (rq->nr_running == 1)
|
|
break;
|
|
|
|
/*
|
|
* pick_next_task() assumes pinned rq->lock:
|
|
*/
|
|
next = pick_next_task(rq, &fake_task, rf);
|
|
BUG_ON(!next);
|
|
next->sched_class->put_prev_task(rq, next);
|
|
|
|
/*
|
|
* Rules for changing task_struct::cpus_allowed are holding
|
|
* both pi_lock and rq->lock, such that holding either
|
|
* stabilizes the mask.
|
|
*
|
|
* Drop rq->lock is not quite as disastrous as it usually is
|
|
* because !cpu_active at this point, which means load-balance
|
|
* will not interfere. Also, stop-machine.
|
|
*/
|
|
rq_unlock(rq, rf);
|
|
raw_spin_lock(&next->pi_lock);
|
|
rq_relock(rq, rf);
|
|
|
|
/*
|
|
* Since we're inside stop-machine, _nothing_ should have
|
|
* changed the task, WARN if weird stuff happened, because in
|
|
* that case the above rq->lock drop is a fail too.
|
|
*/
|
|
if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) {
|
|
raw_spin_unlock(&next->pi_lock);
|
|
continue;
|
|
}
|
|
|
|
/* Find suitable destination for @next, with force if needed. */
|
|
dest_cpu = select_fallback_rq(dead_rq->cpu, next);
|
|
rq = __migrate_task(rq, rf, next, dest_cpu);
|
|
if (rq != dead_rq) {
|
|
rq_unlock(rq, rf);
|
|
rq = dead_rq;
|
|
*rf = orf;
|
|
rq_relock(rq, rf);
|
|
}
|
|
raw_spin_unlock(&next->pi_lock);
|
|
}
|
|
|
|
rq->stop = stop;
|
|
}
|
|
#endif /* CONFIG_HOTPLUG_CPU */
|
|
|
|
void set_rq_online(struct rq *rq)
|
|
{
|
|
if (!rq->online) {
|
|
const struct sched_class *class;
|
|
|
|
cpumask_set_cpu(rq->cpu, rq->rd->online);
|
|
rq->online = 1;
|
|
|
|
for_each_class(class) {
|
|
if (class->rq_online)
|
|
class->rq_online(rq);
|
|
}
|
|
}
|
|
}
|
|
|
|
void set_rq_offline(struct rq *rq)
|
|
{
|
|
if (rq->online) {
|
|
const struct sched_class *class;
|
|
|
|
for_each_class(class) {
|
|
if (class->rq_offline)
|
|
class->rq_offline(rq);
|
|
}
|
|
|
|
cpumask_clear_cpu(rq->cpu, rq->rd->online);
|
|
rq->online = 0;
|
|
}
|
|
}
|
|
|
|
static void set_cpu_rq_start_time(unsigned int cpu)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
rq->age_stamp = sched_clock_cpu(cpu);
|
|
}
|
|
|
|
/*
|
|
* used to mark begin/end of suspend/resume:
|
|
*/
|
|
static int num_cpus_frozen;
|
|
|
|
/*
|
|
* Update cpusets according to cpu_active mask. If cpusets are
|
|
* disabled, cpuset_update_active_cpus() becomes a simple wrapper
|
|
* around partition_sched_domains().
|
|
*
|
|
* If we come here as part of a suspend/resume, don't touch cpusets because we
|
|
* want to restore it back to its original state upon resume anyway.
|
|
*/
|
|
static void cpuset_cpu_active(void)
|
|
{
|
|
if (cpuhp_tasks_frozen) {
|
|
/*
|
|
* num_cpus_frozen tracks how many CPUs are involved in suspend
|
|
* resume sequence. As long as this is not the last online
|
|
* operation in the resume sequence, just build a single sched
|
|
* domain, ignoring cpusets.
|
|
*/
|
|
num_cpus_frozen--;
|
|
if (likely(num_cpus_frozen)) {
|
|
partition_sched_domains(1, NULL, NULL);
|
|
return;
|
|
}
|
|
/*
|
|
* This is the last CPU online operation. So fall through and
|
|
* restore the original sched domains by considering the
|
|
* cpuset configurations.
|
|
*/
|
|
}
|
|
cpuset_update_active_cpus();
|
|
}
|
|
|
|
static int cpuset_cpu_inactive(unsigned int cpu)
|
|
{
|
|
unsigned long flags;
|
|
struct dl_bw *dl_b;
|
|
bool overflow;
|
|
int cpus;
|
|
|
|
if (!cpuhp_tasks_frozen) {
|
|
rcu_read_lock_sched();
|
|
dl_b = dl_bw_of(cpu);
|
|
|
|
raw_spin_lock_irqsave(&dl_b->lock, flags);
|
|
cpus = dl_bw_cpus(cpu);
|
|
overflow = __dl_overflow(dl_b, cpus, 0, 0);
|
|
raw_spin_unlock_irqrestore(&dl_b->lock, flags);
|
|
|
|
rcu_read_unlock_sched();
|
|
|
|
if (overflow)
|
|
return -EBUSY;
|
|
cpuset_update_active_cpus();
|
|
} else {
|
|
num_cpus_frozen++;
|
|
partition_sched_domains(1, NULL, NULL);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int sched_cpu_activate(unsigned int cpu)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
struct rq_flags rf;
|
|
|
|
set_cpu_active(cpu, true);
|
|
|
|
if (sched_smp_initialized) {
|
|
sched_domains_numa_masks_set(cpu);
|
|
cpuset_cpu_active();
|
|
}
|
|
|
|
/*
|
|
* Put the rq online, if not already. This happens:
|
|
*
|
|
* 1) In the early boot process, because we build the real domains
|
|
* after all CPUs have been brought up.
|
|
*
|
|
* 2) At runtime, if cpuset_cpu_active() fails to rebuild the
|
|
* domains.
|
|
*/
|
|
rq_lock_irqsave(rq, &rf);
|
|
if (rq->rd) {
|
|
BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
|
|
set_rq_online(rq);
|
|
}
|
|
rq_unlock_irqrestore(rq, &rf);
|
|
|
|
update_max_interval();
|
|
|
|
return 0;
|
|
}
|
|
|
|
int sched_cpu_deactivate(unsigned int cpu)
|
|
{
|
|
int ret;
|
|
|
|
set_cpu_active(cpu, false);
|
|
/*
|
|
* We've cleared cpu_active_mask, wait for all preempt-disabled and RCU
|
|
* users of this state to go away such that all new such users will
|
|
* observe it.
|
|
*
|
|
* For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might
|
|
* not imply sync_sched(), so wait for both.
|
|
*
|
|
* Do sync before park smpboot threads to take care the rcu boost case.
|
|
*/
|
|
if (IS_ENABLED(CONFIG_PREEMPT))
|
|
synchronize_rcu_mult(call_rcu, call_rcu_sched);
|
|
else
|
|
synchronize_rcu();
|
|
|
|
if (!sched_smp_initialized)
|
|
return 0;
|
|
|
|
ret = cpuset_cpu_inactive(cpu);
|
|
if (ret) {
|
|
set_cpu_active(cpu, true);
|
|
return ret;
|
|
}
|
|
sched_domains_numa_masks_clear(cpu);
|
|
return 0;
|
|
}
|
|
|
|
static void sched_rq_cpu_starting(unsigned int cpu)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
rq->calc_load_update = calc_load_update;
|
|
update_max_interval();
|
|
}
|
|
|
|
int sched_cpu_starting(unsigned int cpu)
|
|
{
|
|
set_cpu_rq_start_time(cpu);
|
|
sched_rq_cpu_starting(cpu);
|
|
return 0;
|
|
}
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
int sched_cpu_dying(unsigned int cpu)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
struct rq_flags rf;
|
|
|
|
/* Handle pending wakeups and then migrate everything off */
|
|
sched_ttwu_pending();
|
|
|
|
rq_lock_irqsave(rq, &rf);
|
|
if (rq->rd) {
|
|
BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
|
|
set_rq_offline(rq);
|
|
}
|
|
migrate_tasks(rq, &rf);
|
|
BUG_ON(rq->nr_running != 1);
|
|
rq_unlock_irqrestore(rq, &rf);
|
|
|
|
calc_load_migrate(rq);
|
|
update_max_interval();
|
|
nohz_balance_exit_idle(cpu);
|
|
hrtick_clear(rq);
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
#ifdef CONFIG_SCHED_SMT
|
|
DEFINE_STATIC_KEY_FALSE(sched_smt_present);
|
|
|
|
static void sched_init_smt(void)
|
|
{
|
|
/*
|
|
* We've enumerated all CPUs and will assume that if any CPU
|
|
* has SMT siblings, CPU0 will too.
|
|
*/
|
|
if (cpumask_weight(cpu_smt_mask(0)) > 1)
|
|
static_branch_enable(&sched_smt_present);
|
|
}
|
|
#else
|
|
static inline void sched_init_smt(void) { }
|
|
#endif
|
|
|
|
void __init sched_init_smp(void)
|
|
{
|
|
cpumask_var_t non_isolated_cpus;
|
|
|
|
alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
|
|
alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
|
|
|
|
sched_init_numa();
|
|
|
|
/*
|
|
* There's no userspace yet to cause hotplug operations; hence all the
|
|
* CPU masks are stable and all blatant races in the below code cannot
|
|
* happen.
|
|
*/
|
|
mutex_lock(&sched_domains_mutex);
|
|
init_sched_domains(cpu_active_mask);
|
|
cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
|
|
if (cpumask_empty(non_isolated_cpus))
|
|
cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
|
|
mutex_unlock(&sched_domains_mutex);
|
|
|
|
/* Move init over to a non-isolated CPU */
|
|
if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
|
|
BUG();
|
|
sched_init_granularity();
|
|
free_cpumask_var(non_isolated_cpus);
|
|
|
|
init_sched_rt_class();
|
|
init_sched_dl_class();
|
|
|
|
sched_init_smt();
|
|
sched_clock_init_late();
|
|
|
|
sched_smp_initialized = true;
|
|
}
|
|
|
|
static int __init migration_init(void)
|
|
{
|
|
sched_rq_cpu_starting(smp_processor_id());
|
|
return 0;
|
|
}
|
|
early_initcall(migration_init);
|
|
|
|
#else
|
|
void __init sched_init_smp(void)
|
|
{
|
|
sched_init_granularity();
|
|
sched_clock_init_late();
|
|
}
|
|
#endif /* CONFIG_SMP */
|
|
|
|
int in_sched_functions(unsigned long addr)
|
|
{
|
|
return in_lock_functions(addr) ||
|
|
(addr >= (unsigned long)__sched_text_start
|
|
&& addr < (unsigned long)__sched_text_end);
|
|
}
|
|
|
|
#ifdef CONFIG_CGROUP_SCHED
|
|
/*
|
|
* Default task group.
|
|
* Every task in system belongs to this group at bootup.
|
|
*/
|
|
struct task_group root_task_group;
|
|
LIST_HEAD(task_groups);
|
|
|
|
/* Cacheline aligned slab cache for task_group */
|
|
static struct kmem_cache *task_group_cache __read_mostly;
|
|
#endif
|
|
|
|
DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
|
|
DECLARE_PER_CPU(cpumask_var_t, select_idle_mask);
|
|
|
|
#define WAIT_TABLE_BITS 8
|
|
#define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
|
|
static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
|
|
|
|
wait_queue_head_t *bit_waitqueue(void *word, int bit)
|
|
{
|
|
const int shift = BITS_PER_LONG == 32 ? 5 : 6;
|
|
unsigned long val = (unsigned long)word << shift | bit;
|
|
|
|
return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
|
|
}
|
|
EXPORT_SYMBOL(bit_waitqueue);
|
|
|
|
void __init sched_init(void)
|
|
{
|
|
int i, j;
|
|
unsigned long alloc_size = 0, ptr;
|
|
|
|
sched_clock_init();
|
|
|
|
for (i = 0; i < WAIT_TABLE_SIZE; i++)
|
|
init_waitqueue_head(bit_wait_table + i);
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
alloc_size += 2 * nr_cpu_ids * sizeof(void **);
|
|
#endif
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
alloc_size += 2 * nr_cpu_ids * sizeof(void **);
|
|
#endif
|
|
if (alloc_size) {
|
|
ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
root_task_group.se = (struct sched_entity **)ptr;
|
|
ptr += nr_cpu_ids * sizeof(void **);
|
|
|
|
root_task_group.cfs_rq = (struct cfs_rq **)ptr;
|
|
ptr += nr_cpu_ids * sizeof(void **);
|
|
|
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
root_task_group.rt_se = (struct sched_rt_entity **)ptr;
|
|
ptr += nr_cpu_ids * sizeof(void **);
|
|
|
|
root_task_group.rt_rq = (struct rt_rq **)ptr;
|
|
ptr += nr_cpu_ids * sizeof(void **);
|
|
|
|
#endif /* CONFIG_RT_GROUP_SCHED */
|
|
}
|
|
#ifdef CONFIG_CPUMASK_OFFSTACK
|
|
for_each_possible_cpu(i) {
|
|
per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node(
|
|
cpumask_size(), GFP_KERNEL, cpu_to_node(i));
|
|
per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node(
|
|
cpumask_size(), GFP_KERNEL, cpu_to_node(i));
|
|
}
|
|
#endif /* CONFIG_CPUMASK_OFFSTACK */
|
|
|
|
init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime());
|
|
init_dl_bandwidth(&def_dl_bandwidth, global_rt_period(), global_rt_runtime());
|
|
|
|
#ifdef CONFIG_SMP
|
|
init_defrootdomain();
|
|
#endif
|
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
init_rt_bandwidth(&root_task_group.rt_bandwidth,
|
|
global_rt_period(), global_rt_runtime());
|
|
#endif /* CONFIG_RT_GROUP_SCHED */
|
|
|
|
#ifdef CONFIG_CGROUP_SCHED
|
|
task_group_cache = KMEM_CACHE(task_group, 0);
|
|
|
|
list_add(&root_task_group.list, &task_groups);
|
|
INIT_LIST_HEAD(&root_task_group.children);
|
|
INIT_LIST_HEAD(&root_task_group.siblings);
|
|
autogroup_init(&init_task);
|
|
#endif /* CONFIG_CGROUP_SCHED */
|
|
|
|
for_each_possible_cpu(i) {
|
|
struct rq *rq;
|
|
|
|
rq = cpu_rq(i);
|
|
raw_spin_lock_init(&rq->lock);
|
|
rq->nr_running = 0;
|
|
rq->calc_load_active = 0;
|
|
rq->calc_load_update = jiffies + LOAD_FREQ;
|
|
init_cfs_rq(&rq->cfs);
|
|
init_rt_rq(&rq->rt);
|
|
init_dl_rq(&rq->dl);
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
root_task_group.shares = ROOT_TASK_GROUP_LOAD;
|
|
INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
|
|
rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
|
|
/*
|
|
* How much CPU bandwidth does root_task_group get?
|
|
*
|
|
* In case of task-groups formed thr' the cgroup filesystem, it
|
|
* gets 100% of the CPU resources in the system. This overall
|
|
* system CPU resource is divided among the tasks of
|
|
* root_task_group and its child task-groups in a fair manner,
|
|
* based on each entity's (task or task-group's) weight
|
|
* (se->load.weight).
|
|
*
|
|
* In other words, if root_task_group has 10 tasks of weight
|
|
* 1024) and two child groups A0 and A1 (of weight 1024 each),
|
|
* then A0's share of the CPU resource is:
|
|
*
|
|
* A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
|
|
*
|
|
* We achieve this by letting root_task_group's tasks sit
|
|
* directly in rq->cfs (i.e root_task_group->se[] = NULL).
|
|
*/
|
|
init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
|
|
init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
|
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
|
|
|
rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
|
|
#endif
|
|
|
|
for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
|
|
rq->cpu_load[j] = 0;
|
|
|
|
#ifdef CONFIG_SMP
|
|
rq->sd = NULL;
|
|
rq->rd = NULL;
|
|
rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE;
|
|
rq->balance_callback = NULL;
|
|
rq->active_balance = 0;
|
|
rq->next_balance = jiffies;
|
|
rq->push_cpu = 0;
|
|
rq->cpu = i;
|
|
rq->online = 0;
|
|
rq->idle_stamp = 0;
|
|
rq->avg_idle = 2*sysctl_sched_migration_cost;
|
|
rq->max_idle_balance_cost = sysctl_sched_migration_cost;
|
|
|
|
INIT_LIST_HEAD(&rq->cfs_tasks);
|
|
|
|
rq_attach_root(rq, &def_root_domain);
|
|
#ifdef CONFIG_NO_HZ_COMMON
|
|
rq->last_load_update_tick = jiffies;
|
|
rq->nohz_flags = 0;
|
|
#endif
|
|
#ifdef CONFIG_NO_HZ_FULL
|
|
rq->last_sched_tick = 0;
|
|
#endif
|
|
#endif /* CONFIG_SMP */
|
|
init_rq_hrtick(rq);
|
|
atomic_set(&rq->nr_iowait, 0);
|
|
}
|
|
|
|
set_load_weight(&init_task);
|
|
|
|
/*
|
|
* The boot idle thread does lazy MMU switching as well:
|
|
*/
|
|
mmgrab(&init_mm);
|
|
enter_lazy_tlb(&init_mm, current);
|
|
|
|
/*
|
|
* Make us the idle thread. Technically, schedule() should not be
|
|
* called from this thread, however somewhere below it might be,
|
|
* but because we are the idle thread, we just pick up running again
|
|
* when this runqueue becomes "idle".
|
|
*/
|
|
init_idle(current, smp_processor_id());
|
|
|
|
calc_load_update = jiffies + LOAD_FREQ;
|
|
|
|
#ifdef CONFIG_SMP
|
|
zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
|
|
/* May be allocated at isolcpus cmdline parse time */
|
|
if (cpu_isolated_map == NULL)
|
|
zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
|
|
idle_thread_set_boot_cpu();
|
|
set_cpu_rq_start_time(smp_processor_id());
|
|
#endif
|
|
init_sched_fair_class();
|
|
|
|
init_schedstats();
|
|
|
|
scheduler_running = 1;
|
|
}
|
|
|
|
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
|
|
static inline int preempt_count_equals(int preempt_offset)
|
|
{
|
|
int nested = preempt_count() + rcu_preempt_depth();
|
|
|
|
return (nested == preempt_offset);
|
|
}
|
|
|
|
void __might_sleep(const char *file, int line, int preempt_offset)
|
|
{
|
|
/*
|
|
* Blocking primitives will set (and therefore destroy) current->state,
|
|
* since we will exit with TASK_RUNNING make sure we enter with it,
|
|
* otherwise we will destroy state.
|
|
*/
|
|
WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change,
|
|
"do not call blocking ops when !TASK_RUNNING; "
|
|
"state=%lx set at [<%p>] %pS\n",
|
|
current->state,
|
|
(void *)current->task_state_change,
|
|
(void *)current->task_state_change);
|
|
|
|
___might_sleep(file, line, preempt_offset);
|
|
}
|
|
EXPORT_SYMBOL(__might_sleep);
|
|
|
|
void ___might_sleep(const char *file, int line, int preempt_offset)
|
|
{
|
|
/* Ratelimiting timestamp: */
|
|
static unsigned long prev_jiffy;
|
|
|
|
unsigned long preempt_disable_ip;
|
|
|
|
/* WARN_ON_ONCE() by default, no rate limit required: */
|
|
rcu_sleep_check();
|
|
|
|
if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
|
|
!is_idle_task(current)) ||
|
|
system_state != SYSTEM_RUNNING || oops_in_progress)
|
|
return;
|
|
if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
|
|
return;
|
|
prev_jiffy = jiffies;
|
|
|
|
/* Save this before calling printk(), since that will clobber it: */
|
|
preempt_disable_ip = get_preempt_disable_ip(current);
|
|
|
|
printk(KERN_ERR
|
|
"BUG: sleeping function called from invalid context at %s:%d\n",
|
|
file, line);
|
|
printk(KERN_ERR
|
|
"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
|
|
in_atomic(), irqs_disabled(),
|
|
current->pid, current->comm);
|
|
|
|
if (task_stack_end_corrupted(current))
|
|
printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
|
|
|
|
debug_show_held_locks(current);
|
|
if (irqs_disabled())
|
|
print_irqtrace_events(current);
|
|
if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
|
|
&& !preempt_count_equals(preempt_offset)) {
|
|
pr_err("Preemption disabled at:");
|
|
print_ip_sym(preempt_disable_ip);
|
|
pr_cont("\n");
|
|
}
|
|
dump_stack();
|
|
add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
|
|
}
|
|
EXPORT_SYMBOL(___might_sleep);
|
|
#endif
|
|
|
|
#ifdef CONFIG_MAGIC_SYSRQ
|
|
void normalize_rt_tasks(void)
|
|
{
|
|
struct task_struct *g, *p;
|
|
struct sched_attr attr = {
|
|
.sched_policy = SCHED_NORMAL,
|
|
};
|
|
|
|
read_lock(&tasklist_lock);
|
|
for_each_process_thread(g, p) {
|
|
/*
|
|
* Only normalize user tasks:
|
|
*/
|
|
if (p->flags & PF_KTHREAD)
|
|
continue;
|
|
|
|
p->se.exec_start = 0;
|
|
schedstat_set(p->se.statistics.wait_start, 0);
|
|
schedstat_set(p->se.statistics.sleep_start, 0);
|
|
schedstat_set(p->se.statistics.block_start, 0);
|
|
|
|
if (!dl_task(p) && !rt_task(p)) {
|
|
/*
|
|
* Renice negative nice level userspace
|
|
* tasks back to 0:
|
|
*/
|
|
if (task_nice(p) < 0)
|
|
set_user_nice(p, 0);
|
|
continue;
|
|
}
|
|
|
|
__sched_setscheduler(p, &attr, false, false);
|
|
}
|
|
read_unlock(&tasklist_lock);
|
|
}
|
|
|
|
#endif /* CONFIG_MAGIC_SYSRQ */
|
|
|
|
#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
|
|
/*
|
|
* These functions are only useful for the IA64 MCA handling, or kdb.
|
|
*
|
|
* They can only be called when the whole system has been
|
|
* stopped - every CPU needs to be quiescent, and no scheduling
|
|
* activity can take place. Using them for anything else would
|
|
* be a serious bug, and as a result, they aren't even visible
|
|
* under any other configuration.
|
|
*/
|
|
|
|
/**
|
|
* curr_task - return the current task for a given CPU.
|
|
* @cpu: the processor in question.
|
|
*
|
|
* ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
|
|
*
|
|
* Return: The current task for @cpu.
|
|
*/
|
|
struct task_struct *curr_task(int cpu)
|
|
{
|
|
return cpu_curr(cpu);
|
|
}
|
|
|
|
#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
|
|
|
|
#ifdef CONFIG_IA64
|
|
/**
|
|
* set_curr_task - set the current task for a given CPU.
|
|
* @cpu: the processor in question.
|
|
* @p: the task pointer to set.
|
|
*
|
|
* Description: This function must only be used when non-maskable interrupts
|
|
* are serviced on a separate stack. It allows the architecture to switch the
|
|
* notion of the current task on a CPU in a non-blocking manner. This function
|
|
* must be called with all CPU's synchronized, and interrupts disabled, the
|
|
* and caller must save the original value of the current task (see
|
|
* curr_task() above) and restore that value before reenabling interrupts and
|
|
* re-starting the system.
|
|
*
|
|
* ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
|
|
*/
|
|
void ia64_set_curr_task(int cpu, struct task_struct *p)
|
|
{
|
|
cpu_curr(cpu) = p;
|
|
}
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_CGROUP_SCHED
|
|
/* task_group_lock serializes the addition/removal of task groups */
|
|
static DEFINE_SPINLOCK(task_group_lock);
|
|
|
|
static void sched_free_group(struct task_group *tg)
|
|
{
|
|
free_fair_sched_group(tg);
|
|
free_rt_sched_group(tg);
|
|
autogroup_free(tg);
|
|
kmem_cache_free(task_group_cache, tg);
|
|
}
|
|
|
|
/* allocate runqueue etc for a new task group */
|
|
struct task_group *sched_create_group(struct task_group *parent)
|
|
{
|
|
struct task_group *tg;
|
|
|
|
tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO);
|
|
if (!tg)
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
if (!alloc_fair_sched_group(tg, parent))
|
|
goto err;
|
|
|
|
if (!alloc_rt_sched_group(tg, parent))
|
|
goto err;
|
|
|
|
return tg;
|
|
|
|
err:
|
|
sched_free_group(tg);
|
|
return ERR_PTR(-ENOMEM);
|
|
}
|
|
|
|
void sched_online_group(struct task_group *tg, struct task_group *parent)
|
|
{
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(&task_group_lock, flags);
|
|
list_add_rcu(&tg->list, &task_groups);
|
|
|
|
/* Root should already exist: */
|
|
WARN_ON(!parent);
|
|
|
|
tg->parent = parent;
|
|
INIT_LIST_HEAD(&tg->children);
|
|
list_add_rcu(&tg->siblings, &parent->children);
|
|
spin_unlock_irqrestore(&task_group_lock, flags);
|
|
|
|
online_fair_sched_group(tg);
|
|
}
|
|
|
|
/* rcu callback to free various structures associated with a task group */
|
|
static void sched_free_group_rcu(struct rcu_head *rhp)
|
|
{
|
|
/* Now it should be safe to free those cfs_rqs: */
|
|
sched_free_group(container_of(rhp, struct task_group, rcu));
|
|
}
|
|
|
|
void sched_destroy_group(struct task_group *tg)
|
|
{
|
|
/* Wait for possible concurrent references to cfs_rqs complete: */
|
|
call_rcu(&tg->rcu, sched_free_group_rcu);
|
|
}
|
|
|
|
void sched_offline_group(struct task_group *tg)
|
|
{
|
|
unsigned long flags;
|
|
|
|
/* End participation in shares distribution: */
|
|
unregister_fair_sched_group(tg);
|
|
|
|
spin_lock_irqsave(&task_group_lock, flags);
|
|
list_del_rcu(&tg->list);
|
|
list_del_rcu(&tg->siblings);
|
|
spin_unlock_irqrestore(&task_group_lock, flags);
|
|
}
|
|
|
|
static void sched_change_group(struct task_struct *tsk, int type)
|
|
{
|
|
struct task_group *tg;
|
|
|
|
/*
|
|
* All callers are synchronized by task_rq_lock(); we do not use RCU
|
|
* which is pointless here. Thus, we pass "true" to task_css_check()
|
|
* to prevent lockdep warnings.
|
|
*/
|
|
tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
|
|
struct task_group, css);
|
|
tg = autogroup_task_group(tsk, tg);
|
|
tsk->sched_task_group = tg;
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
if (tsk->sched_class->task_change_group)
|
|
tsk->sched_class->task_change_group(tsk, type);
|
|
else
|
|
#endif
|
|
set_task_rq(tsk, task_cpu(tsk));
|
|
}
|
|
|
|
/*
|
|
* Change task's runqueue when it moves between groups.
|
|
*
|
|
* The caller of this function should have put the task in its new group by
|
|
* now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect
|
|
* its new group.
|
|
*/
|
|
void sched_move_task(struct task_struct *tsk)
|
|
{
|
|
int queued, running, queue_flags =
|
|
DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
|
|
struct rq_flags rf;
|
|
struct rq *rq;
|
|
|
|
rq = task_rq_lock(tsk, &rf);
|
|
update_rq_clock(rq);
|
|
|
|
running = task_current(rq, tsk);
|
|
queued = task_on_rq_queued(tsk);
|
|
|
|
if (queued)
|
|
dequeue_task(rq, tsk, queue_flags);
|
|
if (running)
|
|
put_prev_task(rq, tsk);
|
|
|
|
sched_change_group(tsk, TASK_MOVE_GROUP);
|
|
|
|
if (queued)
|
|
enqueue_task(rq, tsk, queue_flags);
|
|
if (running)
|
|
set_curr_task(rq, tsk);
|
|
|
|
task_rq_unlock(rq, tsk, &rf);
|
|
}
|
|
#endif /* CONFIG_CGROUP_SCHED */
|
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
/*
|
|
* Ensure that the real time constraints are schedulable.
|
|
*/
|
|
static DEFINE_MUTEX(rt_constraints_mutex);
|
|
|
|
/* Must be called with tasklist_lock held */
|
|
static inline int tg_has_rt_tasks(struct task_group *tg)
|
|
{
|
|
struct task_struct *g, *p;
|
|
|
|
/*
|
|
* Autogroups do not have RT tasks; see autogroup_create().
|
|
*/
|
|
if (task_group_is_autogroup(tg))
|
|
return 0;
|
|
|
|
for_each_process_thread(g, p) {
|
|
if (rt_task(p) && task_group(p) == tg)
|
|
return 1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
struct rt_schedulable_data {
|
|
struct task_group *tg;
|
|
u64 rt_period;
|
|
u64 rt_runtime;
|
|
};
|
|
|
|
static int tg_rt_schedulable(struct task_group *tg, void *data)
|
|
{
|
|
struct rt_schedulable_data *d = data;
|
|
struct task_group *child;
|
|
unsigned long total, sum = 0;
|
|
u64 period, runtime;
|
|
|
|
period = ktime_to_ns(tg->rt_bandwidth.rt_period);
|
|
runtime = tg->rt_bandwidth.rt_runtime;
|
|
|
|
if (tg == d->tg) {
|
|
period = d->rt_period;
|
|
runtime = d->rt_runtime;
|
|
}
|
|
|
|
/*
|
|
* Cannot have more runtime than the period.
|
|
*/
|
|
if (runtime > period && runtime != RUNTIME_INF)
|
|
return -EINVAL;
|
|
|
|
/*
|
|
* Ensure we don't starve existing RT tasks.
|
|
*/
|
|
if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
|
|
return -EBUSY;
|
|
|
|
total = to_ratio(period, runtime);
|
|
|
|
/*
|
|
* Nobody can have more than the global setting allows.
|
|
*/
|
|
if (total > to_ratio(global_rt_period(), global_rt_runtime()))
|
|
return -EINVAL;
|
|
|
|
/*
|
|
* The sum of our children's runtime should not exceed our own.
|
|
*/
|
|
list_for_each_entry_rcu(child, &tg->children, siblings) {
|
|
period = ktime_to_ns(child->rt_bandwidth.rt_period);
|
|
runtime = child->rt_bandwidth.rt_runtime;
|
|
|
|
if (child == d->tg) {
|
|
period = d->rt_period;
|
|
runtime = d->rt_runtime;
|
|
}
|
|
|
|
sum += to_ratio(period, runtime);
|
|
}
|
|
|
|
if (sum > total)
|
|
return -EINVAL;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
|
|
{
|
|
int ret;
|
|
|
|
struct rt_schedulable_data data = {
|
|
.tg = tg,
|
|
.rt_period = period,
|
|
.rt_runtime = runtime,
|
|
};
|
|
|
|
rcu_read_lock();
|
|
ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
|
|
rcu_read_unlock();
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int tg_set_rt_bandwidth(struct task_group *tg,
|
|
u64 rt_period, u64 rt_runtime)
|
|
{
|
|
int i, err = 0;
|
|
|
|
/*
|
|
* Disallowing the root group RT runtime is BAD, it would disallow the
|
|
* kernel creating (and or operating) RT threads.
|
|
*/
|
|
if (tg == &root_task_group && rt_runtime == 0)
|
|
return -EINVAL;
|
|
|
|
/* No period doesn't make any sense. */
|
|
if (rt_period == 0)
|
|
return -EINVAL;
|
|
|
|
mutex_lock(&rt_constraints_mutex);
|
|
read_lock(&tasklist_lock);
|
|
err = __rt_schedulable(tg, rt_period, rt_runtime);
|
|
if (err)
|
|
goto unlock;
|
|
|
|
raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
|
|
tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
|
|
tg->rt_bandwidth.rt_runtime = rt_runtime;
|
|
|
|
for_each_possible_cpu(i) {
|
|
struct rt_rq *rt_rq = tg->rt_rq[i];
|
|
|
|
raw_spin_lock(&rt_rq->rt_runtime_lock);
|
|
rt_rq->rt_runtime = rt_runtime;
|
|
raw_spin_unlock(&rt_rq->rt_runtime_lock);
|
|
}
|
|
raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
|
|
unlock:
|
|
read_unlock(&tasklist_lock);
|
|
mutex_unlock(&rt_constraints_mutex);
|
|
|
|
return err;
|
|
}
|
|
|
|
static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
|
|
{
|
|
u64 rt_runtime, rt_period;
|
|
|
|
rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
|
|
rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
|
|
if (rt_runtime_us < 0)
|
|
rt_runtime = RUNTIME_INF;
|
|
|
|
return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
|
|
}
|
|
|
|
static long sched_group_rt_runtime(struct task_group *tg)
|
|
{
|
|
u64 rt_runtime_us;
|
|
|
|
if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
|
|
return -1;
|
|
|
|
rt_runtime_us = tg->rt_bandwidth.rt_runtime;
|
|
do_div(rt_runtime_us, NSEC_PER_USEC);
|
|
return rt_runtime_us;
|
|
}
|
|
|
|
static int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us)
|
|
{
|
|
u64 rt_runtime, rt_period;
|
|
|
|
rt_period = rt_period_us * NSEC_PER_USEC;
|
|
rt_runtime = tg->rt_bandwidth.rt_runtime;
|
|
|
|
return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
|
|
}
|
|
|
|
static long sched_group_rt_period(struct task_group *tg)
|
|
{
|
|
u64 rt_period_us;
|
|
|
|
rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
|
|
do_div(rt_period_us, NSEC_PER_USEC);
|
|
return rt_period_us;
|
|
}
|
|
#endif /* CONFIG_RT_GROUP_SCHED */
|
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
static int sched_rt_global_constraints(void)
|
|
{
|
|
int ret = 0;
|
|
|
|
mutex_lock(&rt_constraints_mutex);
|
|
read_lock(&tasklist_lock);
|
|
ret = __rt_schedulable(NULL, 0, 0);
|
|
read_unlock(&tasklist_lock);
|
|
mutex_unlock(&rt_constraints_mutex);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
|
|
{
|
|
/* Don't accept realtime tasks when there is no way for them to run */
|
|
if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
|
|
return 0;
|
|
|
|
return 1;
|
|
}
|
|
|
|
#else /* !CONFIG_RT_GROUP_SCHED */
|
|
static int sched_rt_global_constraints(void)
|
|
{
|
|
unsigned long flags;
|
|
int i;
|
|
|
|
raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
|
|
for_each_possible_cpu(i) {
|
|
struct rt_rq *rt_rq = &cpu_rq(i)->rt;
|
|
|
|
raw_spin_lock(&rt_rq->rt_runtime_lock);
|
|
rt_rq->rt_runtime = global_rt_runtime();
|
|
raw_spin_unlock(&rt_rq->rt_runtime_lock);
|
|
}
|
|
raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
|
|
|
|
return 0;
|
|
}
|
|
#endif /* CONFIG_RT_GROUP_SCHED */
|
|
|
|
static int sched_dl_global_validate(void)
|
|
{
|
|
u64 runtime = global_rt_runtime();
|
|
u64 period = global_rt_period();
|
|
u64 new_bw = to_ratio(period, runtime);
|
|
struct dl_bw *dl_b;
|
|
int cpu, ret = 0;
|
|
unsigned long flags;
|
|
|
|
/*
|
|
* Here we want to check the bandwidth not being set to some
|
|
* value smaller than the currently allocated bandwidth in
|
|
* any of the root_domains.
|
|
*
|
|
* FIXME: Cycling on all the CPUs is overdoing, but simpler than
|
|
* cycling on root_domains... Discussion on different/better
|
|
* solutions is welcome!
|
|
*/
|
|
for_each_possible_cpu(cpu) {
|
|
rcu_read_lock_sched();
|
|
dl_b = dl_bw_of(cpu);
|
|
|
|
raw_spin_lock_irqsave(&dl_b->lock, flags);
|
|
if (new_bw < dl_b->total_bw)
|
|
ret = -EBUSY;
|
|
raw_spin_unlock_irqrestore(&dl_b->lock, flags);
|
|
|
|
rcu_read_unlock_sched();
|
|
|
|
if (ret)
|
|
break;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void sched_dl_do_global(void)
|
|
{
|
|
u64 new_bw = -1;
|
|
struct dl_bw *dl_b;
|
|
int cpu;
|
|
unsigned long flags;
|
|
|
|
def_dl_bandwidth.dl_period = global_rt_period();
|
|
def_dl_bandwidth.dl_runtime = global_rt_runtime();
|
|
|
|
if (global_rt_runtime() != RUNTIME_INF)
|
|
new_bw = to_ratio(global_rt_period(), global_rt_runtime());
|
|
|
|
/*
|
|
* FIXME: As above...
|
|
*/
|
|
for_each_possible_cpu(cpu) {
|
|
rcu_read_lock_sched();
|
|
dl_b = dl_bw_of(cpu);
|
|
|
|
raw_spin_lock_irqsave(&dl_b->lock, flags);
|
|
dl_b->bw = new_bw;
|
|
raw_spin_unlock_irqrestore(&dl_b->lock, flags);
|
|
|
|
rcu_read_unlock_sched();
|
|
}
|
|
}
|
|
|
|
static int sched_rt_global_validate(void)
|
|
{
|
|
if (sysctl_sched_rt_period <= 0)
|
|
return -EINVAL;
|
|
|
|
if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
|
|
(sysctl_sched_rt_runtime > sysctl_sched_rt_period))
|
|
return -EINVAL;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void sched_rt_do_global(void)
|
|
{
|
|
def_rt_bandwidth.rt_runtime = global_rt_runtime();
|
|
def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
|
|
}
|
|
|
|
int sched_rt_handler(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *lenp,
|
|
loff_t *ppos)
|
|
{
|
|
int old_period, old_runtime;
|
|
static DEFINE_MUTEX(mutex);
|
|
int ret;
|
|
|
|
mutex_lock(&mutex);
|
|
old_period = sysctl_sched_rt_period;
|
|
old_runtime = sysctl_sched_rt_runtime;
|
|
|
|
ret = proc_dointvec(table, write, buffer, lenp, ppos);
|
|
|
|
if (!ret && write) {
|
|
ret = sched_rt_global_validate();
|
|
if (ret)
|
|
goto undo;
|
|
|
|
ret = sched_dl_global_validate();
|
|
if (ret)
|
|
goto undo;
|
|
|
|
ret = sched_rt_global_constraints();
|
|
if (ret)
|
|
goto undo;
|
|
|
|
sched_rt_do_global();
|
|
sched_dl_do_global();
|
|
}
|
|
if (0) {
|
|
undo:
|
|
sysctl_sched_rt_period = old_period;
|
|
sysctl_sched_rt_runtime = old_runtime;
|
|
}
|
|
mutex_unlock(&mutex);
|
|
|
|
return ret;
|
|
}
|
|
|
|
int sched_rr_handler(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *lenp,
|
|
loff_t *ppos)
|
|
{
|
|
int ret;
|
|
static DEFINE_MUTEX(mutex);
|
|
|
|
mutex_lock(&mutex);
|
|
ret = proc_dointvec(table, write, buffer, lenp, ppos);
|
|
/*
|
|
* Make sure that internally we keep jiffies.
|
|
* Also, writing zero resets the timeslice to default:
|
|
*/
|
|
if (!ret && write) {
|
|
sched_rr_timeslice =
|
|
sysctl_sched_rr_timeslice <= 0 ? RR_TIMESLICE :
|
|
msecs_to_jiffies(sysctl_sched_rr_timeslice);
|
|
}
|
|
mutex_unlock(&mutex);
|
|
return ret;
|
|
}
|
|
|
|
#ifdef CONFIG_CGROUP_SCHED
|
|
|
|
static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
|
|
{
|
|
return css ? container_of(css, struct task_group, css) : NULL;
|
|
}
|
|
|
|
static struct cgroup_subsys_state *
|
|
cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
|
|
{
|
|
struct task_group *parent = css_tg(parent_css);
|
|
struct task_group *tg;
|
|
|
|
if (!parent) {
|
|
/* This is early initialization for the top cgroup */
|
|
return &root_task_group.css;
|
|
}
|
|
|
|
tg = sched_create_group(parent);
|
|
if (IS_ERR(tg))
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
return &tg->css;
|
|
}
|
|
|
|
/* Expose task group only after completing cgroup initialization */
|
|
static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
|
|
{
|
|
struct task_group *tg = css_tg(css);
|
|
struct task_group *parent = css_tg(css->parent);
|
|
|
|
if (parent)
|
|
sched_online_group(tg, parent);
|
|
return 0;
|
|
}
|
|
|
|
static void cpu_cgroup_css_released(struct cgroup_subsys_state *css)
|
|
{
|
|
struct task_group *tg = css_tg(css);
|
|
|
|
sched_offline_group(tg);
|
|
}
|
|
|
|
static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
|
|
{
|
|
struct task_group *tg = css_tg(css);
|
|
|
|
/*
|
|
* Relies on the RCU grace period between css_released() and this.
|
|
*/
|
|
sched_free_group(tg);
|
|
}
|
|
|
|
/*
|
|
* This is called before wake_up_new_task(), therefore we really only
|
|
* have to set its group bits, all the other stuff does not apply.
|
|
*/
|
|
static void cpu_cgroup_fork(struct task_struct *task)
|
|
{
|
|
struct rq_flags rf;
|
|
struct rq *rq;
|
|
|
|
rq = task_rq_lock(task, &rf);
|
|
|
|
update_rq_clock(rq);
|
|
sched_change_group(task, TASK_SET_GROUP);
|
|
|
|
task_rq_unlock(rq, task, &rf);
|
|
}
|
|
|
|
static int cpu_cgroup_can_attach(struct cgroup_taskset *tset)
|
|
{
|
|
struct task_struct *task;
|
|
struct cgroup_subsys_state *css;
|
|
int ret = 0;
|
|
|
|
cgroup_taskset_for_each(task, css, tset) {
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
if (!sched_rt_can_attach(css_tg(css), task))
|
|
return -EINVAL;
|
|
#else
|
|
/* We don't support RT-tasks being in separate groups */
|
|
if (task->sched_class != &fair_sched_class)
|
|
return -EINVAL;
|
|
#endif
|
|
/*
|
|
* Serialize against wake_up_new_task() such that if its
|
|
* running, we're sure to observe its full state.
|
|
*/
|
|
raw_spin_lock_irq(&task->pi_lock);
|
|
/*
|
|
* Avoid calling sched_move_task() before wake_up_new_task()
|
|
* has happened. This would lead to problems with PELT, due to
|
|
* move wanting to detach+attach while we're not attached yet.
|
|
*/
|
|
if (task->state == TASK_NEW)
|
|
ret = -EINVAL;
|
|
raw_spin_unlock_irq(&task->pi_lock);
|
|
|
|
if (ret)
|
|
break;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static void cpu_cgroup_attach(struct cgroup_taskset *tset)
|
|
{
|
|
struct task_struct *task;
|
|
struct cgroup_subsys_state *css;
|
|
|
|
cgroup_taskset_for_each(task, css, tset)
|
|
sched_move_task(task);
|
|
}
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
|
|
struct cftype *cftype, u64 shareval)
|
|
{
|
|
return sched_group_set_shares(css_tg(css), scale_load(shareval));
|
|
}
|
|
|
|
static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
|
|
struct cftype *cft)
|
|
{
|
|
struct task_group *tg = css_tg(css);
|
|
|
|
return (u64) scale_load_down(tg->shares);
|
|
}
|
|
|
|
#ifdef CONFIG_CFS_BANDWIDTH
|
|
static DEFINE_MUTEX(cfs_constraints_mutex);
|
|
|
|
const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
|
|
const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
|
|
|
|
static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
|
|
|
|
static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
|
|
{
|
|
int i, ret = 0, runtime_enabled, runtime_was_enabled;
|
|
struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
|
|
|
|
if (tg == &root_task_group)
|
|
return -EINVAL;
|
|
|
|
/*
|
|
* Ensure we have at some amount of bandwidth every period. This is
|
|
* to prevent reaching a state of large arrears when throttled via
|
|
* entity_tick() resulting in prolonged exit starvation.
|
|
*/
|
|
if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
|
|
return -EINVAL;
|
|
|
|
/*
|
|
* Likewise, bound things on the otherside by preventing insane quota
|
|
* periods. This also allows us to normalize in computing quota
|
|
* feasibility.
|
|
*/
|
|
if (period > max_cfs_quota_period)
|
|
return -EINVAL;
|
|
|
|
/*
|
|
* Prevent race between setting of cfs_rq->runtime_enabled and
|
|
* unthrottle_offline_cfs_rqs().
|
|
*/
|
|
get_online_cpus();
|
|
mutex_lock(&cfs_constraints_mutex);
|
|
ret = __cfs_schedulable(tg, period, quota);
|
|
if (ret)
|
|
goto out_unlock;
|
|
|
|
runtime_enabled = quota != RUNTIME_INF;
|
|
runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
|
|
/*
|
|
* If we need to toggle cfs_bandwidth_used, off->on must occur
|
|
* before making related changes, and on->off must occur afterwards
|
|
*/
|
|
if (runtime_enabled && !runtime_was_enabled)
|
|
cfs_bandwidth_usage_inc();
|
|
raw_spin_lock_irq(&cfs_b->lock);
|
|
cfs_b->period = ns_to_ktime(period);
|
|
cfs_b->quota = quota;
|
|
|
|
__refill_cfs_bandwidth_runtime(cfs_b);
|
|
|
|
/* Restart the period timer (if active) to handle new period expiry: */
|
|
if (runtime_enabled)
|
|
start_cfs_bandwidth(cfs_b);
|
|
|
|
raw_spin_unlock_irq(&cfs_b->lock);
|
|
|
|
for_each_online_cpu(i) {
|
|
struct cfs_rq *cfs_rq = tg->cfs_rq[i];
|
|
struct rq *rq = cfs_rq->rq;
|
|
struct rq_flags rf;
|
|
|
|
rq_lock_irq(rq, &rf);
|
|
cfs_rq->runtime_enabled = runtime_enabled;
|
|
cfs_rq->runtime_remaining = 0;
|
|
|
|
if (cfs_rq->throttled)
|
|
unthrottle_cfs_rq(cfs_rq);
|
|
rq_unlock_irq(rq, &rf);
|
|
}
|
|
if (runtime_was_enabled && !runtime_enabled)
|
|
cfs_bandwidth_usage_dec();
|
|
out_unlock:
|
|
mutex_unlock(&cfs_constraints_mutex);
|
|
put_online_cpus();
|
|
|
|
return ret;
|
|
}
|
|
|
|
int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
|
|
{
|
|
u64 quota, period;
|
|
|
|
period = ktime_to_ns(tg->cfs_bandwidth.period);
|
|
if (cfs_quota_us < 0)
|
|
quota = RUNTIME_INF;
|
|
else
|
|
quota = (u64)cfs_quota_us * NSEC_PER_USEC;
|
|
|
|
return tg_set_cfs_bandwidth(tg, period, quota);
|
|
}
|
|
|
|
long tg_get_cfs_quota(struct task_group *tg)
|
|
{
|
|
u64 quota_us;
|
|
|
|
if (tg->cfs_bandwidth.quota == RUNTIME_INF)
|
|
return -1;
|
|
|
|
quota_us = tg->cfs_bandwidth.quota;
|
|
do_div(quota_us, NSEC_PER_USEC);
|
|
|
|
return quota_us;
|
|
}
|
|
|
|
int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
|
|
{
|
|
u64 quota, period;
|
|
|
|
period = (u64)cfs_period_us * NSEC_PER_USEC;
|
|
quota = tg->cfs_bandwidth.quota;
|
|
|
|
return tg_set_cfs_bandwidth(tg, period, quota);
|
|
}
|
|
|
|
long tg_get_cfs_period(struct task_group *tg)
|
|
{
|
|
u64 cfs_period_us;
|
|
|
|
cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
|
|
do_div(cfs_period_us, NSEC_PER_USEC);
|
|
|
|
return cfs_period_us;
|
|
}
|
|
|
|
static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
|
|
struct cftype *cft)
|
|
{
|
|
return tg_get_cfs_quota(css_tg(css));
|
|
}
|
|
|
|
static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
|
|
struct cftype *cftype, s64 cfs_quota_us)
|
|
{
|
|
return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
|
|
}
|
|
|
|
static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
|
|
struct cftype *cft)
|
|
{
|
|
return tg_get_cfs_period(css_tg(css));
|
|
}
|
|
|
|
static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
|
|
struct cftype *cftype, u64 cfs_period_us)
|
|
{
|
|
return tg_set_cfs_period(css_tg(css), cfs_period_us);
|
|
}
|
|
|
|
struct cfs_schedulable_data {
|
|
struct task_group *tg;
|
|
u64 period, quota;
|
|
};
|
|
|
|
/*
|
|
* normalize group quota/period to be quota/max_period
|
|
* note: units are usecs
|
|
*/
|
|
static u64 normalize_cfs_quota(struct task_group *tg,
|
|
struct cfs_schedulable_data *d)
|
|
{
|
|
u64 quota, period;
|
|
|
|
if (tg == d->tg) {
|
|
period = d->period;
|
|
quota = d->quota;
|
|
} else {
|
|
period = tg_get_cfs_period(tg);
|
|
quota = tg_get_cfs_quota(tg);
|
|
}
|
|
|
|
/* note: these should typically be equivalent */
|
|
if (quota == RUNTIME_INF || quota == -1)
|
|
return RUNTIME_INF;
|
|
|
|
return to_ratio(period, quota);
|
|
}
|
|
|
|
static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
|
|
{
|
|
struct cfs_schedulable_data *d = data;
|
|
struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
|
|
s64 quota = 0, parent_quota = -1;
|
|
|
|
if (!tg->parent) {
|
|
quota = RUNTIME_INF;
|
|
} else {
|
|
struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
|
|
|
|
quota = normalize_cfs_quota(tg, d);
|
|
parent_quota = parent_b->hierarchical_quota;
|
|
|
|
/*
|
|
* Ensure max(child_quota) <= parent_quota, inherit when no
|
|
* limit is set:
|
|
*/
|
|
if (quota == RUNTIME_INF)
|
|
quota = parent_quota;
|
|
else if (parent_quota != RUNTIME_INF && quota > parent_quota)
|
|
return -EINVAL;
|
|
}
|
|
cfs_b->hierarchical_quota = quota;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
|
|
{
|
|
int ret;
|
|
struct cfs_schedulable_data data = {
|
|
.tg = tg,
|
|
.period = period,
|
|
.quota = quota,
|
|
};
|
|
|
|
if (quota != RUNTIME_INF) {
|
|
do_div(data.period, NSEC_PER_USEC);
|
|
do_div(data.quota, NSEC_PER_USEC);
|
|
}
|
|
|
|
rcu_read_lock();
|
|
ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
|
|
rcu_read_unlock();
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int cpu_stats_show(struct seq_file *sf, void *v)
|
|
{
|
|
struct task_group *tg = css_tg(seq_css(sf));
|
|
struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
|
|
|
|
seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods);
|
|
seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled);
|
|
seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time);
|
|
|
|
return 0;
|
|
}
|
|
#endif /* CONFIG_CFS_BANDWIDTH */
|
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
|
|
struct cftype *cft, s64 val)
|
|
{
|
|
return sched_group_set_rt_runtime(css_tg(css), val);
|
|
}
|
|
|
|
static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
|
|
struct cftype *cft)
|
|
{
|
|
return sched_group_rt_runtime(css_tg(css));
|
|
}
|
|
|
|
static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
|
|
struct cftype *cftype, u64 rt_period_us)
|
|
{
|
|
return sched_group_set_rt_period(css_tg(css), rt_period_us);
|
|
}
|
|
|
|
static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
|
|
struct cftype *cft)
|
|
{
|
|
return sched_group_rt_period(css_tg(css));
|
|
}
|
|
#endif /* CONFIG_RT_GROUP_SCHED */
|
|
|
|
static struct cftype cpu_files[] = {
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
{
|
|
.name = "shares",
|
|
.read_u64 = cpu_shares_read_u64,
|
|
.write_u64 = cpu_shares_write_u64,
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_CFS_BANDWIDTH
|
|
{
|
|
.name = "cfs_quota_us",
|
|
.read_s64 = cpu_cfs_quota_read_s64,
|
|
.write_s64 = cpu_cfs_quota_write_s64,
|
|
},
|
|
{
|
|
.name = "cfs_period_us",
|
|
.read_u64 = cpu_cfs_period_read_u64,
|
|
.write_u64 = cpu_cfs_period_write_u64,
|
|
},
|
|
{
|
|
.name = "stat",
|
|
.seq_show = cpu_stats_show,
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
{
|
|
.name = "rt_runtime_us",
|
|
.read_s64 = cpu_rt_runtime_read,
|
|
.write_s64 = cpu_rt_runtime_write,
|
|
},
|
|
{
|
|
.name = "rt_period_us",
|
|
.read_u64 = cpu_rt_period_read_uint,
|
|
.write_u64 = cpu_rt_period_write_uint,
|
|
},
|
|
#endif
|
|
{ } /* Terminate */
|
|
};
|
|
|
|
struct cgroup_subsys cpu_cgrp_subsys = {
|
|
.css_alloc = cpu_cgroup_css_alloc,
|
|
.css_online = cpu_cgroup_css_online,
|
|
.css_released = cpu_cgroup_css_released,
|
|
.css_free = cpu_cgroup_css_free,
|
|
.fork = cpu_cgroup_fork,
|
|
.can_attach = cpu_cgroup_can_attach,
|
|
.attach = cpu_cgroup_attach,
|
|
.legacy_cftypes = cpu_files,
|
|
.early_init = true,
|
|
};
|
|
|
|
#endif /* CONFIG_CGROUP_SCHED */
|
|
|
|
void dump_cpu_task(int cpu)
|
|
{
|
|
pr_info("Task dump for CPU %d:\n", cpu);
|
|
sched_show_task(cpu_curr(cpu));
|
|
}
|
|
|
|
/*
|
|
* Nice levels are multiplicative, with a gentle 10% change for every
|
|
* nice level changed. I.e. when a CPU-bound task goes from nice 0 to
|
|
* nice 1, it will get ~10% less CPU time than another CPU-bound task
|
|
* that remained on nice 0.
|
|
*
|
|
* The "10% effect" is relative and cumulative: from _any_ nice level,
|
|
* if you go up 1 level, it's -10% CPU usage, if you go down 1 level
|
|
* it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
|
|
* If a task goes up by ~10% and another task goes down by ~10% then
|
|
* the relative distance between them is ~25%.)
|
|
*/
|
|
const int sched_prio_to_weight[40] = {
|
|
/* -20 */ 88761, 71755, 56483, 46273, 36291,
|
|
/* -15 */ 29154, 23254, 18705, 14949, 11916,
|
|
/* -10 */ 9548, 7620, 6100, 4904, 3906,
|
|
/* -5 */ 3121, 2501, 1991, 1586, 1277,
|
|
/* 0 */ 1024, 820, 655, 526, 423,
|
|
/* 5 */ 335, 272, 215, 172, 137,
|
|
/* 10 */ 110, 87, 70, 56, 45,
|
|
/* 15 */ 36, 29, 23, 18, 15,
|
|
};
|
|
|
|
/*
|
|
* Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated.
|
|
*
|
|
* In cases where the weight does not change often, we can use the
|
|
* precalculated inverse to speed up arithmetics by turning divisions
|
|
* into multiplications:
|
|
*/
|
|
const u32 sched_prio_to_wmult[40] = {
|
|
/* -20 */ 48388, 59856, 76040, 92818, 118348,
|
|
/* -15 */ 147320, 184698, 229616, 287308, 360437,
|
|
/* -10 */ 449829, 563644, 704093, 875809, 1099582,
|
|
/* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326,
|
|
/* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587,
|
|
/* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126,
|
|
/* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717,
|
|
/* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
|
|
};
|