2007-07-09 16:51:58 +00:00
|
|
|
/*
|
|
|
|
* Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH)
|
|
|
|
*
|
|
|
|
* Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
|
|
|
|
*
|
|
|
|
* Interactivity improvements by Mike Galbraith
|
|
|
|
* (C) 2007 Mike Galbraith <efault@gmx.de>
|
|
|
|
*
|
|
|
|
* Various enhancements by Dmitry Adamushko.
|
|
|
|
* (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com>
|
|
|
|
*
|
|
|
|
* Group scheduling enhancements by Srivatsa Vaddagiri
|
|
|
|
* Copyright IBM Corporation, 2007
|
|
|
|
* Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
|
|
|
|
*
|
|
|
|
* Scaled math optimizations by Thomas Gleixner
|
|
|
|
* Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de>
|
2007-08-25 16:41:53 +00:00
|
|
|
*
|
|
|
|
* Adaptive scheduling granularity, math enhancements by Peter Zijlstra
|
|
|
|
* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
|
2007-07-09 16:51:58 +00:00
|
|
|
*/
|
|
|
|
|
2008-01-25 20:08:34 +00:00
|
|
|
#include <linux/latencytop.h>
|
2009-11-30 11:16:47 +00:00
|
|
|
#include <linux/sched.h>
|
2008-01-25 20:08:34 +00:00
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
/*
|
2007-08-25 16:41:53 +00:00
|
|
|
* Targeted preemption latency for CPU-bound tasks:
|
2009-09-09 13:41:37 +00:00
|
|
|
* (default: 5ms * (1 + ilog(ncpus)), units: nanoseconds)
|
2007-07-09 16:51:58 +00:00
|
|
|
*
|
2007-08-25 16:41:53 +00:00
|
|
|
* NOTE: this latency value is not the same as the concept of
|
2007-10-15 15:00:14 +00:00
|
|
|
* 'timeslice length' - timeslices in CFS are of variable length
|
|
|
|
* and have no persistent notion like in traditional, time-slice
|
|
|
|
* based scheduling concepts.
|
2007-07-09 16:51:58 +00:00
|
|
|
*
|
2007-10-15 15:00:14 +00:00
|
|
|
* (to see the precise effective timeslice length of your workload,
|
|
|
|
* run vmstat and monitor the context-switches (cs) field)
|
2007-07-09 16:51:58 +00:00
|
|
|
*/
|
2010-03-11 16:17:15 +00:00
|
|
|
unsigned int sysctl_sched_latency = 6000000ULL;
|
|
|
|
unsigned int normalized_sysctl_sched_latency = 6000000ULL;
|
2007-10-15 15:00:02 +00:00
|
|
|
|
2009-11-30 11:16:47 +00:00
|
|
|
/*
|
|
|
|
* The initial- and re-scaling of tunables is configurable
|
|
|
|
* (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
|
|
|
|
*
|
|
|
|
* Options are:
|
|
|
|
* SCHED_TUNABLESCALING_NONE - unscaled, always *1
|
|
|
|
* SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
|
|
|
|
* SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
|
|
|
|
*/
|
|
|
|
enum sched_tunable_scaling sysctl_sched_tunable_scaling
|
|
|
|
= SCHED_TUNABLESCALING_LOG;
|
|
|
|
|
2007-10-15 15:00:02 +00:00
|
|
|
/*
|
2007-11-09 21:39:37 +00:00
|
|
|
* Minimal preemption granularity for CPU-bound tasks:
|
2010-03-11 16:17:15 +00:00
|
|
|
* (default: 2 msec * (1 + ilog(ncpus)), units: nanoseconds)
|
2007-10-15 15:00:02 +00:00
|
|
|
*/
|
2010-03-11 16:17:15 +00:00
|
|
|
unsigned int sysctl_sched_min_granularity = 2000000ULL;
|
|
|
|
unsigned int normalized_sysctl_sched_min_granularity = 2000000ULL;
|
2007-08-25 16:41:53 +00:00
|
|
|
|
|
|
|
/*
|
2007-11-09 21:39:37 +00:00
|
|
|
* is kept at sysctl_sched_latency / sysctl_sched_min_granularity
|
|
|
|
*/
|
2010-03-11 16:17:15 +00:00
|
|
|
static unsigned int sched_nr_latency = 3;
|
2007-11-09 21:39:37 +00:00
|
|
|
|
|
|
|
/*
|
2009-09-09 13:41:37 +00:00
|
|
|
* After fork, child runs first. If set to 0 (default) then
|
2007-11-09 21:39:37 +00:00
|
|
|
* parent will (try to) run first.
|
2007-08-25 16:41:53 +00:00
|
|
|
*/
|
2009-09-09 13:41:37 +00:00
|
|
|
unsigned int sysctl_sched_child_runs_first __read_mostly;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2007-09-19 21:34:46 +00:00
|
|
|
/*
|
|
|
|
* sys_sched_yield() compat mode
|
|
|
|
*
|
|
|
|
* This option switches the agressive yield implementation of the
|
|
|
|
* old scheduler back on.
|
|
|
|
*/
|
|
|
|
unsigned int __read_mostly sysctl_sched_compat_yield;
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
/*
|
|
|
|
* SCHED_OTHER wake-up granularity.
|
2009-09-09 13:41:37 +00:00
|
|
|
* (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
|
2007-07-09 16:51:58 +00:00
|
|
|
*
|
|
|
|
* This option delays the preemption effects of decoupled workloads
|
|
|
|
* and reduces their over-scheduling. Synchronous workloads will still
|
|
|
|
* have immediate wakeup/sleep latencies.
|
|
|
|
*/
|
2009-09-09 13:41:37 +00:00
|
|
|
unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
|
2009-11-30 11:16:46 +00:00
|
|
|
unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2007-10-15 15:00:18 +00:00
|
|
|
const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
|
|
|
|
|
2008-10-17 17:27:03 +00:00
|
|
|
static const struct sched_class fair_sched_class;
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
/**************************************************************
|
|
|
|
* CFS operations on generic schedulable entities:
|
|
|
|
*/
|
|
|
|
|
2007-10-15 15:00:03 +00:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2007-10-15 15:00:03 +00:00
|
|
|
/* cpu runqueue to which this cfs_rq is attached */
|
2007-07-09 16:51:58 +00:00
|
|
|
static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
2007-10-15 15:00:03 +00:00
|
|
|
return cfs_rq->rq;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2007-10-15 15:00:03 +00:00
|
|
|
/* An entity is a task if it doesn't "own" a runqueue */
|
|
|
|
#define entity_is_task(se) (!se->my_q)
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2009-07-24 10:25:30 +00:00
|
|
|
static inline struct task_struct *task_of(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
|
|
WARN_ON_ONCE(!entity_is_task(se));
|
|
|
|
#endif
|
|
|
|
return container_of(se, struct task_struct, se);
|
|
|
|
}
|
|
|
|
|
2008-04-19 17:45:00 +00:00
|
|
|
/* Walk up scheduling entities hierarchy */
|
|
|
|
#define for_each_sched_entity(se) \
|
|
|
|
for (; se; se = se->parent)
|
|
|
|
|
|
|
|
static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
|
|
|
|
{
|
|
|
|
return p->se.cfs_rq;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* runqueue on which this entity is (to be) queued */
|
|
|
|
static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
return se->cfs_rq;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* runqueue "owned" by this group */
|
|
|
|
static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
|
|
|
|
{
|
|
|
|
return grp->my_q;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Given a group's cfs_rq on one cpu, return its corresponding cfs_rq on
|
|
|
|
* another cpu ('this_cpu')
|
|
|
|
*/
|
|
|
|
static inline struct cfs_rq *cpu_cfs_rq(struct cfs_rq *cfs_rq, int this_cpu)
|
|
|
|
{
|
|
|
|
return cfs_rq->tg->cfs_rq[this_cpu];
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Iterate thr' all leaf cfs_rq's on a runqueue */
|
|
|
|
#define for_each_leaf_cfs_rq(rq, cfs_rq) \
|
|
|
|
list_for_each_entry_rcu(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list)
|
|
|
|
|
|
|
|
/* Do the two (enqueued) entities belong to the same group ? */
|
|
|
|
static inline int
|
|
|
|
is_same_group(struct sched_entity *se, struct sched_entity *pse)
|
|
|
|
{
|
|
|
|
if (se->cfs_rq == pse->cfs_rq)
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct sched_entity *parent_entity(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
return se->parent;
|
|
|
|
}
|
|
|
|
|
2008-10-24 09:06:15 +00:00
|
|
|
/* return depth at which a sched entity is present in the hierarchy */
|
|
|
|
static inline int depth_se(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
int depth = 0;
|
|
|
|
|
|
|
|
for_each_sched_entity(se)
|
|
|
|
depth++;
|
|
|
|
|
|
|
|
return depth;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
find_matching_se(struct sched_entity **se, struct sched_entity **pse)
|
|
|
|
{
|
|
|
|
int se_depth, pse_depth;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* preemption test can be made between sibling entities who are in the
|
|
|
|
* same cfs_rq i.e who have a common parent. Walk up the hierarchy of
|
|
|
|
* both tasks until we find their ancestors who are siblings of common
|
|
|
|
* parent.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* First walk up until both entities are at same depth */
|
|
|
|
se_depth = depth_se(*se);
|
|
|
|
pse_depth = depth_se(*pse);
|
|
|
|
|
|
|
|
while (se_depth > pse_depth) {
|
|
|
|
se_depth--;
|
|
|
|
*se = parent_entity(*se);
|
|
|
|
}
|
|
|
|
|
|
|
|
while (pse_depth > se_depth) {
|
|
|
|
pse_depth--;
|
|
|
|
*pse = parent_entity(*pse);
|
|
|
|
}
|
|
|
|
|
|
|
|
while (!is_same_group(*se, *pse)) {
|
|
|
|
*se = parent_entity(*se);
|
|
|
|
*pse = parent_entity(*pse);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-07-24 10:25:30 +00:00
|
|
|
#else /* !CONFIG_FAIR_GROUP_SCHED */
|
|
|
|
|
|
|
|
static inline struct task_struct *task_of(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
return container_of(se, struct task_struct, se);
|
|
|
|
}
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2007-10-15 15:00:03 +00:00
|
|
|
static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
return container_of(cfs_rq, struct rq, cfs);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#define entity_is_task(se) 1
|
|
|
|
|
2008-04-19 17:45:00 +00:00
|
|
|
#define for_each_sched_entity(se) \
|
|
|
|
for (; se; se = NULL)
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2008-04-19 17:45:00 +00:00
|
|
|
static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2008-04-19 17:45:00 +00:00
|
|
|
return &task_rq(p)->cfs;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2008-04-19 17:45:00 +00:00
|
|
|
static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
struct task_struct *p = task_of(se);
|
|
|
|
struct rq *rq = task_rq(p);
|
|
|
|
|
|
|
|
return &rq->cfs;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* runqueue "owned" by this group */
|
|
|
|
static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct cfs_rq *cpu_cfs_rq(struct cfs_rq *cfs_rq, int this_cpu)
|
|
|
|
{
|
|
|
|
return &cpu_rq(this_cpu)->cfs;
|
|
|
|
}
|
|
|
|
|
|
|
|
#define for_each_leaf_cfs_rq(rq, cfs_rq) \
|
|
|
|
for (cfs_rq = &rq->cfs; cfs_rq; cfs_rq = NULL)
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
is_same_group(struct sched_entity *se, struct sched_entity *pse)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct sched_entity *parent_entity(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2008-10-24 09:06:15 +00:00
|
|
|
static inline void
|
|
|
|
find_matching_se(struct sched_entity **se, struct sched_entity **pse)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2008-04-19 17:45:00 +00:00
|
|
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
|
|
|
|
/**************************************************************
|
|
|
|
* Scheduling class tree data structure manipulation methods:
|
|
|
|
*/
|
|
|
|
|
2007-10-15 15:00:14 +00:00
|
|
|
static inline u64 max_vruntime(u64 min_vruntime, u64 vruntime)
|
2007-10-15 15:00:07 +00:00
|
|
|
{
|
2007-10-15 15:00:11 +00:00
|
|
|
s64 delta = (s64)(vruntime - min_vruntime);
|
|
|
|
if (delta > 0)
|
2007-10-15 15:00:07 +00:00
|
|
|
min_vruntime = vruntime;
|
|
|
|
|
|
|
|
return min_vruntime;
|
|
|
|
}
|
|
|
|
|
2007-10-15 15:00:14 +00:00
|
|
|
static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
|
2007-10-15 15:00:12 +00:00
|
|
|
{
|
|
|
|
s64 delta = (s64)(vruntime - min_vruntime);
|
|
|
|
if (delta < 0)
|
|
|
|
min_vruntime = vruntime;
|
|
|
|
|
|
|
|
return min_vruntime;
|
|
|
|
}
|
|
|
|
|
2009-07-16 10:32:27 +00:00
|
|
|
static inline int entity_before(struct sched_entity *a,
|
|
|
|
struct sched_entity *b)
|
|
|
|
{
|
|
|
|
return (s64)(a->vruntime - b->vruntime) < 0;
|
|
|
|
}
|
|
|
|
|
2007-10-15 15:00:14 +00:00
|
|
|
static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-10-15 15:00:05 +00:00
|
|
|
{
|
2007-10-15 15:00:07 +00:00
|
|
|
return se->vruntime - cfs_rq->min_vruntime;
|
2007-10-15 15:00:05 +00:00
|
|
|
}
|
|
|
|
|
2008-10-24 09:06:13 +00:00
|
|
|
static void update_min_vruntime(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
u64 vruntime = cfs_rq->min_vruntime;
|
|
|
|
|
|
|
|
if (cfs_rq->curr)
|
|
|
|
vruntime = cfs_rq->curr->vruntime;
|
|
|
|
|
|
|
|
if (cfs_rq->rb_leftmost) {
|
|
|
|
struct sched_entity *se = rb_entry(cfs_rq->rb_leftmost,
|
|
|
|
struct sched_entity,
|
|
|
|
run_node);
|
|
|
|
|
sched: fix update_min_vruntime
Impact: fix SCHED_IDLE latency problems
OK, so we have 1 running task A (which is obviously curr and the tree is
equally obviously empty).
'A' nicely chugs along, doing its thing, carrying min_vruntime along as it
goes.
Then some whacko speed freak SCHED_IDLE task gets inserted due to SMP
balancing, which is very likely far right, in that case
update_curr
update_min_vruntime
cfs_rq->rb_leftmost := true (the crazy task sitting in a tree)
vruntime = se->vruntime
and voila, min_vruntime is waaay right of where it ought to be.
OK, so why did I write it like that to begin with...
Aah, yes.
Say we've just dequeued current
schedule
deactivate_task(prev)
dequeue_entity
update_min_vruntime
Then we'll set
vruntime = cfs_rq->min_vruntime;
we find !cfs_rq->curr, but do find someone in the tree. Then we _must_
do vruntime = se->vruntime, because
vruntime = min_vruntime(vruntime := cfs_rq->min_vruntime, se->vruntime)
will not advance vruntime, and cause lags the other way around (which we
fixed with that initial patch: 1af5f730fc1bf7c62ec9fb2d307206e18bf40a69
(sched: more accurate min_vruntime accounting).
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Tested-by: Mike Galbraith <efault@gmx.de>
Acked-by: Mike Galbraith <efault@gmx.de>
Cc: <stable@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-01-15 13:53:39 +00:00
|
|
|
if (!cfs_rq->curr)
|
2008-10-24 09:06:13 +00:00
|
|
|
vruntime = se->vruntime;
|
|
|
|
else
|
|
|
|
vruntime = min_vruntime(vruntime, se->vruntime);
|
|
|
|
}
|
|
|
|
|
|
|
|
cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime);
|
|
|
|
}
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
/*
|
|
|
|
* Enqueue an entity into the rb-tree:
|
|
|
|
*/
|
2007-10-15 15:00:14 +00:00
|
|
|
static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
struct rb_node **link = &cfs_rq->tasks_timeline.rb_node;
|
|
|
|
struct rb_node *parent = NULL;
|
|
|
|
struct sched_entity *entry;
|
2007-10-15 15:00:05 +00:00
|
|
|
s64 key = entity_key(cfs_rq, se);
|
2007-07-09 16:51:58 +00:00
|
|
|
int leftmost = 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Find the right place in the rbtree:
|
|
|
|
*/
|
|
|
|
while (*link) {
|
|
|
|
parent = *link;
|
|
|
|
entry = rb_entry(parent, struct sched_entity, run_node);
|
|
|
|
/*
|
|
|
|
* We dont care about collisions. Nodes with
|
|
|
|
* the same key stay together.
|
|
|
|
*/
|
2007-10-15 15:00:05 +00:00
|
|
|
if (key < entity_key(cfs_rq, entry)) {
|
2007-07-09 16:51:58 +00:00
|
|
|
link = &parent->rb_left;
|
|
|
|
} else {
|
|
|
|
link = &parent->rb_right;
|
|
|
|
leftmost = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Maintain a cache of leftmost tree entries (it is frequently
|
|
|
|
* used):
|
|
|
|
*/
|
2008-10-24 09:06:13 +00:00
|
|
|
if (leftmost)
|
2007-10-15 15:00:11 +00:00
|
|
|
cfs_rq->rb_leftmost = &se->run_node;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
|
|
|
rb_link_node(&se->run_node, parent, link);
|
|
|
|
rb_insert_color(&se->run_node, &cfs_rq->tasks_timeline);
|
|
|
|
}
|
|
|
|
|
2007-10-15 15:00:14 +00:00
|
|
|
static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2008-03-14 19:55:51 +00:00
|
|
|
if (cfs_rq->rb_leftmost == &se->run_node) {
|
|
|
|
struct rb_node *next_node;
|
|
|
|
|
|
|
|
next_node = rb_next(&se->run_node);
|
|
|
|
cfs_rq->rb_leftmost = next_node;
|
|
|
|
}
|
2007-10-15 15:00:04 +00:00
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
rb_erase(&se->run_node, &cfs_rq->tasks_timeline);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct sched_entity *__pick_next_entity(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
2008-11-04 20:25:07 +00:00
|
|
|
struct rb_node *left = cfs_rq->rb_leftmost;
|
|
|
|
|
|
|
|
if (!left)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
return rb_entry(left, struct sched_entity, run_node);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2008-11-04 20:25:07 +00:00
|
|
|
static struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
|
2007-10-15 15:00:05 +00:00
|
|
|
{
|
2008-02-22 09:32:21 +00:00
|
|
|
struct rb_node *last = rb_last(&cfs_rq->tasks_timeline);
|
2007-10-15 15:00:05 +00:00
|
|
|
|
2008-02-22 07:55:53 +00:00
|
|
|
if (!last)
|
|
|
|
return NULL;
|
2008-02-22 09:32:21 +00:00
|
|
|
|
|
|
|
return rb_entry(last, struct sched_entity, run_node);
|
2007-10-15 15:00:05 +00:00
|
|
|
}
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
/**************************************************************
|
|
|
|
* Scheduling class statistics methods:
|
|
|
|
*/
|
|
|
|
|
2007-11-09 21:39:37 +00:00
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
2009-11-30 11:16:48 +00:00
|
|
|
int sched_proc_update_handler(struct ctl_table *table, int write,
|
2009-09-23 22:57:19 +00:00
|
|
|
void __user *buffer, size_t *lenp,
|
2007-11-09 21:39:37 +00:00
|
|
|
loff_t *ppos)
|
|
|
|
{
|
2009-09-23 22:57:19 +00:00
|
|
|
int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
|
2009-11-30 11:16:48 +00:00
|
|
|
int factor = get_update_sysctl_factor();
|
2007-11-09 21:39:37 +00:00
|
|
|
|
|
|
|
if (ret || !write)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency,
|
|
|
|
sysctl_sched_min_granularity);
|
|
|
|
|
2009-11-30 11:16:48 +00:00
|
|
|
#define WRT_SYSCTL(name) \
|
|
|
|
(normalized_sysctl_##name = sysctl_##name / (factor))
|
|
|
|
WRT_SYSCTL(sched_min_granularity);
|
|
|
|
WRT_SYSCTL(sched_latency);
|
|
|
|
WRT_SYSCTL(sched_wakeup_granularity);
|
|
|
|
WRT_SYSCTL(sched_shares_ratelimit);
|
|
|
|
#undef WRT_SYSCTL
|
|
|
|
|
2007-11-09 21:39:37 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif
|
2007-10-15 15:00:13 +00:00
|
|
|
|
2008-06-27 11:41:11 +00:00
|
|
|
/*
|
2008-10-17 17:27:04 +00:00
|
|
|
* delta /= w
|
2008-06-27 11:41:11 +00:00
|
|
|
*/
|
|
|
|
static inline unsigned long
|
|
|
|
calc_delta_fair(unsigned long delta, struct sched_entity *se)
|
|
|
|
{
|
2008-10-17 17:27:04 +00:00
|
|
|
if (unlikely(se->load.weight != NICE_0_LOAD))
|
|
|
|
delta = calc_delta_mine(delta, NICE_0_LOAD, &se->load);
|
2008-06-27 11:41:11 +00:00
|
|
|
|
|
|
|
return delta;
|
|
|
|
}
|
|
|
|
|
2007-10-15 15:00:13 +00:00
|
|
|
/*
|
|
|
|
* The idea is to set a period in which each task runs once.
|
|
|
|
*
|
|
|
|
* When there are too many tasks (sysctl_sched_nr_latency) we have to stretch
|
|
|
|
* this period because otherwise the slices get too small.
|
|
|
|
*
|
|
|
|
* p = (nr <= nl) ? l : l*nr/nl
|
|
|
|
*/
|
2007-10-15 15:00:04 +00:00
|
|
|
static u64 __sched_period(unsigned long nr_running)
|
|
|
|
{
|
|
|
|
u64 period = sysctl_sched_latency;
|
2007-11-09 21:39:37 +00:00
|
|
|
unsigned long nr_latency = sched_nr_latency;
|
2007-10-15 15:00:04 +00:00
|
|
|
|
|
|
|
if (unlikely(nr_running > nr_latency)) {
|
2008-01-25 20:08:21 +00:00
|
|
|
period = sysctl_sched_min_granularity;
|
2007-10-15 15:00:04 +00:00
|
|
|
period *= nr_running;
|
|
|
|
}
|
|
|
|
|
|
|
|
return period;
|
|
|
|
}
|
|
|
|
|
2007-10-15 15:00:13 +00:00
|
|
|
/*
|
|
|
|
* We calculate the wall-time slice from the period by taking a part
|
|
|
|
* proportional to the weight.
|
|
|
|
*
|
2008-10-17 17:27:04 +00:00
|
|
|
* s = p*P[w/rw]
|
2007-10-15 15:00:13 +00:00
|
|
|
*/
|
2007-10-15 15:00:05 +00:00
|
|
|
static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-08-25 16:41:53 +00:00
|
|
|
{
|
2009-01-02 11:16:42 +00:00
|
|
|
u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq);
|
2008-10-17 17:27:04 +00:00
|
|
|
|
2009-01-02 11:16:42 +00:00
|
|
|
for_each_sched_entity(se) {
|
2009-01-15 16:17:15 +00:00
|
|
|
struct load_weight *load;
|
2009-06-16 08:35:12 +00:00
|
|
|
struct load_weight lw;
|
2009-01-15 16:17:15 +00:00
|
|
|
|
|
|
|
cfs_rq = cfs_rq_of(se);
|
|
|
|
load = &cfs_rq->load;
|
2008-10-17 17:27:04 +00:00
|
|
|
|
2009-01-02 11:16:42 +00:00
|
|
|
if (unlikely(!se->on_rq)) {
|
2009-06-16 08:35:12 +00:00
|
|
|
lw = cfs_rq->load;
|
2009-01-02 11:16:42 +00:00
|
|
|
|
|
|
|
update_load_add(&lw, se->load.weight);
|
|
|
|
load = &lw;
|
|
|
|
}
|
|
|
|
slice = calc_delta_mine(slice, se->load.weight, load);
|
|
|
|
}
|
|
|
|
return slice;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2007-10-15 15:00:13 +00:00
|
|
|
/*
|
2008-04-19 17:45:00 +00:00
|
|
|
* We calculate the vruntime slice of a to be inserted task
|
2007-10-15 15:00:13 +00:00
|
|
|
*
|
2008-10-17 17:27:04 +00:00
|
|
|
* vs = s/w
|
2007-10-15 15:00:13 +00:00
|
|
|
*/
|
2008-10-17 17:27:04 +00:00
|
|
|
static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-10-15 15:00:10 +00:00
|
|
|
{
|
2008-10-17 17:27:04 +00:00
|
|
|
return calc_delta_fair(sched_slice(cfs_rq, se), se);
|
2008-06-27 11:41:11 +00:00
|
|
|
}
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
/*
|
|
|
|
* Update the current task's runtime statistics. Skip current tasks that
|
|
|
|
* are not in our scheduling class.
|
|
|
|
*/
|
|
|
|
static inline void
|
2007-10-15 15:00:03 +00:00
|
|
|
__update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
|
|
|
|
unsigned long delta_exec)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2007-10-15 15:00:06 +00:00
|
|
|
unsigned long delta_exec_weighted;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2010-03-11 02:37:45 +00:00
|
|
|
schedstat_set(curr->statistics.exec_max,
|
|
|
|
max((u64)delta_exec, curr->statistics.exec_max));
|
2007-07-09 16:51:58 +00:00
|
|
|
|
|
|
|
curr->sum_exec_runtime += delta_exec;
|
2007-10-15 15:00:06 +00:00
|
|
|
schedstat_add(cfs_rq, exec_clock, delta_exec);
|
2008-06-27 11:41:11 +00:00
|
|
|
delta_exec_weighted = calc_delta_fair(delta_exec, curr);
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
|
2007-10-15 15:00:04 +00:00
|
|
|
curr->vruntime += delta_exec_weighted;
|
2008-10-24 09:06:13 +00:00
|
|
|
update_min_vruntime(cfs_rq);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2007-08-09 09:16:47 +00:00
|
|
|
static void update_curr(struct cfs_rq *cfs_rq)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2007-10-15 15:00:03 +00:00
|
|
|
struct sched_entity *curr = cfs_rq->curr;
|
2007-10-15 15:00:03 +00:00
|
|
|
u64 now = rq_of(cfs_rq)->clock;
|
2007-07-09 16:51:58 +00:00
|
|
|
unsigned long delta_exec;
|
|
|
|
|
|
|
|
if (unlikely(!curr))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the amount of time the current task was running
|
|
|
|
* since the last time we changed load (this cannot
|
|
|
|
* overflow on 32 bits):
|
|
|
|
*/
|
2007-10-15 15:00:03 +00:00
|
|
|
delta_exec = (unsigned long)(now - curr->exec_start);
|
2008-12-16 07:45:31 +00:00
|
|
|
if (!delta_exec)
|
|
|
|
return;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2007-10-15 15:00:03 +00:00
|
|
|
__update_curr(cfs_rq, curr, delta_exec);
|
|
|
|
curr->exec_start = now;
|
2007-12-02 19:04:49 +00:00
|
|
|
|
|
|
|
if (entity_is_task(curr)) {
|
|
|
|
struct task_struct *curtask = task_of(curr);
|
|
|
|
|
2009-09-13 16:15:54 +00:00
|
|
|
trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime);
|
2007-12-02 19:04:49 +00:00
|
|
|
cpuacct_charge(curtask, delta_exec);
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-12 16:54:39 +00:00
|
|
|
account_group_exec_runtime(curtask, delta_exec);
|
2007-12-02 19:04:49 +00:00
|
|
|
}
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
2007-08-09 09:16:47 +00:00
|
|
|
update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2010-03-11 02:37:45 +00:00
|
|
|
schedstat_set(se->statistics.wait_start, rq_of(cfs_rq)->clock);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Task is being enqueued - update stats:
|
|
|
|
*/
|
2007-08-09 09:16:47 +00:00
|
|
|
static void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Are we enqueueing a waiting task? (for current tasks
|
|
|
|
* a dequeue/enqueue event is a NOP)
|
|
|
|
*/
|
2007-10-15 15:00:03 +00:00
|
|
|
if (se != cfs_rq->curr)
|
2007-08-09 09:16:47 +00:00
|
|
|
update_stats_wait_start(cfs_rq, se);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2007-08-09 09:16:47 +00:00
|
|
|
update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2010-03-11 02:37:45 +00:00
|
|
|
schedstat_set(se->statistics.wait_max, max(se->statistics.wait_max,
|
|
|
|
rq_of(cfs_rq)->clock - se->statistics.wait_start));
|
|
|
|
schedstat_set(se->statistics.wait_count, se->statistics.wait_count + 1);
|
|
|
|
schedstat_set(se->statistics.wait_sum, se->statistics.wait_sum +
|
|
|
|
rq_of(cfs_rq)->clock - se->statistics.wait_start);
|
2009-07-23 18:13:26 +00:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
|
if (entity_is_task(se)) {
|
|
|
|
trace_sched_stat_wait(task_of(se),
|
2010-03-11 02:37:45 +00:00
|
|
|
rq_of(cfs_rq)->clock - se->statistics.wait_start);
|
2009-07-23 18:13:26 +00:00
|
|
|
}
|
|
|
|
#endif
|
2010-03-11 02:37:45 +00:00
|
|
|
schedstat_set(se->statistics.wait_start, 0);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
2007-08-09 09:16:48 +00:00
|
|
|
update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Mark the end of the wait period if dequeueing a
|
|
|
|
* waiting task:
|
|
|
|
*/
|
2007-10-15 15:00:03 +00:00
|
|
|
if (se != cfs_rq->curr)
|
2007-08-09 09:16:47 +00:00
|
|
|
update_stats_wait_end(cfs_rq, se);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We are picking a new current task - update its stats:
|
|
|
|
*/
|
|
|
|
static inline void
|
2007-08-09 09:16:47 +00:00
|
|
|
update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We are starting a new run period:
|
|
|
|
*/
|
2007-08-09 09:16:47 +00:00
|
|
|
se->exec_start = rq_of(cfs_rq)->clock;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**************************************************
|
|
|
|
* Scheduling class queueing methods:
|
|
|
|
*/
|
|
|
|
|
2008-06-27 11:41:14 +00:00
|
|
|
#if defined CONFIG_SMP && defined CONFIG_FAIR_GROUP_SCHED
|
|
|
|
static void
|
|
|
|
add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
|
|
|
|
{
|
|
|
|
cfs_rq->task_weight += weight;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline void
|
|
|
|
add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2007-10-15 15:00:07 +00:00
|
|
|
static void
|
|
|
|
account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
|
|
|
{
|
|
|
|
update_load_add(&cfs_rq->load, se->load.weight);
|
2008-06-27 11:41:14 +00:00
|
|
|
if (!parent_entity(se))
|
|
|
|
inc_cpu_load(rq_of(cfs_rq), se->load.weight);
|
2008-09-25 04:23:54 +00:00
|
|
|
if (entity_is_task(se)) {
|
2008-06-27 11:41:14 +00:00
|
|
|
add_cfs_task_weight(cfs_rq, se->load.weight);
|
2008-09-25 04:23:54 +00:00
|
|
|
list_add(&se->group_node, &cfs_rq->tasks);
|
|
|
|
}
|
2007-10-15 15:00:07 +00:00
|
|
|
cfs_rq->nr_running++;
|
|
|
|
se->on_rq = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
|
|
|
{
|
|
|
|
update_load_sub(&cfs_rq->load, se->load.weight);
|
2008-06-27 11:41:14 +00:00
|
|
|
if (!parent_entity(se))
|
|
|
|
dec_cpu_load(rq_of(cfs_rq), se->load.weight);
|
2008-09-25 04:23:54 +00:00
|
|
|
if (entity_is_task(se)) {
|
2008-06-27 11:41:14 +00:00
|
|
|
add_cfs_task_weight(cfs_rq, -se->load.weight);
|
2008-09-25 04:23:54 +00:00
|
|
|
list_del_init(&se->group_node);
|
|
|
|
}
|
2007-10-15 15:00:07 +00:00
|
|
|
cfs_rq->nr_running--;
|
|
|
|
se->on_rq = 0;
|
|
|
|
}
|
|
|
|
|
2007-08-09 09:16:48 +00:00
|
|
|
static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
2009-07-23 18:13:26 +00:00
|
|
|
struct task_struct *tsk = NULL;
|
|
|
|
|
|
|
|
if (entity_is_task(se))
|
|
|
|
tsk = task_of(se);
|
|
|
|
|
2010-03-11 02:37:45 +00:00
|
|
|
if (se->statistics.sleep_start) {
|
|
|
|
u64 delta = rq_of(cfs_rq)->clock - se->statistics.sleep_start;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
|
|
|
if ((s64)delta < 0)
|
|
|
|
delta = 0;
|
|
|
|
|
2010-03-11 02:37:45 +00:00
|
|
|
if (unlikely(delta > se->statistics.sleep_max))
|
|
|
|
se->statistics.sleep_max = delta;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2010-03-11 02:37:45 +00:00
|
|
|
se->statistics.sleep_start = 0;
|
|
|
|
se->statistics.sum_sleep_runtime += delta;
|
2008-01-25 20:08:34 +00:00
|
|
|
|
2009-07-23 18:13:26 +00:00
|
|
|
if (tsk) {
|
2009-07-23 18:13:26 +00:00
|
|
|
account_scheduler_latency(tsk, delta >> 10, 1);
|
2009-07-23 18:13:26 +00:00
|
|
|
trace_sched_stat_sleep(tsk, delta);
|
|
|
|
}
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
2010-03-11 02:37:45 +00:00
|
|
|
if (se->statistics.block_start) {
|
|
|
|
u64 delta = rq_of(cfs_rq)->clock - se->statistics.block_start;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
|
|
|
if ((s64)delta < 0)
|
|
|
|
delta = 0;
|
|
|
|
|
2010-03-11 02:37:45 +00:00
|
|
|
if (unlikely(delta > se->statistics.block_max))
|
|
|
|
se->statistics.block_max = delta;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2010-03-11 02:37:45 +00:00
|
|
|
se->statistics.block_start = 0;
|
|
|
|
se->statistics.sum_sleep_runtime += delta;
|
2007-10-02 12:13:08 +00:00
|
|
|
|
2009-07-23 18:13:26 +00:00
|
|
|
if (tsk) {
|
2009-07-20 18:26:58 +00:00
|
|
|
if (tsk->in_iowait) {
|
2010-03-11 02:37:45 +00:00
|
|
|
se->statistics.iowait_sum += delta;
|
|
|
|
se->statistics.iowait_count++;
|
2009-07-23 18:13:26 +00:00
|
|
|
trace_sched_stat_iowait(tsk, delta);
|
2009-07-20 18:26:58 +00:00
|
|
|
}
|
|
|
|
|
2009-07-23 18:13:26 +00:00
|
|
|
/*
|
|
|
|
* Blocking time is in units of nanosecs, so shift by
|
|
|
|
* 20 to get a milliseconds-range estimation of the
|
|
|
|
* amount of time that the task spent sleeping:
|
|
|
|
*/
|
|
|
|
if (unlikely(prof_on == SLEEP_PROFILING)) {
|
|
|
|
profile_hits(SLEEP_PROFILING,
|
|
|
|
(void *)get_wchan(tsk),
|
|
|
|
delta >> 20);
|
|
|
|
}
|
|
|
|
account_scheduler_latency(tsk, delta >> 10, 0);
|
2007-10-02 12:13:08 +00:00
|
|
|
}
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2007-10-15 15:00:10 +00:00
|
|
|
static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
|
|
s64 d = se->vruntime - cfs_rq->min_vruntime;
|
|
|
|
|
|
|
|
if (d < 0)
|
|
|
|
d = -d;
|
|
|
|
|
|
|
|
if (d > 3*sysctl_sched_latency)
|
|
|
|
schedstat_inc(cfs_rq, nr_spread_over);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2007-10-15 15:00:05 +00:00
|
|
|
static void
|
|
|
|
place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
|
|
|
|
{
|
2008-10-24 09:06:13 +00:00
|
|
|
u64 vruntime = cfs_rq->min_vruntime;
|
2007-10-15 15:00:05 +00:00
|
|
|
|
2007-11-09 21:39:37 +00:00
|
|
|
/*
|
|
|
|
* The 'current' period is already promised to the current tasks,
|
|
|
|
* however the extra weight of the new task will slow them down a
|
|
|
|
* little, place the new task so that it fits in the slot that
|
|
|
|
* stays open at the end.
|
|
|
|
*/
|
2007-10-15 15:00:05 +00:00
|
|
|
if (initial && sched_feat(START_DEBIT))
|
2008-10-17 17:27:04 +00:00
|
|
|
vruntime += sched_vslice(cfs_rq, se);
|
2007-10-15 15:00:05 +00:00
|
|
|
|
2009-09-18 07:19:25 +00:00
|
|
|
/* sleeps up to a single latency don't count. */
|
2010-03-11 16:17:17 +00:00
|
|
|
if (!initial) {
|
2009-09-18 07:19:25 +00:00
|
|
|
unsigned long thresh = sysctl_sched_latency;
|
2008-06-27 11:41:11 +00:00
|
|
|
|
2009-09-18 07:19:25 +00:00
|
|
|
/*
|
|
|
|
* Halve their sleep time's effect, to allow
|
|
|
|
* for a gentler effect of sleepers:
|
|
|
|
*/
|
|
|
|
if (sched_feat(GENTLE_FAIR_SLEEPERS))
|
|
|
|
thresh >>= 1;
|
2009-09-16 06:54:45 +00:00
|
|
|
|
2009-09-18 07:19:25 +00:00
|
|
|
vruntime -= thresh;
|
2007-10-15 15:00:05 +00:00
|
|
|
}
|
|
|
|
|
sched: Ensure that a child can't gain time over it's parent after fork()
A fork/exec load is usually "pass the baton", so the child
should never be placed behind the parent. With START_DEBIT we
make room for the new task, but with child_runs_first, that
room comes out of the _parent's_ hide. There's nothing to say
that the parent wasn't ahead of min_vruntime at fork() time,
which means that the "baton carrier", who is essentially the
parent in drag, can gain time and increase scheduling latencies
for waiters.
With NEW_FAIR_SLEEPERS + START_DEBIT + child_runs_first
enabled, we essentially pass the sleeper fairness off to the
child, which is fine, but if we don't base placement on the
parent's updated vruntime, we can end up compounding latency
woes if the child itself then does fork/exec. The debit
incurred at fork doesn't hurt the parent who is then going to
sleep and maybe exit, but the child who acquires the error
harms all comers.
This improves latencies of make -j<n> kernel build workloads.
Reported-by: Jens Axboe <jens.axboe@oracle.com>
Signed-off-by: Mike Galbraith <efault@gmx.de>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-08 09:12:28 +00:00
|
|
|
/* ensure we never gain time by being placed backwards. */
|
|
|
|
vruntime = max_vruntime(se->vruntime, vruntime);
|
|
|
|
|
2007-10-15 15:00:10 +00:00
|
|
|
se->vruntime = vruntime;
|
2007-10-15 15:00:05 +00:00
|
|
|
}
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
static void
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
/*
|
|
|
|
* Update the normalized vruntime before updating min_vruntime
|
|
|
|
* through callig update_curr().
|
|
|
|
*/
|
2010-03-24 15:38:48 +00:00
|
|
|
if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_WAKING))
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
se->vruntime += cfs_rq->min_vruntime;
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
/*
|
2007-10-15 15:00:13 +00:00
|
|
|
* Update run-time statistics of the 'current'.
|
2007-07-09 16:51:58 +00:00
|
|
|
*/
|
2007-08-09 09:16:47 +00:00
|
|
|
update_curr(cfs_rq);
|
2008-05-05 21:56:17 +00:00
|
|
|
account_entity_enqueue(cfs_rq, se);
|
2007-07-09 16:51:58 +00:00
|
|
|
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
if (flags & ENQUEUE_WAKEUP) {
|
2007-10-15 15:00:05 +00:00
|
|
|
place_entity(cfs_rq, se, 0);
|
2007-08-09 09:16:48 +00:00
|
|
|
enqueue_sleeper(cfs_rq, se);
|
2007-10-15 15:00:04 +00:00
|
|
|
}
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2007-08-09 09:16:47 +00:00
|
|
|
update_stats_enqueue(cfs_rq, se);
|
2007-10-15 15:00:10 +00:00
|
|
|
check_spread(cfs_rq, se);
|
2007-10-15 15:00:08 +00:00
|
|
|
if (se != cfs_rq->curr)
|
|
|
|
__enqueue_entity(cfs_rq, se);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2009-01-28 13:51:40 +00:00
|
|
|
static void __clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2008-11-11 10:52:33 +00:00
|
|
|
{
|
2009-09-17 07:01:20 +00:00
|
|
|
if (!se || cfs_rq->last == se)
|
2008-11-11 10:52:33 +00:00
|
|
|
cfs_rq->last = NULL;
|
|
|
|
|
2009-09-17 07:01:20 +00:00
|
|
|
if (!se || cfs_rq->next == se)
|
2008-11-11 10:52:33 +00:00
|
|
|
cfs_rq->next = NULL;
|
|
|
|
}
|
|
|
|
|
2009-01-28 13:51:40 +00:00
|
|
|
static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
|
|
|
{
|
|
|
|
for_each_sched_entity(se)
|
|
|
|
__clear_buddies(cfs_rq_of(se), se);
|
|
|
|
}
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
static void
|
2010-03-24 15:38:48 +00:00
|
|
|
dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2007-10-15 15:00:13 +00:00
|
|
|
/*
|
|
|
|
* Update run-time statistics of the 'current'.
|
|
|
|
*/
|
|
|
|
update_curr(cfs_rq);
|
|
|
|
|
2007-08-09 09:16:48 +00:00
|
|
|
update_stats_dequeue(cfs_rq, se);
|
2010-03-24 15:38:48 +00:00
|
|
|
if (flags & DEQUEUE_SLEEP) {
|
2007-10-15 15:00:10 +00:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
2007-07-09 16:51:58 +00:00
|
|
|
if (entity_is_task(se)) {
|
|
|
|
struct task_struct *tsk = task_of(se);
|
|
|
|
|
|
|
|
if (tsk->state & TASK_INTERRUPTIBLE)
|
2010-03-11 02:37:45 +00:00
|
|
|
se->statistics.sleep_start = rq_of(cfs_rq)->clock;
|
2007-07-09 16:51:58 +00:00
|
|
|
if (tsk->state & TASK_UNINTERRUPTIBLE)
|
2010-03-11 02:37:45 +00:00
|
|
|
se->statistics.block_start = rq_of(cfs_rq)->clock;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
2007-10-15 15:00:06 +00:00
|
|
|
#endif
|
2007-10-15 15:00:10 +00:00
|
|
|
}
|
|
|
|
|
2008-11-11 10:52:33 +00:00
|
|
|
clear_buddies(cfs_rq, se);
|
sched: backward looking buddy
Impact: improve/change/fix wakeup-buddy scheduling
Currently we only have a forward looking buddy, that is, we prefer to
schedule to the task we last woke up, under the presumption that its
going to consume the data we just produced, and therefore will have
cache hot benefits.
This allows co-waking producer/consumer task pairs to run ahead of the
pack for a little while, keeping their cache warm. Without this, we
would interleave all pairs, utterly trashing the cache.
This patch introduces a backward looking buddy, that is, suppose that
in the above scenario, the consumer preempts the producer before it
can go to sleep, we will therefore miss the wakeup from consumer to
producer (its already running, after all), breaking the cycle and
reverting to the cache-trashing interleaved schedule pattern.
The backward buddy will try to schedule back to the task that woke us
up in case the forward buddy is not available, under the assumption
that the last task will be the one with the most cache hot task around
barring current.
This will basically allow a task to continue after it got preempted.
In order to avoid starvation, we allow either buddy to get wakeup_gran
ahead of the pack.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-04 20:25:09 +00:00
|
|
|
|
2007-10-15 15:00:08 +00:00
|
|
|
if (se != cfs_rq->curr)
|
2007-10-15 15:00:07 +00:00
|
|
|
__dequeue_entity(cfs_rq, se);
|
|
|
|
account_entity_dequeue(cfs_rq, se);
|
2008-10-24 09:06:13 +00:00
|
|
|
update_min_vruntime(cfs_rq);
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Normalize the entity after updating the min_vruntime because the
|
|
|
|
* update can refer to the ->curr item and we need to reflect this
|
|
|
|
* movement in our normalized position.
|
|
|
|
*/
|
2010-03-24 15:38:48 +00:00
|
|
|
if (!(flags & DEQUEUE_SLEEP))
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
se->vruntime -= cfs_rq->min_vruntime;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Preempt the current task with a newly woken task if needed:
|
|
|
|
*/
|
2007-09-05 12:32:49 +00:00
|
|
|
static void
|
2007-10-15 15:00:05 +00:00
|
|
|
check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2007-09-05 12:32:49 +00:00
|
|
|
unsigned long ideal_runtime, delta_exec;
|
|
|
|
|
2007-10-15 15:00:05 +00:00
|
|
|
ideal_runtime = sched_slice(cfs_rq, curr);
|
2007-09-05 12:32:49 +00:00
|
|
|
delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
|
2009-01-28 13:51:39 +00:00
|
|
|
if (delta_exec > ideal_runtime) {
|
2007-07-09 16:51:58 +00:00
|
|
|
resched_task(rq_of(cfs_rq)->curr);
|
2009-01-28 13:51:39 +00:00
|
|
|
/*
|
|
|
|
* The current task ran long enough, ensure it doesn't get
|
|
|
|
* re-elected due to buddy favours.
|
|
|
|
*/
|
|
|
|
clear_buddies(cfs_rq, curr);
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 21:09:22 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ensure that a task that missed wakeup preemption by a
|
|
|
|
* narrow margin doesn't have to wait for a full slice.
|
|
|
|
* This also mitigates buddy induced latencies under load.
|
|
|
|
*/
|
|
|
|
if (!sched_feat(WAKEUP_PREEMPT))
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (delta_exec < sysctl_sched_min_granularity)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (cfs_rq->nr_running > 1) {
|
|
|
|
struct sched_entity *se = __pick_next_entity(cfs_rq);
|
|
|
|
s64 delta = curr->vruntime - se->vruntime;
|
|
|
|
|
|
|
|
if (delta > ideal_runtime)
|
|
|
|
resched_task(rq_of(cfs_rq)->curr);
|
2009-01-28 13:51:39 +00:00
|
|
|
}
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2007-10-15 15:00:08 +00:00
|
|
|
static void
|
2007-08-09 09:16:48 +00:00
|
|
|
set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2007-10-15 15:00:08 +00:00
|
|
|
/* 'current' is not kept within the tree. */
|
|
|
|
if (se->on_rq) {
|
|
|
|
/*
|
|
|
|
* Any task has to be enqueued before it get to execute on
|
|
|
|
* a CPU. So account for the time it spent waiting on the
|
|
|
|
* runqueue.
|
|
|
|
*/
|
|
|
|
update_stats_wait_end(cfs_rq, se);
|
|
|
|
__dequeue_entity(cfs_rq, se);
|
|
|
|
}
|
|
|
|
|
2007-08-09 09:16:47 +00:00
|
|
|
update_stats_curr_start(cfs_rq, se);
|
2007-10-15 15:00:03 +00:00
|
|
|
cfs_rq->curr = se;
|
2007-10-15 15:00:02 +00:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
|
/*
|
|
|
|
* Track our maximum slice length, if the CPU's load is at
|
|
|
|
* least twice that of our own weight (i.e. dont track it
|
|
|
|
* when there are only lesser-weight tasks around):
|
|
|
|
*/
|
2007-10-15 15:00:06 +00:00
|
|
|
if (rq_of(cfs_rq)->load.weight >= 2*se->load.weight) {
|
2010-03-11 02:37:45 +00:00
|
|
|
se->statistics.slice_max = max(se->statistics.slice_max,
|
2007-10-15 15:00:02 +00:00
|
|
|
se->sum_exec_runtime - se->prev_sum_exec_runtime);
|
|
|
|
}
|
|
|
|
#endif
|
2007-09-05 12:32:49 +00:00
|
|
|
se->prev_sum_exec_runtime = se->sum_exec_runtime;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2008-10-24 09:06:16 +00:00
|
|
|
static int
|
|
|
|
wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
|
|
|
|
|
2008-11-04 20:25:07 +00:00
|
|
|
static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq)
|
2008-03-14 20:12:12 +00:00
|
|
|
{
|
2008-11-04 20:25:07 +00:00
|
|
|
struct sched_entity *se = __pick_next_entity(cfs_rq);
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 21:09:22 +00:00
|
|
|
struct sched_entity *left = se;
|
2008-11-04 20:25:07 +00:00
|
|
|
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 21:09:22 +00:00
|
|
|
if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1)
|
|
|
|
se = cfs_rq->next;
|
2008-03-14 20:12:12 +00:00
|
|
|
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 21:09:22 +00:00
|
|
|
/*
|
|
|
|
* Prefer last buddy, try to return the CPU to a preempted task.
|
|
|
|
*/
|
|
|
|
if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1)
|
|
|
|
se = cfs_rq->last;
|
|
|
|
|
|
|
|
clear_buddies(cfs_rq, se);
|
sched: backward looking buddy
Impact: improve/change/fix wakeup-buddy scheduling
Currently we only have a forward looking buddy, that is, we prefer to
schedule to the task we last woke up, under the presumption that its
going to consume the data we just produced, and therefore will have
cache hot benefits.
This allows co-waking producer/consumer task pairs to run ahead of the
pack for a little while, keeping their cache warm. Without this, we
would interleave all pairs, utterly trashing the cache.
This patch introduces a backward looking buddy, that is, suppose that
in the above scenario, the consumer preempts the producer before it
can go to sleep, we will therefore miss the wakeup from consumer to
producer (its already running, after all), breaking the cycle and
reverting to the cache-trashing interleaved schedule pattern.
The backward buddy will try to schedule back to the task that woke us
up in case the forward buddy is not available, under the assumption
that the last task will be the one with the most cache hot task around
barring current.
This will basically allow a task to continue after it got preempted.
In order to avoid starvation, we allow either buddy to get wakeup_gran
ahead of the pack.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-04 20:25:09 +00:00
|
|
|
|
|
|
|
return se;
|
2008-03-14 20:12:12 +00:00
|
|
|
}
|
|
|
|
|
2007-08-09 09:16:48 +00:00
|
|
|
static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If still on the runqueue then deactivate_task()
|
|
|
|
* was not called and update_curr() has to be done:
|
|
|
|
*/
|
|
|
|
if (prev->on_rq)
|
2007-08-09 09:16:47 +00:00
|
|
|
update_curr(cfs_rq);
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2007-10-15 15:00:10 +00:00
|
|
|
check_spread(cfs_rq, prev);
|
2007-10-15 15:00:07 +00:00
|
|
|
if (prev->on_rq) {
|
2007-08-09 09:16:47 +00:00
|
|
|
update_stats_wait_start(cfs_rq, prev);
|
2007-10-15 15:00:07 +00:00
|
|
|
/* Put 'current' back into the tree. */
|
|
|
|
__enqueue_entity(cfs_rq, prev);
|
|
|
|
}
|
2007-10-15 15:00:03 +00:00
|
|
|
cfs_rq->curr = NULL;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2008-01-25 20:08:29 +00:00
|
|
|
static void
|
|
|
|
entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
/*
|
2007-10-15 15:00:07 +00:00
|
|
|
* Update run-time statistics of the 'current'.
|
2007-07-09 16:51:58 +00:00
|
|
|
*/
|
2007-10-15 15:00:07 +00:00
|
|
|
update_curr(cfs_rq);
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2008-01-25 20:08:29 +00:00
|
|
|
#ifdef CONFIG_SCHED_HRTICK
|
|
|
|
/*
|
|
|
|
* queued ticks are scheduled to match the slice, so don't bother
|
|
|
|
* validating it and just reschedule.
|
|
|
|
*/
|
2008-04-25 01:17:55 +00:00
|
|
|
if (queued) {
|
|
|
|
resched_task(rq_of(cfs_rq)->curr);
|
|
|
|
return;
|
|
|
|
}
|
2008-01-25 20:08:29 +00:00
|
|
|
/*
|
|
|
|
* don't let the period tick interfere with the hrtick preemption
|
|
|
|
*/
|
|
|
|
if (!sched_feat(DOUBLE_TICK) &&
|
|
|
|
hrtimer_active(&rq_of(cfs_rq)->hrtick_timer))
|
|
|
|
return;
|
|
|
|
#endif
|
|
|
|
|
2007-10-15 15:00:14 +00:00
|
|
|
if (cfs_rq->nr_running > 1 || !sched_feat(WAKEUP_PREEMPT))
|
2007-10-15 15:00:05 +00:00
|
|
|
check_preempt_tick(cfs_rq, curr);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**************************************************
|
|
|
|
* CFS operations on tasks:
|
|
|
|
*/
|
|
|
|
|
2008-01-25 20:08:29 +00:00
|
|
|
#ifdef CONFIG_SCHED_HRTICK
|
|
|
|
static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
|
|
|
|
{
|
|
|
|
struct sched_entity *se = &p->se;
|
|
|
|
struct cfs_rq *cfs_rq = cfs_rq_of(se);
|
|
|
|
|
|
|
|
WARN_ON(task_rq(p) != rq);
|
|
|
|
|
|
|
|
if (hrtick_enabled(rq) && cfs_rq->nr_running > 1) {
|
|
|
|
u64 slice = sched_slice(cfs_rq, se);
|
|
|
|
u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;
|
|
|
|
s64 delta = slice - ran;
|
|
|
|
|
|
|
|
if (delta < 0) {
|
|
|
|
if (rq->curr == p)
|
|
|
|
resched_task(p);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Don't schedule slices shorter than 10000ns, that just
|
|
|
|
* doesn't make sense. Rely on vruntime for fairness.
|
|
|
|
*/
|
2008-07-18 16:01:23 +00:00
|
|
|
if (rq->curr != p)
|
2008-07-28 09:53:11 +00:00
|
|
|
delta = max_t(s64, 10000LL, delta);
|
2008-01-25 20:08:29 +00:00
|
|
|
|
2008-07-18 16:01:23 +00:00
|
|
|
hrtick_start(rq, delta);
|
2008-01-25 20:08:29 +00:00
|
|
|
}
|
|
|
|
}
|
2008-10-17 17:27:03 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* called from enqueue/dequeue and updates the hrtick when the
|
|
|
|
* current task is from our class and nr_running is low enough
|
|
|
|
* to matter.
|
|
|
|
*/
|
|
|
|
static void hrtick_update(struct rq *rq)
|
|
|
|
{
|
|
|
|
struct task_struct *curr = rq->curr;
|
|
|
|
|
|
|
|
if (curr->sched_class != &fair_sched_class)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency)
|
|
|
|
hrtick_start_fair(rq, curr);
|
|
|
|
}
|
2008-06-24 18:09:43 +00:00
|
|
|
#else /* !CONFIG_SCHED_HRTICK */
|
2008-01-25 20:08:29 +00:00
|
|
|
static inline void
|
|
|
|
hrtick_start_fair(struct rq *rq, struct task_struct *p)
|
|
|
|
{
|
|
|
|
}
|
2008-10-17 17:27:03 +00:00
|
|
|
|
|
|
|
static inline void hrtick_update(struct rq *rq)
|
|
|
|
{
|
|
|
|
}
|
2008-01-25 20:08:29 +00:00
|
|
|
#endif
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
/*
|
|
|
|
* The enqueue_task method is called before nr_running is
|
|
|
|
* increased. Here we update the fair scheduling stats and
|
|
|
|
* then put the task into the rbtree:
|
|
|
|
*/
|
2010-01-20 20:58:57 +00:00
|
|
|
static void
|
2010-03-24 15:38:48 +00:00
|
|
|
enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
2008-02-25 16:34:02 +00:00
|
|
|
struct sched_entity *se = &p->se;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
|
|
|
for_each_sched_entity(se) {
|
2008-02-25 16:34:02 +00:00
|
|
|
if (se->on_rq)
|
2007-07-09 16:51:58 +00:00
|
|
|
break;
|
|
|
|
cfs_rq = cfs_rq_of(se);
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
enqueue_entity(cfs_rq, se, flags);
|
|
|
|
flags = ENQUEUE_WAKEUP;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
2008-01-25 20:08:29 +00:00
|
|
|
|
2008-10-17 17:27:03 +00:00
|
|
|
hrtick_update(rq);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The dequeue_task method is called before nr_running is
|
|
|
|
* decreased. We remove the task from the rbtree and
|
|
|
|
* update the fair scheduling stats:
|
|
|
|
*/
|
2010-03-24 15:38:48 +00:00
|
|
|
static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
2008-02-25 16:34:02 +00:00
|
|
|
struct sched_entity *se = &p->se;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
|
|
|
for_each_sched_entity(se) {
|
|
|
|
cfs_rq = cfs_rq_of(se);
|
2010-03-24 15:38:48 +00:00
|
|
|
dequeue_entity(cfs_rq, se, flags);
|
2007-07-09 16:51:58 +00:00
|
|
|
/* Don't dequeue parent if it has other entities besides us */
|
2008-02-25 16:34:02 +00:00
|
|
|
if (cfs_rq->load.weight)
|
2007-07-09 16:51:58 +00:00
|
|
|
break;
|
2010-03-24 15:38:48 +00:00
|
|
|
flags |= DEQUEUE_SLEEP;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
2008-01-25 20:08:29 +00:00
|
|
|
|
2008-10-17 17:27:03 +00:00
|
|
|
hrtick_update(rq);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2007-09-19 21:34:46 +00:00
|
|
|
* sched_yield() support is very simple - we dequeue and enqueue.
|
|
|
|
*
|
|
|
|
* If compat_yield is turned on then we requeue to the end of the tree.
|
2007-07-09 16:51:58 +00:00
|
|
|
*/
|
2007-10-15 15:00:08 +00:00
|
|
|
static void yield_task_fair(struct rq *rq)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2007-12-04 16:04:39 +00:00
|
|
|
struct task_struct *curr = rq->curr;
|
|
|
|
struct cfs_rq *cfs_rq = task_cfs_rq(curr);
|
|
|
|
struct sched_entity *rightmost, *se = &curr->se;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
|
|
|
/*
|
2007-09-19 21:34:46 +00:00
|
|
|
* Are we the only task in the tree?
|
|
|
|
*/
|
|
|
|
if (unlikely(cfs_rq->nr_running == 1))
|
|
|
|
return;
|
|
|
|
|
2008-11-11 10:52:33 +00:00
|
|
|
clear_buddies(cfs_rq, se);
|
|
|
|
|
2007-12-04 16:04:39 +00:00
|
|
|
if (likely(!sysctl_sched_compat_yield) && curr->policy != SCHED_BATCH) {
|
2008-05-03 16:29:28 +00:00
|
|
|
update_rq_clock(rq);
|
2007-09-19 21:34:46 +00:00
|
|
|
/*
|
2007-10-15 15:00:13 +00:00
|
|
|
* Update run-time statistics of the 'current'.
|
2007-09-19 21:34:46 +00:00
|
|
|
*/
|
2007-10-15 15:00:12 +00:00
|
|
|
update_curr(cfs_rq);
|
2007-09-19 21:34:46 +00:00
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Find the rightmost entry in the rbtree:
|
2007-07-09 16:51:58 +00:00
|
|
|
*/
|
2007-10-15 15:00:12 +00:00
|
|
|
rightmost = __pick_last_entity(cfs_rq);
|
2007-09-19 21:34:46 +00:00
|
|
|
/*
|
|
|
|
* Already in the rightmost position?
|
|
|
|
*/
|
2009-07-16 10:32:27 +00:00
|
|
|
if (unlikely(!rightmost || entity_before(rightmost, se)))
|
2007-09-19 21:34:46 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Minimally necessary key value to be last in the tree:
|
2007-10-15 15:00:12 +00:00
|
|
|
* Upon rescheduling, sched_class::put_prev_task() will place
|
|
|
|
* 'current' within the tree based on its new key value.
|
2007-09-19 21:34:46 +00:00
|
|
|
*/
|
2007-10-15 15:00:07 +00:00
|
|
|
se->vruntime = rightmost->vruntime + 1;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2008-01-25 20:08:09 +00:00
|
|
|
#ifdef CONFIG_SMP
|
2008-03-16 19:36:10 +00:00
|
|
|
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
static void task_waking_fair(struct rq *rq, struct task_struct *p)
|
|
|
|
{
|
|
|
|
struct sched_entity *se = &p->se;
|
|
|
|
struct cfs_rq *cfs_rq = cfs_rq_of(se);
|
|
|
|
|
|
|
|
se->vruntime -= cfs_rq->min_vruntime;
|
|
|
|
}
|
|
|
|
|
2008-06-27 11:41:27 +00:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2008-06-27 11:41:39 +00:00
|
|
|
/*
|
|
|
|
* effective_load() calculates the load change as seen from the root_task_group
|
|
|
|
*
|
|
|
|
* Adding load to a group doesn't make a group heavier, but can cause movement
|
|
|
|
* of group shares between cpus. Assuming the shares were perfectly aligned one
|
|
|
|
* can calculate the shift in shares.
|
|
|
|
*
|
|
|
|
* The problem is that perfectly aligning the shares is rather expensive, hence
|
|
|
|
* we try to avoid doing that too often - see update_shares(), which ratelimits
|
|
|
|
* this change.
|
|
|
|
*
|
|
|
|
* We compensate this by not only taking the current delta into account, but
|
|
|
|
* also considering the delta between when the shares were last adjusted and
|
|
|
|
* now.
|
|
|
|
*
|
|
|
|
* We still saw a performance dip, some tracing learned us that between
|
|
|
|
* cgroup:/ and cgroup:/foo balancing the number of affine wakeups increased
|
|
|
|
* significantly. Therefore try to bias the error in direction of failing
|
|
|
|
* the affine wakeup.
|
|
|
|
*
|
|
|
|
*/
|
2008-06-27 11:41:38 +00:00
|
|
|
static long effective_load(struct task_group *tg, int cpu,
|
|
|
|
long wl, long wg)
|
2008-06-27 11:41:27 +00:00
|
|
|
{
|
2008-06-27 11:41:30 +00:00
|
|
|
struct sched_entity *se = tg->se[cpu];
|
2008-06-27 11:41:38 +00:00
|
|
|
|
|
|
|
if (!tg->parent)
|
|
|
|
return wl;
|
|
|
|
|
2008-06-27 11:41:39 +00:00
|
|
|
/*
|
|
|
|
* By not taking the decrease of shares on the other cpu into
|
|
|
|
* account our error leans towards reducing the affine wakeups.
|
|
|
|
*/
|
|
|
|
if (!wl && sched_feat(ASYM_EFF_LOAD))
|
|
|
|
return wl;
|
|
|
|
|
2008-06-27 11:41:30 +00:00
|
|
|
for_each_sched_entity(se) {
|
2008-06-27 11:41:32 +00:00
|
|
|
long S, rw, s, a, b;
|
2008-09-23 13:33:42 +00:00
|
|
|
long more_w;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Instead of using this increment, also add the difference
|
|
|
|
* between when the shares were last updated and now.
|
|
|
|
*/
|
|
|
|
more_w = se->my_q->load.weight - se->my_q->rq_weight;
|
|
|
|
wl += more_w;
|
|
|
|
wg += more_w;
|
2008-06-27 11:41:30 +00:00
|
|
|
|
|
|
|
S = se->my_q->tg->shares;
|
|
|
|
s = se->my_q->shares;
|
2008-06-27 11:41:38 +00:00
|
|
|
rw = se->my_q->rq_weight;
|
2008-06-27 11:41:27 +00:00
|
|
|
|
2008-06-27 11:41:32 +00:00
|
|
|
a = S*(rw + wl);
|
|
|
|
b = S*rw + s*wg;
|
2008-06-27 11:41:30 +00:00
|
|
|
|
2008-09-23 13:33:42 +00:00
|
|
|
wl = s*(a-b);
|
|
|
|
|
|
|
|
if (likely(b))
|
|
|
|
wl /= b;
|
|
|
|
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 11:41:37 +00:00
|
|
|
/*
|
|
|
|
* Assume the group is already running and will
|
|
|
|
* thus already be accounted for in the weight.
|
|
|
|
*
|
|
|
|
* That is, moving shares between CPUs, does not
|
|
|
|
* alter the group weight.
|
|
|
|
*/
|
2008-06-27 11:41:30 +00:00
|
|
|
wg = 0;
|
|
|
|
}
|
2008-06-27 11:41:27 +00:00
|
|
|
|
2008-06-27 11:41:30 +00:00
|
|
|
return wl;
|
2008-06-27 11:41:27 +00:00
|
|
|
}
|
2008-06-27 11:41:30 +00:00
|
|
|
|
2008-06-27 11:41:27 +00:00
|
|
|
#else
|
2008-06-27 11:41:30 +00:00
|
|
|
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 11:41:37 +00:00
|
|
|
static inline unsigned long effective_load(struct task_group *tg, int cpu,
|
|
|
|
unsigned long wl, unsigned long wg)
|
2008-06-27 11:41:30 +00:00
|
|
|
{
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 11:41:37 +00:00
|
|
|
return wl;
|
2008-06-27 11:41:27 +00:00
|
|
|
}
|
2008-06-27 11:41:30 +00:00
|
|
|
|
2008-06-27 11:41:27 +00:00
|
|
|
#endif
|
|
|
|
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
|
2008-03-16 19:36:10 +00:00
|
|
|
{
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
unsigned long this_load, load;
|
|
|
|
int idx, this_cpu, prev_cpu;
|
2008-03-16 19:36:10 +00:00
|
|
|
unsigned long tl_per_task;
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
struct task_group *tg;
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 11:41:37 +00:00
|
|
|
unsigned long weight;
|
2008-05-29 09:11:41 +00:00
|
|
|
int balanced;
|
2008-03-16 19:36:10 +00:00
|
|
|
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
idx = sd->wake_idx;
|
|
|
|
this_cpu = smp_processor_id();
|
|
|
|
prev_cpu = task_cpu(p);
|
|
|
|
load = source_load(prev_cpu, idx);
|
|
|
|
this_load = target_load(this_cpu, idx);
|
2008-03-16 19:36:10 +00:00
|
|
|
|
2008-05-29 09:11:41 +00:00
|
|
|
/*
|
|
|
|
* If sync wakeup then subtract the (maximum possible)
|
|
|
|
* effect of the currently running task from the load
|
|
|
|
* of the current CPU:
|
|
|
|
*/
|
2010-06-01 13:06:13 +00:00
|
|
|
rcu_read_lock();
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 11:41:37 +00:00
|
|
|
if (sync) {
|
|
|
|
tg = task_group(current);
|
|
|
|
weight = current->se.load.weight;
|
|
|
|
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
this_load += effective_load(tg, this_cpu, -weight, -weight);
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 11:41:37 +00:00
|
|
|
load += effective_load(tg, prev_cpu, 0, -weight);
|
|
|
|
}
|
2008-05-29 09:11:41 +00:00
|
|
|
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 11:41:37 +00:00
|
|
|
tg = task_group(p);
|
|
|
|
weight = p->se.load.weight;
|
2008-05-29 09:11:41 +00:00
|
|
|
|
2009-09-07 16:28:05 +00:00
|
|
|
/*
|
|
|
|
* In low-load situations, where prev_cpu is idle and this_cpu is idle
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
* due to the sync cause above having dropped this_load to 0, we'll
|
|
|
|
* always have an imbalance, but there's really nothing you can do
|
|
|
|
* about that, so that's good too.
|
2009-09-07 16:28:05 +00:00
|
|
|
*
|
|
|
|
* Otherwise check if either cpus are near enough in load to allow this
|
|
|
|
* task to be woken on this_cpu.
|
|
|
|
*/
|
2010-05-31 10:37:30 +00:00
|
|
|
if (this_load) {
|
|
|
|
unsigned long this_eff_load, prev_eff_load;
|
|
|
|
|
|
|
|
this_eff_load = 100;
|
|
|
|
this_eff_load *= power_of(prev_cpu);
|
|
|
|
this_eff_load *= this_load +
|
|
|
|
effective_load(tg, this_cpu, weight, weight);
|
|
|
|
|
|
|
|
prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2;
|
|
|
|
prev_eff_load *= power_of(this_cpu);
|
|
|
|
prev_eff_load *= load + effective_load(tg, prev_cpu, 0, weight);
|
|
|
|
|
|
|
|
balanced = this_eff_load <= prev_eff_load;
|
|
|
|
} else
|
|
|
|
balanced = true;
|
2010-06-01 13:06:13 +00:00
|
|
|
rcu_read_unlock();
|
2008-05-29 09:11:41 +00:00
|
|
|
|
2008-03-16 19:36:10 +00:00
|
|
|
/*
|
2008-03-19 00:42:00 +00:00
|
|
|
* If the currently running task will sleep within
|
|
|
|
* a reasonable amount of time then attract this newly
|
|
|
|
* woken task:
|
2008-03-16 19:36:10 +00:00
|
|
|
*/
|
2008-10-08 07:16:04 +00:00
|
|
|
if (sync && balanced)
|
|
|
|
return 1;
|
2008-03-16 19:36:10 +00:00
|
|
|
|
2010-03-11 02:37:45 +00:00
|
|
|
schedstat_inc(p, se.statistics.nr_wakeups_affine_attempts);
|
2008-03-16 19:36:10 +00:00
|
|
|
tl_per_task = cpu_avg_load_per_task(this_cpu);
|
|
|
|
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
if (balanced ||
|
|
|
|
(this_load <= load &&
|
|
|
|
this_load + target_load(prev_cpu, idx) <= tl_per_task)) {
|
2008-03-16 19:36:10 +00:00
|
|
|
/*
|
|
|
|
* This domain has SD_WAKE_AFFINE and
|
|
|
|
* p is cache cold in this domain, and
|
|
|
|
* there is no bad imbalance.
|
|
|
|
*/
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
schedstat_inc(sd, ttwu_move_affine);
|
2010-03-11 02:37:45 +00:00
|
|
|
schedstat_inc(p, se.statistics.nr_wakeups_affine);
|
2008-03-16 19:36:10 +00:00
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-09-10 11:36:25 +00:00
|
|
|
/*
|
|
|
|
* find_idlest_group finds and returns the least busy CPU group within the
|
|
|
|
* domain.
|
|
|
|
*/
|
|
|
|
static struct sched_group *
|
2009-09-03 11:16:51 +00:00
|
|
|
find_idlest_group(struct sched_domain *sd, struct task_struct *p,
|
2009-09-16 11:46:59 +00:00
|
|
|
int this_cpu, int load_idx)
|
2008-01-25 20:08:09 +00:00
|
|
|
{
|
2009-09-10 11:36:25 +00:00
|
|
|
struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
|
|
|
|
unsigned long min_load = ULONG_MAX, this_load = 0;
|
|
|
|
int imbalance = 100 + (sd->imbalance_pct-100)/2;
|
2008-01-25 20:08:09 +00:00
|
|
|
|
2009-09-10 11:36:25 +00:00
|
|
|
do {
|
|
|
|
unsigned long load, avg_load;
|
|
|
|
int local_group;
|
|
|
|
int i;
|
2008-01-25 20:08:09 +00:00
|
|
|
|
2009-09-10 11:36:25 +00:00
|
|
|
/* Skip over this group if it has no CPUs allowed */
|
|
|
|
if (!cpumask_intersects(sched_group_cpus(group),
|
|
|
|
&p->cpus_allowed))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
local_group = cpumask_test_cpu(this_cpu,
|
|
|
|
sched_group_cpus(group));
|
|
|
|
|
|
|
|
/* Tally up the load of all CPUs in the group */
|
|
|
|
avg_load = 0;
|
|
|
|
|
|
|
|
for_each_cpu(i, sched_group_cpus(group)) {
|
|
|
|
/* Bias balancing toward cpus of our domain */
|
|
|
|
if (local_group)
|
|
|
|
load = source_load(i, load_idx);
|
|
|
|
else
|
|
|
|
load = target_load(i, load_idx);
|
|
|
|
|
|
|
|
avg_load += load;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Adjust by relative CPU power of the group */
|
|
|
|
avg_load = (avg_load * SCHED_LOAD_SCALE) / group->cpu_power;
|
|
|
|
|
|
|
|
if (local_group) {
|
|
|
|
this_load = avg_load;
|
|
|
|
this = group;
|
|
|
|
} else if (avg_load < min_load) {
|
|
|
|
min_load = avg_load;
|
|
|
|
idlest = group;
|
|
|
|
}
|
|
|
|
} while (group = group->next, group != sd->groups);
|
|
|
|
|
|
|
|
if (!idlest || 100*this_load < imbalance*min_load)
|
|
|
|
return NULL;
|
|
|
|
return idlest;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* find_idlest_cpu - find the idlest cpu among the cpus in group.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
|
|
|
|
{
|
|
|
|
unsigned long load, min_load = ULONG_MAX;
|
|
|
|
int idlest = -1;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/* Traverse only the allowed CPUs */
|
|
|
|
for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
|
|
|
|
load = weighted_cpuload(i);
|
|
|
|
|
|
|
|
if (load < min_load || (load == min_load && i == this_cpu)) {
|
|
|
|
min_load = load;
|
|
|
|
idlest = i;
|
2008-01-25 20:08:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-09-10 11:36:25 +00:00
|
|
|
return idlest;
|
|
|
|
}
|
2008-01-25 20:08:09 +00:00
|
|
|
|
2009-11-12 14:55:28 +00:00
|
|
|
/*
|
|
|
|
* Try and locate an idle CPU in the sched_domain.
|
|
|
|
*/
|
2010-03-31 23:47:45 +00:00
|
|
|
static int select_idle_sibling(struct task_struct *p, int target)
|
2009-11-12 14:55:28 +00:00
|
|
|
{
|
|
|
|
int cpu = smp_processor_id();
|
|
|
|
int prev_cpu = task_cpu(p);
|
2010-03-31 23:47:45 +00:00
|
|
|
struct sched_domain *sd;
|
2009-11-12 14:55:28 +00:00
|
|
|
int i;
|
|
|
|
|
|
|
|
/*
|
2010-03-31 23:47:45 +00:00
|
|
|
* If the task is going to be woken-up on this cpu and if it is
|
|
|
|
* already idle, then it is the right target.
|
2009-11-12 14:55:28 +00:00
|
|
|
*/
|
2010-03-31 23:47:45 +00:00
|
|
|
if (target == cpu && idle_cpu(cpu))
|
|
|
|
return cpu;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the task is going to be woken-up on the cpu where it previously
|
|
|
|
* ran and if it is currently idle, then it the right target.
|
|
|
|
*/
|
|
|
|
if (target == prev_cpu && idle_cpu(prev_cpu))
|
2009-11-12 14:55:29 +00:00
|
|
|
return prev_cpu;
|
2009-11-12 14:55:28 +00:00
|
|
|
|
|
|
|
/*
|
2010-03-31 23:47:45 +00:00
|
|
|
* Otherwise, iterate the domains and find an elegible idle cpu.
|
2009-11-12 14:55:28 +00:00
|
|
|
*/
|
2010-03-31 23:47:45 +00:00
|
|
|
for_each_domain(target, sd) {
|
|
|
|
if (!(sd->flags & SD_SHARE_PKG_RESOURCES))
|
2009-11-12 14:55:29 +00:00
|
|
|
break;
|
2010-03-31 23:47:45 +00:00
|
|
|
|
|
|
|
for_each_cpu_and(i, sched_domain_span(sd), &p->cpus_allowed) {
|
|
|
|
if (idle_cpu(i)) {
|
|
|
|
target = i;
|
|
|
|
break;
|
|
|
|
}
|
2009-11-12 14:55:28 +00:00
|
|
|
}
|
2010-03-31 23:47:45 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Lets stop looking for an idle sibling when we reached
|
|
|
|
* the domain that spans the current cpu and prev_cpu.
|
|
|
|
*/
|
|
|
|
if (cpumask_test_cpu(cpu, sched_domain_span(sd)) &&
|
|
|
|
cpumask_test_cpu(prev_cpu, sched_domain_span(sd)))
|
|
|
|
break;
|
2009-11-12 14:55:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return target;
|
|
|
|
}
|
|
|
|
|
2009-09-10 11:36:25 +00:00
|
|
|
/*
|
|
|
|
* sched_balance_self: balance the current task (running on cpu) in domains
|
|
|
|
* that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
|
|
|
|
* SD_BALANCE_EXEC.
|
|
|
|
*
|
|
|
|
* Balance, ie. select the least loaded group.
|
|
|
|
*
|
|
|
|
* Returns the target CPU number, or the same CPU if no balancing is needed.
|
|
|
|
*
|
|
|
|
* preempt must be disabled.
|
|
|
|
*/
|
2010-03-24 17:34:10 +00:00
|
|
|
static int
|
|
|
|
select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_flags)
|
2009-09-10 11:36:25 +00:00
|
|
|
{
|
2009-09-17 07:01:14 +00:00
|
|
|
struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
int cpu = smp_processor_id();
|
|
|
|
int prev_cpu = task_cpu(p);
|
|
|
|
int new_cpu = cpu;
|
2010-03-31 23:47:45 +00:00
|
|
|
int want_affine = 0;
|
2009-09-17 07:01:14 +00:00
|
|
|
int want_sd = 1;
|
2009-09-16 11:46:59 +00:00
|
|
|
int sync = wake_flags & WF_SYNC;
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
|
2009-09-14 17:37:39 +00:00
|
|
|
if (sd_flag & SD_BALANCE_WAKE) {
|
2010-03-11 16:17:20 +00:00
|
|
|
if (cpumask_test_cpu(cpu, &p->cpus_allowed))
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
want_affine = 1;
|
|
|
|
new_cpu = prev_cpu;
|
|
|
|
}
|
2009-09-10 11:36:25 +00:00
|
|
|
|
|
|
|
for_each_domain(cpu, tmp) {
|
2009-12-16 17:04:34 +00:00
|
|
|
if (!(tmp->flags & SD_LOAD_BALANCE))
|
|
|
|
continue;
|
|
|
|
|
2009-09-10 11:36:25 +00:00
|
|
|
/*
|
2009-09-10 12:40:57 +00:00
|
|
|
* If power savings logic is enabled for a domain, see if we
|
|
|
|
* are not overloaded, if so, don't balance wider.
|
2009-09-10 11:36:25 +00:00
|
|
|
*/
|
2009-09-16 06:28:30 +00:00
|
|
|
if (tmp->flags & (SD_POWERSAVINGS_BALANCE|SD_PREFER_LOCAL)) {
|
2009-09-10 12:40:57 +00:00
|
|
|
unsigned long power = 0;
|
|
|
|
unsigned long nr_running = 0;
|
|
|
|
unsigned long capacity;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_cpu(i, sched_domain_span(tmp)) {
|
|
|
|
power += power_of(i);
|
|
|
|
nr_running += cpu_rq(i)->cfs.nr_running;
|
|
|
|
}
|
|
|
|
|
|
|
|
capacity = DIV_ROUND_CLOSEST(power, SCHED_LOAD_SCALE);
|
|
|
|
|
2009-09-16 06:28:30 +00:00
|
|
|
if (tmp->flags & SD_POWERSAVINGS_BALANCE)
|
|
|
|
nr_running /= 2;
|
|
|
|
|
|
|
|
if (nr_running < capacity)
|
2009-09-17 07:01:14 +00:00
|
|
|
want_sd = 0;
|
2009-09-10 12:40:57 +00:00
|
|
|
}
|
2009-09-10 11:36:25 +00:00
|
|
|
|
2009-11-12 14:55:29 +00:00
|
|
|
/*
|
2010-03-31 23:47:45 +00:00
|
|
|
* If both cpu and prev_cpu are part of this domain,
|
|
|
|
* cpu is a valid SD_WAKE_AFFINE target.
|
2009-11-12 14:55:29 +00:00
|
|
|
*/
|
2010-03-31 23:47:45 +00:00
|
|
|
if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
|
|
|
|
cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
|
|
|
|
affine_sd = tmp;
|
|
|
|
want_affine = 0;
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
}
|
|
|
|
|
2009-09-17 07:01:14 +00:00
|
|
|
if (!want_sd && !want_affine)
|
|
|
|
break;
|
|
|
|
|
2009-09-14 17:37:39 +00:00
|
|
|
if (!(tmp->flags & sd_flag))
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
continue;
|
|
|
|
|
2009-09-17 07:01:14 +00:00
|
|
|
if (want_sd)
|
|
|
|
sd = tmp;
|
|
|
|
}
|
|
|
|
|
2010-03-11 16:17:16 +00:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2009-09-17 07:01:14 +00:00
|
|
|
if (sched_feat(LB_SHARES_UPDATE)) {
|
|
|
|
/*
|
|
|
|
* Pick the largest domain to update shares over
|
|
|
|
*/
|
|
|
|
tmp = sd;
|
2010-04-16 12:59:29 +00:00
|
|
|
if (affine_sd && (!tmp || affine_sd->span_weight > sd->span_weight))
|
2009-09-17 07:01:14 +00:00
|
|
|
tmp = affine_sd;
|
|
|
|
|
2010-03-24 17:34:10 +00:00
|
|
|
if (tmp) {
|
|
|
|
raw_spin_unlock(&rq->lock);
|
2009-09-17 07:01:14 +00:00
|
|
|
update_shares(tmp);
|
2010-03-24 17:34:10 +00:00
|
|
|
raw_spin_lock(&rq->lock);
|
|
|
|
}
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
}
|
2010-03-11 16:17:16 +00:00
|
|
|
#endif
|
2009-09-10 11:36:25 +00:00
|
|
|
|
2010-03-11 16:17:16 +00:00
|
|
|
if (affine_sd) {
|
2010-03-31 23:47:45 +00:00
|
|
|
if (cpu == prev_cpu || wake_affine(affine_sd, p, sync))
|
|
|
|
return select_idle_sibling(p, cpu);
|
|
|
|
else
|
|
|
|
return select_idle_sibling(p, prev_cpu);
|
2010-03-11 16:17:16 +00:00
|
|
|
}
|
2008-01-25 20:08:09 +00:00
|
|
|
|
2009-09-10 11:36:25 +00:00
|
|
|
while (sd) {
|
2009-09-16 11:46:59 +00:00
|
|
|
int load_idx = sd->forkexec_idx;
|
2009-09-10 11:36:25 +00:00
|
|
|
struct sched_group *group;
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
int weight;
|
2008-03-16 19:36:10 +00:00
|
|
|
|
2009-09-14 17:37:39 +00:00
|
|
|
if (!(sd->flags & sd_flag)) {
|
2009-09-10 11:36:25 +00:00
|
|
|
sd = sd->child;
|
|
|
|
continue;
|
|
|
|
}
|
2008-03-16 19:36:10 +00:00
|
|
|
|
2009-09-16 11:46:59 +00:00
|
|
|
if (sd_flag & SD_BALANCE_WAKE)
|
|
|
|
load_idx = sd->wake_idx;
|
2008-03-16 19:36:10 +00:00
|
|
|
|
2009-09-16 11:46:59 +00:00
|
|
|
group = find_idlest_group(sd, p, cpu, load_idx);
|
2009-09-10 11:36:25 +00:00
|
|
|
if (!group) {
|
|
|
|
sd = sd->child;
|
|
|
|
continue;
|
|
|
|
}
|
2008-03-19 00:42:00 +00:00
|
|
|
|
2009-09-11 10:45:38 +00:00
|
|
|
new_cpu = find_idlest_cpu(group, p, cpu);
|
2009-09-10 11:36:25 +00:00
|
|
|
if (new_cpu == -1 || new_cpu == cpu) {
|
|
|
|
/* Now try balancing at a lower domain level of cpu */
|
|
|
|
sd = sd->child;
|
|
|
|
continue;
|
2008-01-25 20:08:09 +00:00
|
|
|
}
|
2009-09-10 11:36:25 +00:00
|
|
|
|
|
|
|
/* Now try balancing at a lower domain level of new_cpu */
|
|
|
|
cpu = new_cpu;
|
2010-04-16 12:59:29 +00:00
|
|
|
weight = sd->span_weight;
|
2009-09-10 11:36:25 +00:00
|
|
|
sd = NULL;
|
|
|
|
for_each_domain(cpu, tmp) {
|
2010-04-16 12:59:29 +00:00
|
|
|
if (weight <= tmp->span_weight)
|
2009-09-10 11:36:25 +00:00
|
|
|
break;
|
2009-09-14 17:37:39 +00:00
|
|
|
if (tmp->flags & sd_flag)
|
2009-09-10 11:36:25 +00:00
|
|
|
sd = tmp;
|
|
|
|
}
|
|
|
|
/* while loop will break here if sd == NULL */
|
2008-01-25 20:08:09 +00:00
|
|
|
}
|
|
|
|
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 11:50:02 +00:00
|
|
|
return new_cpu;
|
2008-01-25 20:08:09 +00:00
|
|
|
}
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
2009-01-14 11:39:19 +00:00
|
|
|
static unsigned long
|
|
|
|
wakeup_gran(struct sched_entity *curr, struct sched_entity *se)
|
2008-04-19 17:44:57 +00:00
|
|
|
{
|
|
|
|
unsigned long gran = sysctl_sched_wakeup_granularity;
|
|
|
|
|
|
|
|
/*
|
2009-01-14 11:39:19 +00:00
|
|
|
* Since its curr running now, convert the gran from real-time
|
|
|
|
* to virtual-time in his units.
|
2010-03-11 16:17:04 +00:00
|
|
|
*
|
|
|
|
* By using 'se' instead of 'curr' we penalize light tasks, so
|
|
|
|
* they get preempted easier. That is, if 'se' < 'curr' then
|
|
|
|
* the resulting gran will be larger, therefore penalizing the
|
|
|
|
* lighter, if otoh 'se' > 'curr' then the resulting gran will
|
|
|
|
* be smaller, again penalizing the lighter task.
|
|
|
|
*
|
|
|
|
* This is especially important for buddies when the leftmost
|
|
|
|
* task is higher priority than the buddy.
|
2008-04-19 17:44:57 +00:00
|
|
|
*/
|
2010-03-11 16:17:04 +00:00
|
|
|
if (unlikely(se->load.weight != NICE_0_LOAD))
|
|
|
|
gran = calc_delta_fair(gran, se);
|
2008-04-19 17:44:57 +00:00
|
|
|
|
|
|
|
return gran;
|
|
|
|
}
|
|
|
|
|
2008-10-24 09:06:15 +00:00
|
|
|
/*
|
|
|
|
* Should 'se' preempt 'curr'.
|
|
|
|
*
|
|
|
|
* |s1
|
|
|
|
* |s2
|
|
|
|
* |s3
|
|
|
|
* g
|
|
|
|
* |<--->|c
|
|
|
|
*
|
|
|
|
* w(c, s1) = -1
|
|
|
|
* w(c, s2) = 0
|
|
|
|
* w(c, s3) = 1
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se)
|
|
|
|
{
|
|
|
|
s64 gran, vdiff = curr->vruntime - se->vruntime;
|
|
|
|
|
|
|
|
if (vdiff <= 0)
|
|
|
|
return -1;
|
|
|
|
|
2009-01-14 11:39:19 +00:00
|
|
|
gran = wakeup_gran(curr, se);
|
2008-10-24 09:06:15 +00:00
|
|
|
if (vdiff > gran)
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-11-04 20:25:10 +00:00
|
|
|
static void set_last_buddy(struct sched_entity *se)
|
|
|
|
{
|
2009-01-15 13:53:38 +00:00
|
|
|
if (likely(task_of(se)->policy != SCHED_IDLE)) {
|
|
|
|
for_each_sched_entity(se)
|
|
|
|
cfs_rq_of(se)->last = se;
|
|
|
|
}
|
2008-11-04 20:25:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void set_next_buddy(struct sched_entity *se)
|
|
|
|
{
|
2009-01-15 13:53:38 +00:00
|
|
|
if (likely(task_of(se)->policy != SCHED_IDLE)) {
|
|
|
|
for_each_sched_entity(se)
|
|
|
|
cfs_rq_of(se)->next = se;
|
|
|
|
}
|
2008-11-04 20:25:10 +00:00
|
|
|
}
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
/*
|
|
|
|
* Preempt the current task with a newly woken task if needed:
|
|
|
|
*/
|
2009-09-16 11:47:58 +00:00
|
|
|
static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
struct task_struct *curr = rq->curr;
|
2007-10-15 15:00:12 +00:00
|
|
|
struct sched_entity *se = &curr->se, *pse = &p->se;
|
2008-12-16 07:45:30 +00:00
|
|
|
struct cfs_rq *cfs_rq = task_cfs_rq(curr);
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 21:09:22 +00:00
|
|
|
int scale = cfs_rq->nr_running >= sched_nr_latency;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2009-11-28 17:51:02 +00:00
|
|
|
if (unlikely(rt_prio(p->prio)))
|
|
|
|
goto preempt;
|
2008-03-14 20:12:12 +00:00
|
|
|
|
2008-11-04 20:25:08 +00:00
|
|
|
if (unlikely(p->sched_class != &fair_sched_class))
|
|
|
|
return;
|
|
|
|
|
2008-03-19 00:42:00 +00:00
|
|
|
if (unlikely(se == pse))
|
|
|
|
return;
|
|
|
|
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 21:09:22 +00:00
|
|
|
if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK))
|
2009-09-11 10:01:17 +00:00
|
|
|
set_next_buddy(pse);
|
2008-09-23 13:33:45 +00:00
|
|
|
|
2008-08-28 09:12:49 +00:00
|
|
|
/*
|
|
|
|
* We can come here with TIF_NEED_RESCHED already set from new task
|
|
|
|
* wake up path.
|
|
|
|
*/
|
|
|
|
if (test_tsk_need_resched(curr))
|
|
|
|
return;
|
|
|
|
|
2007-10-15 15:00:18 +00:00
|
|
|
/*
|
2009-01-15 13:53:38 +00:00
|
|
|
* Batch and idle tasks do not preempt (their preemption is driven by
|
2007-10-15 15:00:18 +00:00
|
|
|
* the tick):
|
|
|
|
*/
|
2009-01-15 13:53:38 +00:00
|
|
|
if (unlikely(p->policy != SCHED_NORMAL))
|
2007-10-15 15:00:18 +00:00
|
|
|
return;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2009-01-15 13:53:38 +00:00
|
|
|
/* Idle tasks are by definition preempted by everybody. */
|
2009-11-28 17:51:02 +00:00
|
|
|
if (unlikely(curr->policy == SCHED_IDLE))
|
|
|
|
goto preempt;
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2009-09-16 10:31:31 +00:00
|
|
|
if (!sched_feat(WAKEUP_PREEMPT))
|
|
|
|
return;
|
|
|
|
|
2009-11-28 17:51:02 +00:00
|
|
|
update_curr(cfs_rq);
|
2008-10-24 09:06:15 +00:00
|
|
|
find_matching_se(&se, &pse);
|
2009-04-08 22:29:43 +00:00
|
|
|
BUG_ON(!pse);
|
2009-11-28 17:51:02 +00:00
|
|
|
if (wakeup_preempt_entity(se, pse) == 1)
|
|
|
|
goto preempt;
|
2008-10-24 09:06:15 +00:00
|
|
|
|
2009-11-28 17:51:02 +00:00
|
|
|
return;
|
2009-11-17 09:51:40 +00:00
|
|
|
|
2009-11-28 17:51:02 +00:00
|
|
|
preempt:
|
|
|
|
resched_task(curr);
|
|
|
|
/*
|
|
|
|
* Only set the backward buddy when the current task is still
|
|
|
|
* on the rq. This can happen when a wakeup gets interleaved
|
|
|
|
* with schedule on the ->pre_schedule() or idle_balance()
|
|
|
|
* point, either of which can * drop the rq lock.
|
|
|
|
*
|
|
|
|
* Also, during early boot the idle thread is in the fair class,
|
|
|
|
* for obvious reasons its a bad idea to schedule back to it.
|
|
|
|
*/
|
|
|
|
if (unlikely(!se->on_rq || curr == rq->idle))
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se))
|
|
|
|
set_last_buddy(se);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2007-08-09 09:16:48 +00:00
|
|
|
static struct task_struct *pick_next_task_fair(struct rq *rq)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2008-01-25 20:08:29 +00:00
|
|
|
struct task_struct *p;
|
2007-07-09 16:51:58 +00:00
|
|
|
struct cfs_rq *cfs_rq = &rq->cfs;
|
|
|
|
struct sched_entity *se;
|
|
|
|
|
2009-11-24 10:55:45 +00:00
|
|
|
if (!cfs_rq->nr_running)
|
2007-07-09 16:51:58 +00:00
|
|
|
return NULL;
|
|
|
|
|
|
|
|
do {
|
2007-08-09 09:16:48 +00:00
|
|
|
se = pick_next_entity(cfs_rq);
|
2008-11-04 20:25:07 +00:00
|
|
|
set_next_entity(cfs_rq, se);
|
2007-07-09 16:51:58 +00:00
|
|
|
cfs_rq = group_cfs_rq(se);
|
|
|
|
} while (cfs_rq);
|
|
|
|
|
2008-01-25 20:08:29 +00:00
|
|
|
p = task_of(se);
|
|
|
|
hrtick_start_fair(rq, p);
|
|
|
|
|
|
|
|
return p;
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Account for a descheduled task:
|
|
|
|
*/
|
2007-08-09 09:16:49 +00:00
|
|
|
static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
struct sched_entity *se = &prev->se;
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
|
|
|
|
for_each_sched_entity(se) {
|
|
|
|
cfs_rq = cfs_rq_of(se);
|
2007-08-09 09:16:48 +00:00
|
|
|
put_prev_entity(cfs_rq, se);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-10-24 16:23:51 +00:00
|
|
|
#ifdef CONFIG_SMP
|
2007-07-09 16:51:58 +00:00
|
|
|
/**************************************************
|
|
|
|
* Fair scheduling class load-balancing methods:
|
|
|
|
*/
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
/*
|
|
|
|
* pull_task - move a task from a remote runqueue to the local runqueue.
|
|
|
|
* Both runqueues must be locked.
|
|
|
|
*/
|
|
|
|
static void pull_task(struct rq *src_rq, struct task_struct *p,
|
|
|
|
struct rq *this_rq, int this_cpu)
|
|
|
|
{
|
|
|
|
deactivate_task(src_rq, p, 0);
|
|
|
|
set_task_cpu(p, this_cpu);
|
|
|
|
activate_task(this_rq, p, 0);
|
|
|
|
check_preempt_curr(this_rq, p, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
|
|
|
|
*/
|
|
|
|
static
|
|
|
|
int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle,
|
|
|
|
int *all_pinned)
|
|
|
|
{
|
|
|
|
int tsk_cache_hot = 0;
|
|
|
|
/*
|
|
|
|
* We do not migrate tasks that are:
|
|
|
|
* 1) running (obviously), or
|
|
|
|
* 2) cannot be migrated to this CPU due to cpus_allowed, or
|
|
|
|
* 3) are cache-hot on their current CPU.
|
|
|
|
*/
|
|
|
|
if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
|
2010-03-11 02:37:45 +00:00
|
|
|
schedstat_inc(p, se.statistics.nr_failed_migrations_affine);
|
2009-12-17 16:00:43 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
*all_pinned = 0;
|
|
|
|
|
|
|
|
if (task_running(rq, p)) {
|
2010-03-11 02:37:45 +00:00
|
|
|
schedstat_inc(p, se.statistics.nr_failed_migrations_running);
|
2009-12-17 16:00:43 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Aggressive migration if:
|
|
|
|
* 1) task is cache cold, or
|
|
|
|
* 2) too many balance attempts have failed.
|
|
|
|
*/
|
|
|
|
|
|
|
|
tsk_cache_hot = task_hot(p, rq->clock, sd);
|
|
|
|
if (!tsk_cache_hot ||
|
|
|
|
sd->nr_balance_failed > sd->cache_nice_tries) {
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
|
if (tsk_cache_hot) {
|
|
|
|
schedstat_inc(sd, lb_hot_gained[idle]);
|
2010-03-11 02:37:45 +00:00
|
|
|
schedstat_inc(p, se.statistics.nr_forced_migrations);
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (tsk_cache_hot) {
|
2010-03-11 02:37:45 +00:00
|
|
|
schedstat_inc(p, se.statistics.nr_failed_migrations_hot);
|
2009-12-17 16:00:43 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2009-12-17 16:45:42 +00:00
|
|
|
/*
|
|
|
|
* move_one_task tries to move exactly one task from busiest to this_rq, as
|
|
|
|
* part of active balancing operations within "domain".
|
|
|
|
* Returns 1 if successful and 0 otherwise.
|
|
|
|
*
|
|
|
|
* Called with both runqueues locked.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
struct task_struct *p, *n;
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
int pinned = 0;
|
|
|
|
|
|
|
|
for_each_leaf_cfs_rq(busiest, cfs_rq) {
|
|
|
|
list_for_each_entry_safe(p, n, &cfs_rq->tasks, se.group_node) {
|
|
|
|
|
|
|
|
if (!can_migrate_task(p, busiest, this_cpu,
|
|
|
|
sd, idle, &pinned))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
pull_task(busiest, p, this_rq, this_cpu);
|
|
|
|
/*
|
|
|
|
* Right now, this is only the second place pull_task()
|
|
|
|
* is called, so we can safely collect pull_task()
|
|
|
|
* stats here rather than inside pull_task().
|
|
|
|
*/
|
|
|
|
schedstat_inc(sd, lb_gained[idle]);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
static unsigned long
|
|
|
|
balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
|
|
|
unsigned long max_load_move, struct sched_domain *sd,
|
|
|
|
enum cpu_idle_type idle, int *all_pinned,
|
2009-12-17 16:25:20 +00:00
|
|
|
int *this_best_prio, struct cfs_rq *busiest_cfs_rq)
|
2009-12-17 16:00:43 +00:00
|
|
|
{
|
|
|
|
int loops = 0, pulled = 0, pinned = 0;
|
|
|
|
long rem_load_move = max_load_move;
|
2009-12-17 16:25:20 +00:00
|
|
|
struct task_struct *p, *n;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
if (max_load_move == 0)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
pinned = 1;
|
|
|
|
|
2009-12-17 16:25:20 +00:00
|
|
|
list_for_each_entry_safe(p, n, &busiest_cfs_rq->tasks, se.group_node) {
|
|
|
|
if (loops++ > sysctl_sched_nr_migrate)
|
|
|
|
break;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
2009-12-17 16:25:20 +00:00
|
|
|
if ((p->se.load.weight >> 1) > rem_load_move ||
|
|
|
|
!can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned))
|
|
|
|
continue;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
2009-12-17 16:25:20 +00:00
|
|
|
pull_task(busiest, p, this_rq, this_cpu);
|
|
|
|
pulled++;
|
|
|
|
rem_load_move -= p->se.load.weight;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_PREEMPT
|
2009-12-17 16:25:20 +00:00
|
|
|
/*
|
|
|
|
* NEWIDLE balancing is a source of latency, so preemptible
|
|
|
|
* kernels will stop after the first task is pulled to minimize
|
|
|
|
* the critical section.
|
|
|
|
*/
|
|
|
|
if (idle == CPU_NEWLY_IDLE)
|
|
|
|
break;
|
2009-12-17 16:00:43 +00:00
|
|
|
#endif
|
|
|
|
|
2009-12-17 16:25:20 +00:00
|
|
|
/*
|
|
|
|
* We only want to steal up to the prescribed amount of
|
|
|
|
* weighted load.
|
|
|
|
*/
|
|
|
|
if (rem_load_move <= 0)
|
|
|
|
break;
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
if (p->prio < *this_best_prio)
|
|
|
|
*this_best_prio = p->prio;
|
|
|
|
}
|
|
|
|
out:
|
|
|
|
/*
|
|
|
|
* Right now, this is one of only two places pull_task() is called,
|
|
|
|
* so we can safely collect pull_task() stats here rather than
|
|
|
|
* inside pull_task().
|
|
|
|
*/
|
|
|
|
schedstat_add(sd, lb_gained[idle], pulled);
|
|
|
|
|
|
|
|
if (all_pinned)
|
|
|
|
*all_pinned = pinned;
|
|
|
|
|
|
|
|
return max_load_move - rem_load_move;
|
|
|
|
}
|
|
|
|
|
2009-12-17 16:47:12 +00:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
|
|
static unsigned long
|
|
|
|
load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
|
|
|
unsigned long max_load_move,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle,
|
|
|
|
int *all_pinned, int *this_best_prio)
|
|
|
|
{
|
|
|
|
long rem_load_move = max_load_move;
|
|
|
|
int busiest_cpu = cpu_of(busiest);
|
|
|
|
struct task_group *tg;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
update_h_load(busiest_cpu);
|
|
|
|
|
|
|
|
list_for_each_entry_rcu(tg, &task_groups, list) {
|
|
|
|
struct cfs_rq *busiest_cfs_rq = tg->cfs_rq[busiest_cpu];
|
|
|
|
unsigned long busiest_h_load = busiest_cfs_rq->h_load;
|
|
|
|
unsigned long busiest_weight = busiest_cfs_rq->load.weight;
|
|
|
|
u64 rem_load, moved_load;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* empty group
|
|
|
|
*/
|
|
|
|
if (!busiest_cfs_rq->task_weight)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
rem_load = (u64)rem_load_move * busiest_weight;
|
|
|
|
rem_load = div_u64(rem_load, busiest_h_load + 1);
|
|
|
|
|
|
|
|
moved_load = balance_tasks(this_rq, this_cpu, busiest,
|
|
|
|
rem_load, sd, idle, all_pinned, this_best_prio,
|
|
|
|
busiest_cfs_rq);
|
|
|
|
|
|
|
|
if (!moved_load)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
moved_load *= busiest_h_load;
|
|
|
|
moved_load = div_u64(moved_load, busiest_weight + 1);
|
|
|
|
|
|
|
|
rem_load_move -= moved_load;
|
|
|
|
if (rem_load_move < 0)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
return max_load_move - rem_load_move;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static unsigned long
|
|
|
|
load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
|
|
|
unsigned long max_load_move,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle,
|
|
|
|
int *all_pinned, int *this_best_prio)
|
|
|
|
{
|
|
|
|
return balance_tasks(this_rq, this_cpu, busiest,
|
|
|
|
max_load_move, sd, idle, all_pinned,
|
|
|
|
this_best_prio, &busiest->cfs);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
/*
|
|
|
|
* move_tasks tries to move up to max_load_move weighted load from busiest to
|
|
|
|
* this_rq, as part of a balancing operation within domain "sd".
|
|
|
|
* Returns 1 if successful and 0 otherwise.
|
|
|
|
*
|
|
|
|
* Called with both runqueues locked.
|
|
|
|
*/
|
|
|
|
static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
|
|
|
unsigned long max_load_move,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle,
|
|
|
|
int *all_pinned)
|
|
|
|
{
|
2009-12-17 16:12:46 +00:00
|
|
|
unsigned long total_load_moved = 0, load_moved;
|
2009-12-17 16:00:43 +00:00
|
|
|
int this_best_prio = this_rq->curr->prio;
|
|
|
|
|
|
|
|
do {
|
2009-12-17 16:12:46 +00:00
|
|
|
load_moved = load_balance_fair(this_rq, this_cpu, busiest,
|
2009-12-17 16:00:43 +00:00
|
|
|
max_load_move - total_load_moved,
|
|
|
|
sd, idle, all_pinned, &this_best_prio);
|
2009-12-17 16:12:46 +00:00
|
|
|
|
|
|
|
total_load_moved += load_moved;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_PREEMPT
|
|
|
|
/*
|
|
|
|
* NEWIDLE balancing is a source of latency, so preemptible
|
|
|
|
* kernels will stop after the first task is pulled to minimize
|
|
|
|
* the critical section.
|
|
|
|
*/
|
|
|
|
if (idle == CPU_NEWLY_IDLE && this_rq->nr_running)
|
|
|
|
break;
|
2009-12-17 17:10:09 +00:00
|
|
|
|
|
|
|
if (raw_spin_is_contended(&this_rq->lock) ||
|
|
|
|
raw_spin_is_contended(&busiest->lock))
|
|
|
|
break;
|
2009-12-17 16:00:43 +00:00
|
|
|
#endif
|
2009-12-17 16:12:46 +00:00
|
|
|
} while (load_moved && max_load_move > total_load_moved);
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
return total_load_moved > 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/********** Helpers for find_busiest_group ************************/
|
|
|
|
/*
|
|
|
|
* sd_lb_stats - Structure to store the statistics of a sched_domain
|
|
|
|
* during load balancing.
|
|
|
|
*/
|
|
|
|
struct sd_lb_stats {
|
|
|
|
struct sched_group *busiest; /* Busiest group in this sd */
|
|
|
|
struct sched_group *this; /* Local group in this sd */
|
|
|
|
unsigned long total_load; /* Total load of all groups in sd */
|
|
|
|
unsigned long total_pwr; /* Total power of all groups in sd */
|
|
|
|
unsigned long avg_load; /* Average load across all groups in sd */
|
|
|
|
|
|
|
|
/** Statistics of this group */
|
|
|
|
unsigned long this_load;
|
|
|
|
unsigned long this_load_per_task;
|
|
|
|
unsigned long this_nr_running;
|
|
|
|
|
|
|
|
/* Statistics of the busiest group */
|
|
|
|
unsigned long max_load;
|
|
|
|
unsigned long busiest_load_per_task;
|
|
|
|
unsigned long busiest_nr_running;
|
2010-02-24 00:13:52 +00:00
|
|
|
unsigned long busiest_group_capacity;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
int group_imb; /* Is there imbalance in this sd */
|
|
|
|
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
|
|
|
|
int power_savings_balance; /* Is powersave balance needed for this sd */
|
|
|
|
struct sched_group *group_min; /* Least loaded group in sd */
|
|
|
|
struct sched_group *group_leader; /* Group which relieves group_min */
|
|
|
|
unsigned long min_load_per_task; /* load_per_task in group_min */
|
|
|
|
unsigned long leader_nr_running; /* Nr running of group_leader */
|
|
|
|
unsigned long min_nr_running; /* Nr running of group_min */
|
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* sg_lb_stats - stats of a sched_group required for load_balancing
|
|
|
|
*/
|
|
|
|
struct sg_lb_stats {
|
|
|
|
unsigned long avg_load; /*Avg load across the CPUs of the group */
|
|
|
|
unsigned long group_load; /* Total load over the CPUs of the group */
|
|
|
|
unsigned long sum_nr_running; /* Nr tasks running in the group */
|
|
|
|
unsigned long sum_weighted_load; /* Weighted load of group's tasks */
|
|
|
|
unsigned long group_capacity;
|
|
|
|
int group_imb; /* Is there an imbalance in the group ? */
|
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* group_first_cpu - Returns the first cpu in the cpumask of a sched_group.
|
|
|
|
* @group: The group whose first cpu is to be returned.
|
|
|
|
*/
|
|
|
|
static inline unsigned int group_first_cpu(struct sched_group *group)
|
|
|
|
{
|
|
|
|
return cpumask_first(sched_group_cpus(group));
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* get_sd_load_idx - Obtain the load index for a given sched domain.
|
|
|
|
* @sd: The sched_domain whose load_idx is to be obtained.
|
|
|
|
* @idle: The Idle status of the CPU for whose sd load_icx is obtained.
|
|
|
|
*/
|
|
|
|
static inline int get_sd_load_idx(struct sched_domain *sd,
|
|
|
|
enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
int load_idx;
|
|
|
|
|
|
|
|
switch (idle) {
|
|
|
|
case CPU_NOT_IDLE:
|
|
|
|
load_idx = sd->busy_idx;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case CPU_NEWLY_IDLE:
|
|
|
|
load_idx = sd->newidle_idx;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
load_idx = sd->idle_idx;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return load_idx;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
|
|
|
|
/**
|
|
|
|
* init_sd_power_savings_stats - Initialize power savings statistics for
|
|
|
|
* the given sched_domain, during load balancing.
|
|
|
|
*
|
|
|
|
* @sd: Sched domain whose power-savings statistics are to be initialized.
|
|
|
|
* @sds: Variable containing the statistics for sd.
|
|
|
|
* @idle: Idle status of the CPU at which we're performing load-balancing.
|
|
|
|
*/
|
|
|
|
static inline void init_sd_power_savings_stats(struct sched_domain *sd,
|
|
|
|
struct sd_lb_stats *sds, enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Busy processors will not participate in power savings
|
|
|
|
* balance.
|
|
|
|
*/
|
|
|
|
if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
|
|
|
|
sds->power_savings_balance = 0;
|
|
|
|
else {
|
|
|
|
sds->power_savings_balance = 1;
|
|
|
|
sds->min_nr_running = ULONG_MAX;
|
|
|
|
sds->leader_nr_running = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* update_sd_power_savings_stats - Update the power saving stats for a
|
|
|
|
* sched_domain while performing load balancing.
|
|
|
|
*
|
|
|
|
* @group: sched_group belonging to the sched_domain under consideration.
|
|
|
|
* @sds: Variable containing the statistics of the sched_domain
|
|
|
|
* @local_group: Does group contain the CPU for which we're performing
|
|
|
|
* load balancing ?
|
|
|
|
* @sgs: Variable containing the statistics of the group.
|
|
|
|
*/
|
|
|
|
static inline void update_sd_power_savings_stats(struct sched_group *group,
|
|
|
|
struct sd_lb_stats *sds, int local_group, struct sg_lb_stats *sgs)
|
|
|
|
{
|
|
|
|
|
|
|
|
if (!sds->power_savings_balance)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the local group is idle or completely loaded
|
|
|
|
* no need to do power savings balance at this domain
|
|
|
|
*/
|
|
|
|
if (local_group && (sds->this_nr_running >= sgs->group_capacity ||
|
|
|
|
!sds->this_nr_running))
|
|
|
|
sds->power_savings_balance = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If a group is already running at full capacity or idle,
|
|
|
|
* don't include that group in power savings calculations
|
|
|
|
*/
|
|
|
|
if (!sds->power_savings_balance ||
|
|
|
|
sgs->sum_nr_running >= sgs->group_capacity ||
|
|
|
|
!sgs->sum_nr_running)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Calculate the group which has the least non-idle load.
|
|
|
|
* This is the group from where we need to pick up the load
|
|
|
|
* for saving power
|
|
|
|
*/
|
|
|
|
if ((sgs->sum_nr_running < sds->min_nr_running) ||
|
|
|
|
(sgs->sum_nr_running == sds->min_nr_running &&
|
|
|
|
group_first_cpu(group) > group_first_cpu(sds->group_min))) {
|
|
|
|
sds->group_min = group;
|
|
|
|
sds->min_nr_running = sgs->sum_nr_running;
|
|
|
|
sds->min_load_per_task = sgs->sum_weighted_load /
|
|
|
|
sgs->sum_nr_running;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Calculate the group which is almost near its
|
|
|
|
* capacity but still has some space to pick up some load
|
|
|
|
* from other group and save more power
|
|
|
|
*/
|
|
|
|
if (sgs->sum_nr_running + 1 > sgs->group_capacity)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (sgs->sum_nr_running > sds->leader_nr_running ||
|
|
|
|
(sgs->sum_nr_running == sds->leader_nr_running &&
|
|
|
|
group_first_cpu(group) < group_first_cpu(sds->group_leader))) {
|
|
|
|
sds->group_leader = group;
|
|
|
|
sds->leader_nr_running = sgs->sum_nr_running;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* check_power_save_busiest_group - see if there is potential for some power-savings balance
|
|
|
|
* @sds: Variable containing the statistics of the sched_domain
|
|
|
|
* under consideration.
|
|
|
|
* @this_cpu: Cpu at which we're currently performing load-balancing.
|
|
|
|
* @imbalance: Variable to store the imbalance.
|
|
|
|
*
|
|
|
|
* Description:
|
|
|
|
* Check if we have potential to perform some power-savings balance.
|
|
|
|
* If yes, set the busiest group to be the least loaded group in the
|
|
|
|
* sched_domain, so that it's CPUs can be put to idle.
|
|
|
|
*
|
|
|
|
* Returns 1 if there is potential to perform power-savings balance.
|
|
|
|
* Else returns 0.
|
|
|
|
*/
|
|
|
|
static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
|
|
|
|
int this_cpu, unsigned long *imbalance)
|
|
|
|
{
|
|
|
|
if (!sds->power_savings_balance)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (sds->this != sds->group_leader ||
|
|
|
|
sds->group_leader == sds->group_min)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
*imbalance = sds->min_load_per_task;
|
|
|
|
sds->busiest = sds->group_min;
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
}
|
|
|
|
#else /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
|
|
|
|
static inline void init_sd_power_savings_stats(struct sched_domain *sd,
|
|
|
|
struct sd_lb_stats *sds, enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void update_sd_power_savings_stats(struct sched_group *group,
|
|
|
|
struct sd_lb_stats *sds, int local_group, struct sg_lb_stats *sgs)
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
|
|
|
|
int this_cpu, unsigned long *imbalance)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
|
|
|
|
|
|
|
|
|
|
|
|
unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu)
|
|
|
|
{
|
|
|
|
return SCHED_LOAD_SCALE;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu)
|
|
|
|
{
|
|
|
|
return default_scale_freq_power(sd, cpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)
|
|
|
|
{
|
2010-04-16 12:59:29 +00:00
|
|
|
unsigned long weight = sd->span_weight;
|
2009-12-17 16:00:43 +00:00
|
|
|
unsigned long smt_gain = sd->smt_gain;
|
|
|
|
|
|
|
|
smt_gain /= weight;
|
|
|
|
|
|
|
|
return smt_gain;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu)
|
|
|
|
{
|
|
|
|
return default_scale_smt_power(sd, cpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long scale_rt_power(int cpu)
|
|
|
|
{
|
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
u64 total, available;
|
|
|
|
|
|
|
|
sched_avg_update(rq);
|
|
|
|
|
|
|
|
total = sched_avg_period() + (rq->clock - rq->age_stamp);
|
|
|
|
available = total - rq->rt_avg;
|
|
|
|
|
|
|
|
if (unlikely((s64)total < SCHED_LOAD_SCALE))
|
|
|
|
total = SCHED_LOAD_SCALE;
|
|
|
|
|
|
|
|
total >>= SCHED_LOAD_SHIFT;
|
|
|
|
|
|
|
|
return div_u64(available, total);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void update_cpu_power(struct sched_domain *sd, int cpu)
|
|
|
|
{
|
2010-04-16 12:59:29 +00:00
|
|
|
unsigned long weight = sd->span_weight;
|
2009-12-17 16:00:43 +00:00
|
|
|
unsigned long power = SCHED_LOAD_SCALE;
|
|
|
|
struct sched_group *sdg = sd->groups;
|
|
|
|
|
|
|
|
if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
|
|
|
|
if (sched_feat(ARCH_POWER))
|
|
|
|
power *= arch_scale_smt_power(sd, cpu);
|
|
|
|
else
|
|
|
|
power *= default_scale_smt_power(sd, cpu);
|
|
|
|
|
|
|
|
power >>= SCHED_LOAD_SHIFT;
|
|
|
|
}
|
|
|
|
|
2010-06-08 04:57:02 +00:00
|
|
|
sdg->cpu_power_orig = power;
|
|
|
|
|
|
|
|
if (sched_feat(ARCH_POWER))
|
|
|
|
power *= arch_scale_freq_power(sd, cpu);
|
|
|
|
else
|
|
|
|
power *= default_scale_freq_power(sd, cpu);
|
|
|
|
|
|
|
|
power >>= SCHED_LOAD_SHIFT;
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
power *= scale_rt_power(cpu);
|
|
|
|
power >>= SCHED_LOAD_SHIFT;
|
|
|
|
|
|
|
|
if (!power)
|
|
|
|
power = 1;
|
|
|
|
|
2010-05-31 10:37:30 +00:00
|
|
|
cpu_rq(cpu)->cpu_power = power;
|
2009-12-17 16:00:43 +00:00
|
|
|
sdg->cpu_power = power;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void update_group_power(struct sched_domain *sd, int cpu)
|
|
|
|
{
|
|
|
|
struct sched_domain *child = sd->child;
|
|
|
|
struct sched_group *group, *sdg = sd->groups;
|
|
|
|
unsigned long power;
|
|
|
|
|
|
|
|
if (!child) {
|
|
|
|
update_cpu_power(sd, cpu);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
power = 0;
|
|
|
|
|
|
|
|
group = child->groups;
|
|
|
|
do {
|
|
|
|
power += group->cpu_power;
|
|
|
|
group = group->next;
|
|
|
|
} while (group != child->groups);
|
|
|
|
|
|
|
|
sdg->cpu_power = power;
|
|
|
|
}
|
|
|
|
|
2010-06-08 04:57:02 +00:00
|
|
|
/*
|
|
|
|
* Try and fix up capacity for tiny siblings, this is needed when
|
|
|
|
* things like SD_ASYM_PACKING need f_b_g to select another sibling
|
|
|
|
* which on its own isn't powerful enough.
|
|
|
|
*
|
|
|
|
* See update_sd_pick_busiest() and check_asym_packing().
|
|
|
|
*/
|
|
|
|
static inline int
|
|
|
|
fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Only siblings can have significantly less than SCHED_LOAD_SCALE
|
|
|
|
*/
|
|
|
|
if (sd->level != SD_LV_SIBLING)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If ~90% of the cpu_power is still there, we're good.
|
|
|
|
*/
|
2010-06-09 23:03:37 +00:00
|
|
|
if (group->cpu_power * 32 > group->cpu_power_orig * 29)
|
2010-06-08 04:57:02 +00:00
|
|
|
return 1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
/**
|
|
|
|
* update_sg_lb_stats - Update sched_group's statistics for load balancing.
|
|
|
|
* @sd: The sched_domain whose statistics are to be updated.
|
|
|
|
* @group: sched_group whose statistics are to be updated.
|
|
|
|
* @this_cpu: Cpu for which load balance is currently performed.
|
|
|
|
* @idle: Idle status of this_cpu
|
|
|
|
* @load_idx: Load index of sched_domain of this_cpu for load calc.
|
|
|
|
* @sd_idle: Idle status of the sched_domain containing group.
|
|
|
|
* @local_group: Does group contain this_cpu.
|
|
|
|
* @cpus: Set of cpus considered for load balancing.
|
|
|
|
* @balance: Should we balance.
|
|
|
|
* @sgs: variable to hold the statistics for this group.
|
|
|
|
*/
|
|
|
|
static inline void update_sg_lb_stats(struct sched_domain *sd,
|
|
|
|
struct sched_group *group, int this_cpu,
|
|
|
|
enum cpu_idle_type idle, int load_idx, int *sd_idle,
|
|
|
|
int local_group, const struct cpumask *cpus,
|
|
|
|
int *balance, struct sg_lb_stats *sgs)
|
|
|
|
{
|
|
|
|
unsigned long load, max_cpu_load, min_cpu_load;
|
|
|
|
int i;
|
|
|
|
unsigned int balance_cpu = -1, first_idle_cpu = 0;
|
2010-02-24 00:13:52 +00:00
|
|
|
unsigned long avg_load_per_task = 0;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
2010-01-20 20:02:44 +00:00
|
|
|
if (local_group)
|
2009-12-17 16:00:43 +00:00
|
|
|
balance_cpu = group_first_cpu(group);
|
|
|
|
|
|
|
|
/* Tally up the load of all CPUs in the group */
|
|
|
|
max_cpu_load = 0;
|
|
|
|
min_cpu_load = ~0UL;
|
|
|
|
|
|
|
|
for_each_cpu_and(i, sched_group_cpus(group), cpus) {
|
|
|
|
struct rq *rq = cpu_rq(i);
|
|
|
|
|
|
|
|
if (*sd_idle && rq->nr_running)
|
|
|
|
*sd_idle = 0;
|
|
|
|
|
|
|
|
/* Bias balancing toward cpus of our domain */
|
|
|
|
if (local_group) {
|
|
|
|
if (idle_cpu(i) && !first_idle_cpu) {
|
|
|
|
first_idle_cpu = 1;
|
|
|
|
balance_cpu = i;
|
|
|
|
}
|
|
|
|
|
|
|
|
load = target_load(i, load_idx);
|
|
|
|
} else {
|
|
|
|
load = source_load(i, load_idx);
|
|
|
|
if (load > max_cpu_load)
|
|
|
|
max_cpu_load = load;
|
|
|
|
if (min_cpu_load > load)
|
|
|
|
min_cpu_load = load;
|
|
|
|
}
|
|
|
|
|
|
|
|
sgs->group_load += load;
|
|
|
|
sgs->sum_nr_running += rq->nr_running;
|
|
|
|
sgs->sum_weighted_load += weighted_cpuload(i);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* First idle cpu or the first cpu(busiest) in this sched group
|
|
|
|
* is eligible for doing load balancing at this and above
|
|
|
|
* domains. In the newly idle case, we will allow all the cpu's
|
|
|
|
* to do the newly idle load balance.
|
|
|
|
*/
|
2010-07-09 13:15:43 +00:00
|
|
|
if (idle != CPU_NEWLY_IDLE && local_group) {
|
|
|
|
if (balance_cpu != this_cpu) {
|
|
|
|
*balance = 0;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
update_group_power(sd, this_cpu);
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Adjust by relative CPU power of the group */
|
|
|
|
sgs->avg_load = (sgs->group_load * SCHED_LOAD_SCALE) / group->cpu_power;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Consider the group unbalanced when the imbalance is larger
|
|
|
|
* than the average weight of two tasks.
|
|
|
|
*
|
|
|
|
* APZ: with cgroup the avg task weight can vary wildly and
|
|
|
|
* might not be a suitable number - should we keep a
|
|
|
|
* normalized nr_running number somewhere that negates
|
|
|
|
* the hierarchy?
|
|
|
|
*/
|
2010-02-24 00:13:52 +00:00
|
|
|
if (sgs->sum_nr_running)
|
|
|
|
avg_load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
|
|
|
|
sgs->group_imb = 1;
|
|
|
|
|
|
|
|
sgs->group_capacity =
|
|
|
|
DIV_ROUND_CLOSEST(group->cpu_power, SCHED_LOAD_SCALE);
|
2010-06-08 04:57:02 +00:00
|
|
|
if (!sgs->group_capacity)
|
|
|
|
sgs->group_capacity = fix_small_capacity(sd, group);
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
|
2010-06-08 04:57:02 +00:00
|
|
|
/**
|
|
|
|
* update_sd_pick_busiest - return 1 on busiest group
|
|
|
|
* @sd: sched_domain whose statistics are to be checked
|
|
|
|
* @sds: sched_domain statistics
|
|
|
|
* @sg: sched_group candidate to be checked for being the busiest
|
2010-06-10 02:06:21 +00:00
|
|
|
* @sgs: sched_group statistics
|
|
|
|
* @this_cpu: the current cpu
|
2010-06-08 04:57:02 +00:00
|
|
|
*
|
|
|
|
* Determine if @sg is a busier group than the previously selected
|
|
|
|
* busiest group.
|
|
|
|
*/
|
|
|
|
static bool update_sd_pick_busiest(struct sched_domain *sd,
|
|
|
|
struct sd_lb_stats *sds,
|
|
|
|
struct sched_group *sg,
|
|
|
|
struct sg_lb_stats *sgs,
|
|
|
|
int this_cpu)
|
|
|
|
{
|
|
|
|
if (sgs->avg_load <= sds->max_load)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (sgs->sum_nr_running > sgs->group_capacity)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
if (sgs->group_imb)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ASYM_PACKING needs to move all the work to the lowest
|
|
|
|
* numbered CPUs in the group, therefore mark all groups
|
|
|
|
* higher than ourself as busy.
|
|
|
|
*/
|
|
|
|
if ((sd->flags & SD_ASYM_PACKING) && sgs->sum_nr_running &&
|
|
|
|
this_cpu < group_first_cpu(sg)) {
|
|
|
|
if (!sds->busiest)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
if (group_first_cpu(sds->busiest) > group_first_cpu(sg))
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
/**
|
|
|
|
* update_sd_lb_stats - Update sched_group's statistics for load balancing.
|
|
|
|
* @sd: sched_domain whose statistics are to be updated.
|
|
|
|
* @this_cpu: Cpu for which load balance is currently performed.
|
|
|
|
* @idle: Idle status of this_cpu
|
2010-06-08 04:57:02 +00:00
|
|
|
* @sd_idle: Idle status of the sched_domain containing sg.
|
2009-12-17 16:00:43 +00:00
|
|
|
* @cpus: Set of cpus considered for load balancing.
|
|
|
|
* @balance: Should we balance.
|
|
|
|
* @sds: variable to hold the statistics for this sched_domain.
|
|
|
|
*/
|
|
|
|
static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
|
|
|
|
enum cpu_idle_type idle, int *sd_idle,
|
|
|
|
const struct cpumask *cpus, int *balance,
|
|
|
|
struct sd_lb_stats *sds)
|
|
|
|
{
|
|
|
|
struct sched_domain *child = sd->child;
|
2010-06-08 04:57:02 +00:00
|
|
|
struct sched_group *sg = sd->groups;
|
2009-12-17 16:00:43 +00:00
|
|
|
struct sg_lb_stats sgs;
|
|
|
|
int load_idx, prefer_sibling = 0;
|
|
|
|
|
|
|
|
if (child && child->flags & SD_PREFER_SIBLING)
|
|
|
|
prefer_sibling = 1;
|
|
|
|
|
|
|
|
init_sd_power_savings_stats(sd, sds, idle);
|
|
|
|
load_idx = get_sd_load_idx(sd, idle);
|
|
|
|
|
|
|
|
do {
|
|
|
|
int local_group;
|
|
|
|
|
2010-06-08 04:57:02 +00:00
|
|
|
local_group = cpumask_test_cpu(this_cpu, sched_group_cpus(sg));
|
2009-12-17 16:00:43 +00:00
|
|
|
memset(&sgs, 0, sizeof(sgs));
|
2010-06-08 04:57:02 +00:00
|
|
|
update_sg_lb_stats(sd, sg, this_cpu, idle, load_idx, sd_idle,
|
2009-12-17 16:00:43 +00:00
|
|
|
local_group, cpus, balance, &sgs);
|
|
|
|
|
2009-12-24 13:18:21 +00:00
|
|
|
if (local_group && !(*balance))
|
2009-12-17 16:00:43 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
sds->total_load += sgs.group_load;
|
2010-06-08 04:57:02 +00:00
|
|
|
sds->total_pwr += sg->cpu_power;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* In case the child domain prefers tasks go to siblings
|
2010-06-08 04:57:02 +00:00
|
|
|
* first, lower the sg capacity to one so that we'll try
|
2009-12-17 16:00:43 +00:00
|
|
|
* and move all the excess tasks away.
|
|
|
|
*/
|
|
|
|
if (prefer_sibling)
|
|
|
|
sgs.group_capacity = min(sgs.group_capacity, 1UL);
|
|
|
|
|
|
|
|
if (local_group) {
|
|
|
|
sds->this_load = sgs.avg_load;
|
2010-06-08 04:57:02 +00:00
|
|
|
sds->this = sg;
|
2009-12-17 16:00:43 +00:00
|
|
|
sds->this_nr_running = sgs.sum_nr_running;
|
|
|
|
sds->this_load_per_task = sgs.sum_weighted_load;
|
2010-06-08 04:57:02 +00:00
|
|
|
} else if (update_sd_pick_busiest(sd, sds, sg, &sgs, this_cpu)) {
|
2009-12-17 16:00:43 +00:00
|
|
|
sds->max_load = sgs.avg_load;
|
2010-06-08 04:57:02 +00:00
|
|
|
sds->busiest = sg;
|
2009-12-17 16:00:43 +00:00
|
|
|
sds->busiest_nr_running = sgs.sum_nr_running;
|
2010-02-24 00:13:52 +00:00
|
|
|
sds->busiest_group_capacity = sgs.group_capacity;
|
2009-12-17 16:00:43 +00:00
|
|
|
sds->busiest_load_per_task = sgs.sum_weighted_load;
|
|
|
|
sds->group_imb = sgs.group_imb;
|
|
|
|
}
|
|
|
|
|
2010-06-08 04:57:02 +00:00
|
|
|
update_sd_power_savings_stats(sg, sds, local_group, &sgs);
|
|
|
|
sg = sg->next;
|
|
|
|
} while (sg != sd->groups);
|
|
|
|
}
|
|
|
|
|
2010-06-29 02:02:01 +00:00
|
|
|
int __weak arch_sd_sibling_asym_packing(void)
|
2010-06-08 04:57:02 +00:00
|
|
|
{
|
|
|
|
return 0*SD_ASYM_PACKING;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* check_asym_packing - Check to see if the group is packed into the
|
|
|
|
* sched doman.
|
|
|
|
*
|
|
|
|
* This is primarily intended to used at the sibling level. Some
|
|
|
|
* cores like POWER7 prefer to use lower numbered SMT threads. In the
|
|
|
|
* case of POWER7, it can move to lower SMT modes only when higher
|
|
|
|
* threads are idle. When in lower SMT modes, the threads will
|
|
|
|
* perform better since they share less core resources. Hence when we
|
|
|
|
* have idle threads, we want them to be the higher ones.
|
|
|
|
*
|
|
|
|
* This packing function is run on idle threads. It checks to see if
|
|
|
|
* the busiest CPU in this domain (core in the P7 case) has a higher
|
|
|
|
* CPU number than the packing function is being run on. Here we are
|
|
|
|
* assuming lower CPU number will be equivalent to lower a SMT thread
|
|
|
|
* number.
|
|
|
|
*
|
2010-06-10 02:06:21 +00:00
|
|
|
* Returns 1 when packing is required and a task should be moved to
|
|
|
|
* this CPU. The amount of the imbalance is returned in *imbalance.
|
|
|
|
*
|
2010-06-08 04:57:02 +00:00
|
|
|
* @sd: The sched_domain whose packing is to be checked.
|
|
|
|
* @sds: Statistics of the sched_domain which is to be packed
|
|
|
|
* @this_cpu: The cpu at whose sched_domain we're performing load-balance.
|
|
|
|
* @imbalance: returns amount of imbalanced due to packing.
|
|
|
|
*/
|
|
|
|
static int check_asym_packing(struct sched_domain *sd,
|
|
|
|
struct sd_lb_stats *sds,
|
|
|
|
int this_cpu, unsigned long *imbalance)
|
|
|
|
{
|
|
|
|
int busiest_cpu;
|
|
|
|
|
|
|
|
if (!(sd->flags & SD_ASYM_PACKING))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (!sds->busiest)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
busiest_cpu = group_first_cpu(sds->busiest);
|
|
|
|
if (this_cpu > busiest_cpu)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
*imbalance = DIV_ROUND_CLOSEST(sds->max_load * sds->busiest->cpu_power,
|
|
|
|
SCHED_LOAD_SCALE);
|
|
|
|
return 1;
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* fix_small_imbalance - Calculate the minor imbalance that exists
|
|
|
|
* amongst the groups of a sched_domain, during
|
|
|
|
* load balancing.
|
|
|
|
* @sds: Statistics of the sched_domain whose imbalance is to be calculated.
|
|
|
|
* @this_cpu: The cpu at whose sched_domain we're performing load-balance.
|
|
|
|
* @imbalance: Variable to store the imbalance.
|
|
|
|
*/
|
|
|
|
static inline void fix_small_imbalance(struct sd_lb_stats *sds,
|
|
|
|
int this_cpu, unsigned long *imbalance)
|
|
|
|
{
|
|
|
|
unsigned long tmp, pwr_now = 0, pwr_move = 0;
|
|
|
|
unsigned int imbn = 2;
|
2010-02-24 00:13:52 +00:00
|
|
|
unsigned long scaled_busy_load_per_task;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
if (sds->this_nr_running) {
|
|
|
|
sds->this_load_per_task /= sds->this_nr_running;
|
|
|
|
if (sds->busiest_load_per_task >
|
|
|
|
sds->this_load_per_task)
|
|
|
|
imbn = 1;
|
|
|
|
} else
|
|
|
|
sds->this_load_per_task =
|
|
|
|
cpu_avg_load_per_task(this_cpu);
|
|
|
|
|
2010-02-24 00:13:52 +00:00
|
|
|
scaled_busy_load_per_task = sds->busiest_load_per_task
|
|
|
|
* SCHED_LOAD_SCALE;
|
|
|
|
scaled_busy_load_per_task /= sds->busiest->cpu_power;
|
|
|
|
|
|
|
|
if (sds->max_load - sds->this_load + scaled_busy_load_per_task >=
|
|
|
|
(scaled_busy_load_per_task * imbn)) {
|
2009-12-17 16:00:43 +00:00
|
|
|
*imbalance = sds->busiest_load_per_task;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* OK, we don't have enough imbalance to justify moving tasks,
|
|
|
|
* however we may be able to increase total CPU power used by
|
|
|
|
* moving them.
|
|
|
|
*/
|
|
|
|
|
|
|
|
pwr_now += sds->busiest->cpu_power *
|
|
|
|
min(sds->busiest_load_per_task, sds->max_load);
|
|
|
|
pwr_now += sds->this->cpu_power *
|
|
|
|
min(sds->this_load_per_task, sds->this_load);
|
|
|
|
pwr_now /= SCHED_LOAD_SCALE;
|
|
|
|
|
|
|
|
/* Amount of load we'd subtract */
|
|
|
|
tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
|
|
|
|
sds->busiest->cpu_power;
|
|
|
|
if (sds->max_load > tmp)
|
|
|
|
pwr_move += sds->busiest->cpu_power *
|
|
|
|
min(sds->busiest_load_per_task, sds->max_load - tmp);
|
|
|
|
|
|
|
|
/* Amount of load we'd add */
|
|
|
|
if (sds->max_load * sds->busiest->cpu_power <
|
|
|
|
sds->busiest_load_per_task * SCHED_LOAD_SCALE)
|
|
|
|
tmp = (sds->max_load * sds->busiest->cpu_power) /
|
|
|
|
sds->this->cpu_power;
|
|
|
|
else
|
|
|
|
tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
|
|
|
|
sds->this->cpu_power;
|
|
|
|
pwr_move += sds->this->cpu_power *
|
|
|
|
min(sds->this_load_per_task, sds->this_load + tmp);
|
|
|
|
pwr_move /= SCHED_LOAD_SCALE;
|
|
|
|
|
|
|
|
/* Move if we gain throughput */
|
|
|
|
if (pwr_move > pwr_now)
|
|
|
|
*imbalance = sds->busiest_load_per_task;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* calculate_imbalance - Calculate the amount of imbalance present within the
|
|
|
|
* groups of a given sched_domain during load balance.
|
|
|
|
* @sds: statistics of the sched_domain whose imbalance is to be calculated.
|
|
|
|
* @this_cpu: Cpu for which currently load balance is being performed.
|
|
|
|
* @imbalance: The variable to store the imbalance.
|
|
|
|
*/
|
|
|
|
static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu,
|
|
|
|
unsigned long *imbalance)
|
|
|
|
{
|
2010-02-24 00:13:52 +00:00
|
|
|
unsigned long max_pull, load_above_capacity = ~0UL;
|
|
|
|
|
|
|
|
sds->busiest_load_per_task /= sds->busiest_nr_running;
|
|
|
|
if (sds->group_imb) {
|
|
|
|
sds->busiest_load_per_task =
|
|
|
|
min(sds->busiest_load_per_task, sds->avg_load);
|
|
|
|
}
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
/*
|
|
|
|
* In the presence of smp nice balancing, certain scenarios can have
|
|
|
|
* max load less than avg load(as we skip the groups at or below
|
|
|
|
* its cpu_power, while calculating max_load..)
|
|
|
|
*/
|
|
|
|
if (sds->max_load < sds->avg_load) {
|
|
|
|
*imbalance = 0;
|
|
|
|
return fix_small_imbalance(sds, this_cpu, imbalance);
|
|
|
|
}
|
|
|
|
|
2010-02-24 00:13:52 +00:00
|
|
|
if (!sds->group_imb) {
|
|
|
|
/*
|
|
|
|
* Don't want to pull so many tasks that a group would go idle.
|
|
|
|
*/
|
|
|
|
load_above_capacity = (sds->busiest_nr_running -
|
|
|
|
sds->busiest_group_capacity);
|
|
|
|
|
|
|
|
load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_LOAD_SCALE);
|
|
|
|
|
|
|
|
load_above_capacity /= sds->busiest->cpu_power;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We're trying to get all the cpus to the average_load, so we don't
|
|
|
|
* want to push ourselves above the average load, nor do we wish to
|
|
|
|
* reduce the max loaded cpu below the average load. At the same time,
|
|
|
|
* we also don't want to reduce the group load below the group capacity
|
|
|
|
* (so that we can implement power-savings policies etc). Thus we look
|
|
|
|
* for the minimum possible imbalance.
|
|
|
|
* Be careful of negative numbers as they'll appear as very large values
|
|
|
|
* with unsigned longs.
|
|
|
|
*/
|
|
|
|
max_pull = min(sds->max_load - sds->avg_load, load_above_capacity);
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
/* How much load to actually move to equalise the imbalance */
|
|
|
|
*imbalance = min(max_pull * sds->busiest->cpu_power,
|
|
|
|
(sds->avg_load - sds->this_load) * sds->this->cpu_power)
|
|
|
|
/ SCHED_LOAD_SCALE;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* if *imbalance is less than the average load per runnable task
|
|
|
|
* there is no gaurantee that any tasks will be moved so we'll have
|
|
|
|
* a think about bumping its value to force at least one task to be
|
|
|
|
* moved
|
|
|
|
*/
|
|
|
|
if (*imbalance < sds->busiest_load_per_task)
|
|
|
|
return fix_small_imbalance(sds, this_cpu, imbalance);
|
|
|
|
|
|
|
|
}
|
|
|
|
/******* find_busiest_group() helpers end here *********************/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* find_busiest_group - Returns the busiest group within the sched_domain
|
|
|
|
* if there is an imbalance. If there isn't an imbalance, and
|
|
|
|
* the user has opted for power-savings, it returns a group whose
|
|
|
|
* CPUs can be put to idle by rebalancing those tasks elsewhere, if
|
|
|
|
* such a group exists.
|
|
|
|
*
|
|
|
|
* Also calculates the amount of weighted load which should be moved
|
|
|
|
* to restore balance.
|
|
|
|
*
|
|
|
|
* @sd: The sched_domain whose busiest group is to be returned.
|
|
|
|
* @this_cpu: The cpu for which load balancing is currently being performed.
|
|
|
|
* @imbalance: Variable which stores amount of weighted load which should
|
|
|
|
* be moved to restore balance/put a group to idle.
|
|
|
|
* @idle: The idle status of this_cpu.
|
|
|
|
* @sd_idle: The idleness of sd
|
|
|
|
* @cpus: The set of CPUs under consideration for load-balancing.
|
|
|
|
* @balance: Pointer to a variable indicating if this_cpu
|
|
|
|
* is the appropriate cpu to perform load balancing at this_level.
|
|
|
|
*
|
|
|
|
* Returns: - the busiest group if imbalance exists.
|
|
|
|
* - If no imbalance and user has opted for power-savings balance,
|
|
|
|
* return the least loaded group whose CPUs can be
|
|
|
|
* put to idle by rebalancing its tasks onto our group.
|
|
|
|
*/
|
|
|
|
static struct sched_group *
|
|
|
|
find_busiest_group(struct sched_domain *sd, int this_cpu,
|
|
|
|
unsigned long *imbalance, enum cpu_idle_type idle,
|
|
|
|
int *sd_idle, const struct cpumask *cpus, int *balance)
|
|
|
|
{
|
|
|
|
struct sd_lb_stats sds;
|
|
|
|
|
|
|
|
memset(&sds, 0, sizeof(sds));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Compute the various statistics relavent for load balancing at
|
|
|
|
* this level.
|
|
|
|
*/
|
|
|
|
update_sd_lb_stats(sd, this_cpu, idle, sd_idle, cpus,
|
|
|
|
balance, &sds);
|
|
|
|
|
|
|
|
/* Cases where imbalance does not exist from POV of this_cpu */
|
|
|
|
/* 1) this_cpu is not the appropriate cpu to perform load balancing
|
|
|
|
* at this level.
|
|
|
|
* 2) There is no busy sibling group to pull from.
|
|
|
|
* 3) This group is the busiest group.
|
|
|
|
* 4) This group is more busy than the avg busieness at this
|
|
|
|
* sched_domain.
|
|
|
|
* 5) The imbalance is within the specified limit.
|
|
|
|
*/
|
2009-12-24 13:18:21 +00:00
|
|
|
if (!(*balance))
|
2009-12-17 16:00:43 +00:00
|
|
|
goto ret;
|
|
|
|
|
2010-06-08 04:57:02 +00:00
|
|
|
if ((idle == CPU_IDLE || idle == CPU_NEWLY_IDLE) &&
|
|
|
|
check_asym_packing(sd, &sds, this_cpu, imbalance))
|
|
|
|
return sds.busiest;
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
if (!sds.busiest || sds.busiest_nr_running == 0)
|
|
|
|
goto out_balanced;
|
|
|
|
|
|
|
|
if (sds.this_load >= sds.max_load)
|
|
|
|
goto out_balanced;
|
|
|
|
|
|
|
|
sds.avg_load = (SCHED_LOAD_SCALE * sds.total_load) / sds.total_pwr;
|
|
|
|
|
|
|
|
if (sds.this_load >= sds.avg_load)
|
|
|
|
goto out_balanced;
|
|
|
|
|
|
|
|
if (100 * sds.max_load <= sd->imbalance_pct * sds.this_load)
|
|
|
|
goto out_balanced;
|
|
|
|
|
|
|
|
/* Looks like there is an imbalance. Compute it */
|
|
|
|
calculate_imbalance(&sds, this_cpu, imbalance);
|
|
|
|
return sds.busiest;
|
|
|
|
|
|
|
|
out_balanced:
|
|
|
|
/*
|
|
|
|
* There is no obvious imbalance. But check if we can do some balancing
|
|
|
|
* to save power.
|
|
|
|
*/
|
|
|
|
if (check_power_save_busiest_group(&sds, this_cpu, imbalance))
|
|
|
|
return sds.busiest;
|
|
|
|
ret:
|
|
|
|
*imbalance = 0;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* find_busiest_queue - find the busiest runqueue among the cpus in group.
|
|
|
|
*/
|
|
|
|
static struct rq *
|
2010-06-08 04:57:02 +00:00
|
|
|
find_busiest_queue(struct sched_domain *sd, struct sched_group *group,
|
|
|
|
enum cpu_idle_type idle, unsigned long imbalance,
|
|
|
|
const struct cpumask *cpus)
|
2009-12-17 16:00:43 +00:00
|
|
|
{
|
|
|
|
struct rq *busiest = NULL, *rq;
|
|
|
|
unsigned long max_load = 0;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_cpu(i, sched_group_cpus(group)) {
|
|
|
|
unsigned long power = power_of(i);
|
|
|
|
unsigned long capacity = DIV_ROUND_CLOSEST(power, SCHED_LOAD_SCALE);
|
|
|
|
unsigned long wl;
|
|
|
|
|
2010-06-08 04:57:02 +00:00
|
|
|
if (!capacity)
|
|
|
|
capacity = fix_small_capacity(sd, group);
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
if (!cpumask_test_cpu(i, cpus))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
rq = cpu_rq(i);
|
2010-02-16 15:48:56 +00:00
|
|
|
wl = weighted_cpuload(i);
|
2009-12-17 16:00:43 +00:00
|
|
|
|
2010-02-16 15:48:56 +00:00
|
|
|
/*
|
|
|
|
* When comparing with imbalance, use weighted_cpuload()
|
|
|
|
* which is not scaled with the cpu power.
|
|
|
|
*/
|
2009-12-17 16:00:43 +00:00
|
|
|
if (capacity && rq->nr_running == 1 && wl > imbalance)
|
|
|
|
continue;
|
|
|
|
|
2010-02-16 15:48:56 +00:00
|
|
|
/*
|
|
|
|
* For the load comparisons with the other cpu's, consider
|
|
|
|
* the weighted_cpuload() scaled with the cpu power, so that
|
|
|
|
* the load can be moved away from the cpu that is potentially
|
|
|
|
* running at a lower capacity.
|
|
|
|
*/
|
|
|
|
wl = (wl * SCHED_LOAD_SCALE) / power;
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
if (wl > max_load) {
|
|
|
|
max_load = wl;
|
|
|
|
busiest = rq;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return busiest;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
|
|
|
|
* so long as it is large enough.
|
|
|
|
*/
|
|
|
|
#define MAX_PINNED_INTERVAL 512
|
|
|
|
|
|
|
|
/* Working cpumask for load_balance and load_balance_newidle. */
|
|
|
|
static DEFINE_PER_CPU(cpumask_var_t, load_balance_tmpmask);
|
|
|
|
|
2010-06-08 04:57:02 +00:00
|
|
|
static int need_active_balance(struct sched_domain *sd, int sd_idle, int idle,
|
|
|
|
int busiest_cpu, int this_cpu)
|
2009-12-23 14:10:31 +00:00
|
|
|
{
|
|
|
|
if (idle == CPU_NEWLY_IDLE) {
|
2010-06-08 04:57:02 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* ASYM_PACKING needs to force migrate tasks from busy but
|
|
|
|
* higher numbered CPUs in order to pack all tasks in the
|
|
|
|
* lowest numbered CPUs.
|
|
|
|
*/
|
|
|
|
if ((sd->flags & SD_ASYM_PACKING) && busiest_cpu > this_cpu)
|
|
|
|
return 1;
|
|
|
|
|
2009-12-23 14:10:31 +00:00
|
|
|
/*
|
|
|
|
* The only task running in a non-idle cpu can be moved to this
|
|
|
|
* cpu in an attempt to completely freeup the other CPU
|
|
|
|
* package.
|
|
|
|
*
|
|
|
|
* The package power saving logic comes from
|
|
|
|
* find_busiest_group(). If there are no imbalance, then
|
|
|
|
* f_b_g() will return NULL. However when sched_mc={1,2} then
|
|
|
|
* f_b_g() will select a group from which a running task may be
|
|
|
|
* pulled to this cpu in order to make the other package idle.
|
|
|
|
* If there is no opportunity to make a package idle and if
|
|
|
|
* there are no imbalance, then f_b_g() will return NULL and no
|
|
|
|
* action will be taken in load_balance_newidle().
|
|
|
|
*
|
|
|
|
* Under normal task pull operation due to imbalance, there
|
|
|
|
* will be more than one task in the source run queue and
|
|
|
|
* move_tasks() will succeed. ld_moved will be true and this
|
|
|
|
* active balance code will not be triggered.
|
|
|
|
*/
|
|
|
|
if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
|
|
|
|
!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2);
|
|
|
|
}
|
|
|
|
|
2010-05-06 16:49:21 +00:00
|
|
|
static int active_load_balance_cpu_stop(void *data);
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
/*
|
|
|
|
* Check this_cpu to ensure it is balanced within domain. Attempt to move
|
|
|
|
* tasks if there is an imbalance.
|
|
|
|
*/
|
|
|
|
static int load_balance(int this_cpu, struct rq *this_rq,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle,
|
|
|
|
int *balance)
|
|
|
|
{
|
|
|
|
int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
|
|
|
|
struct sched_group *group;
|
|
|
|
unsigned long imbalance;
|
|
|
|
struct rq *busiest;
|
|
|
|
unsigned long flags;
|
|
|
|
struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
|
|
|
|
|
|
|
|
cpumask_copy(cpus, cpu_active_mask);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* When power savings policy is enabled for the parent domain, idle
|
|
|
|
* sibling can pick up load irrespective of busy siblings. In this case,
|
|
|
|
* let the state of idle sibling percolate up as CPU_IDLE, instead of
|
|
|
|
* portraying it as CPU_NOT_IDLE.
|
|
|
|
*/
|
|
|
|
if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
|
|
|
|
!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
|
|
|
|
sd_idle = 1;
|
|
|
|
|
|
|
|
schedstat_inc(sd, lb_count[idle]);
|
|
|
|
|
|
|
|
redo:
|
|
|
|
update_shares(sd);
|
|
|
|
group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
|
|
|
|
cpus, balance);
|
|
|
|
|
|
|
|
if (*balance == 0)
|
|
|
|
goto out_balanced;
|
|
|
|
|
|
|
|
if (!group) {
|
|
|
|
schedstat_inc(sd, lb_nobusyg[idle]);
|
|
|
|
goto out_balanced;
|
|
|
|
}
|
|
|
|
|
2010-06-08 04:57:02 +00:00
|
|
|
busiest = find_busiest_queue(sd, group, idle, imbalance, cpus);
|
2009-12-17 16:00:43 +00:00
|
|
|
if (!busiest) {
|
|
|
|
schedstat_inc(sd, lb_nobusyq[idle]);
|
|
|
|
goto out_balanced;
|
|
|
|
}
|
|
|
|
|
|
|
|
BUG_ON(busiest == this_rq);
|
|
|
|
|
|
|
|
schedstat_add(sd, lb_imbalance[idle], imbalance);
|
|
|
|
|
|
|
|
ld_moved = 0;
|
|
|
|
if (busiest->nr_running > 1) {
|
|
|
|
/*
|
|
|
|
* Attempt to move tasks. If find_busiest_group has found
|
|
|
|
* an imbalance but busiest->nr_running <= 1, the group is
|
|
|
|
* still unbalanced. ld_moved simply stays zero, so it is
|
|
|
|
* correctly treated as an imbalance.
|
|
|
|
*/
|
|
|
|
local_irq_save(flags);
|
|
|
|
double_rq_lock(this_rq, busiest);
|
|
|
|
ld_moved = move_tasks(this_rq, this_cpu, busiest,
|
|
|
|
imbalance, sd, idle, &all_pinned);
|
|
|
|
double_rq_unlock(this_rq, busiest);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* some other cpu did the load balance for us.
|
|
|
|
*/
|
|
|
|
if (ld_moved && this_cpu != smp_processor_id())
|
|
|
|
resched_cpu(this_cpu);
|
|
|
|
|
|
|
|
/* All tasks on this runqueue were pinned by CPU affinity */
|
|
|
|
if (unlikely(all_pinned)) {
|
|
|
|
cpumask_clear_cpu(cpu_of(busiest), cpus);
|
|
|
|
if (!cpumask_empty(cpus))
|
|
|
|
goto redo;
|
|
|
|
goto out_balanced;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!ld_moved) {
|
|
|
|
schedstat_inc(sd, lb_failed[idle]);
|
|
|
|
sd->nr_balance_failed++;
|
|
|
|
|
2010-06-08 04:57:02 +00:00
|
|
|
if (need_active_balance(sd, sd_idle, idle, cpu_of(busiest),
|
|
|
|
this_cpu)) {
|
2009-12-17 16:00:43 +00:00
|
|
|
raw_spin_lock_irqsave(&busiest->lock, flags);
|
|
|
|
|
2010-05-06 16:49:21 +00:00
|
|
|
/* don't kick the active_load_balance_cpu_stop,
|
|
|
|
* if the curr task on busiest cpu can't be
|
|
|
|
* moved to this_cpu
|
2009-12-17 16:00:43 +00:00
|
|
|
*/
|
|
|
|
if (!cpumask_test_cpu(this_cpu,
|
|
|
|
&busiest->curr->cpus_allowed)) {
|
|
|
|
raw_spin_unlock_irqrestore(&busiest->lock,
|
|
|
|
flags);
|
|
|
|
all_pinned = 1;
|
|
|
|
goto out_one_pinned;
|
|
|
|
}
|
|
|
|
|
2010-05-06 16:49:21 +00:00
|
|
|
/*
|
|
|
|
* ->active_balance synchronizes accesses to
|
|
|
|
* ->active_balance_work. Once set, it's cleared
|
|
|
|
* only after active load balance is finished.
|
|
|
|
*/
|
2009-12-17 16:00:43 +00:00
|
|
|
if (!busiest->active_balance) {
|
|
|
|
busiest->active_balance = 1;
|
|
|
|
busiest->push_cpu = this_cpu;
|
|
|
|
active_balance = 1;
|
|
|
|
}
|
|
|
|
raw_spin_unlock_irqrestore(&busiest->lock, flags);
|
2010-05-06 16:49:21 +00:00
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
if (active_balance)
|
2010-05-06 16:49:21 +00:00
|
|
|
stop_one_cpu_nowait(cpu_of(busiest),
|
|
|
|
active_load_balance_cpu_stop, busiest,
|
|
|
|
&busiest->active_balance_work);
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We've kicked active balancing, reset the failure
|
|
|
|
* counter.
|
|
|
|
*/
|
|
|
|
sd->nr_balance_failed = sd->cache_nice_tries+1;
|
|
|
|
}
|
|
|
|
} else
|
|
|
|
sd->nr_balance_failed = 0;
|
|
|
|
|
|
|
|
if (likely(!active_balance)) {
|
|
|
|
/* We were unbalanced, so reset the balancing interval */
|
|
|
|
sd->balance_interval = sd->min_interval;
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* If we've begun active balancing, start to back off. This
|
|
|
|
* case may not be covered by the all_pinned logic if there
|
|
|
|
* is only 1 task on the busy runqueue (because we don't call
|
|
|
|
* move_tasks).
|
|
|
|
*/
|
|
|
|
if (sd->balance_interval < sd->max_interval)
|
|
|
|
sd->balance_interval *= 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
|
|
|
|
!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
|
|
|
|
ld_moved = -1;
|
|
|
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
out_balanced:
|
|
|
|
schedstat_inc(sd, lb_balanced[idle]);
|
|
|
|
|
|
|
|
sd->nr_balance_failed = 0;
|
|
|
|
|
|
|
|
out_one_pinned:
|
|
|
|
/* tune up the balancing interval */
|
|
|
|
if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
|
|
|
|
(sd->balance_interval < sd->max_interval))
|
|
|
|
sd->balance_interval *= 2;
|
|
|
|
|
|
|
|
if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
|
|
|
|
!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
|
|
|
|
ld_moved = -1;
|
|
|
|
else
|
|
|
|
ld_moved = 0;
|
|
|
|
out:
|
|
|
|
if (ld_moved)
|
|
|
|
update_shares(sd);
|
|
|
|
return ld_moved;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* idle_balance is called by schedule() if this_cpu is about to become
|
|
|
|
* idle. Attempts to pull tasks from other CPUs.
|
|
|
|
*/
|
|
|
|
static void idle_balance(int this_cpu, struct rq *this_rq)
|
|
|
|
{
|
|
|
|
struct sched_domain *sd;
|
|
|
|
int pulled_task = 0;
|
|
|
|
unsigned long next_balance = jiffies + HZ;
|
|
|
|
|
|
|
|
this_rq->idle_stamp = this_rq->clock;
|
|
|
|
|
|
|
|
if (this_rq->avg_idle < sysctl_sched_migration_cost)
|
|
|
|
return;
|
|
|
|
|
2009-12-23 14:29:42 +00:00
|
|
|
/*
|
|
|
|
* Drop the rq->lock, but keep IRQ/preempt disabled.
|
|
|
|
*/
|
|
|
|
raw_spin_unlock(&this_rq->lock);
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
for_each_domain(this_cpu, sd) {
|
|
|
|
unsigned long interval;
|
2009-12-23 14:29:42 +00:00
|
|
|
int balance = 1;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
if (!(sd->flags & SD_LOAD_BALANCE))
|
|
|
|
continue;
|
|
|
|
|
2009-12-23 14:29:42 +00:00
|
|
|
if (sd->flags & SD_BALANCE_NEWIDLE) {
|
2009-12-17 16:00:43 +00:00
|
|
|
/* If we've pulled tasks over stop searching: */
|
2009-12-23 14:29:42 +00:00
|
|
|
pulled_task = load_balance(this_cpu, this_rq,
|
|
|
|
sd, CPU_NEWLY_IDLE, &balance);
|
|
|
|
}
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
interval = msecs_to_jiffies(sd->balance_interval);
|
|
|
|
if (time_after(next_balance, sd->last_balance + interval))
|
|
|
|
next_balance = sd->last_balance + interval;
|
|
|
|
if (pulled_task) {
|
|
|
|
this_rq->idle_stamp = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2009-12-23 14:29:42 +00:00
|
|
|
|
|
|
|
raw_spin_lock(&this_rq->lock);
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
|
|
|
|
/*
|
|
|
|
* We are going idle. next_balance may be set based on
|
|
|
|
* a busy processor. So reset next_balance.
|
|
|
|
*/
|
|
|
|
this_rq->next_balance = next_balance;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2010-05-06 16:49:21 +00:00
|
|
|
* active_load_balance_cpu_stop is run by cpu stopper. It pushes
|
|
|
|
* running tasks off the busiest CPU onto idle CPUs. It requires at
|
|
|
|
* least 1 task to be running on each physical CPU where possible, and
|
|
|
|
* avoids physical / logical imbalances.
|
2009-12-17 16:00:43 +00:00
|
|
|
*/
|
2010-05-06 16:49:21 +00:00
|
|
|
static int active_load_balance_cpu_stop(void *data)
|
2009-12-17 16:00:43 +00:00
|
|
|
{
|
2010-05-06 16:49:21 +00:00
|
|
|
struct rq *busiest_rq = data;
|
|
|
|
int busiest_cpu = cpu_of(busiest_rq);
|
2009-12-17 16:00:43 +00:00
|
|
|
int target_cpu = busiest_rq->push_cpu;
|
2010-05-06 16:49:21 +00:00
|
|
|
struct rq *target_rq = cpu_rq(target_cpu);
|
2009-12-17 16:00:43 +00:00
|
|
|
struct sched_domain *sd;
|
2010-05-06 16:49:21 +00:00
|
|
|
|
|
|
|
raw_spin_lock_irq(&busiest_rq->lock);
|
|
|
|
|
|
|
|
/* make sure the requested cpu hasn't gone down in the meantime */
|
|
|
|
if (unlikely(busiest_cpu != smp_processor_id() ||
|
|
|
|
!busiest_rq->active_balance))
|
|
|
|
goto out_unlock;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
/* Is there any task to move? */
|
|
|
|
if (busiest_rq->nr_running <= 1)
|
2010-05-06 16:49:21 +00:00
|
|
|
goto out_unlock;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* This condition is "impossible", if it occurs
|
|
|
|
* we need to fix it. Originally reported by
|
|
|
|
* Bjorn Helgaas on a 128-cpu setup.
|
|
|
|
*/
|
|
|
|
BUG_ON(busiest_rq == target_rq);
|
|
|
|
|
|
|
|
/* move a task from busiest_rq to target_rq */
|
|
|
|
double_lock_balance(busiest_rq, target_rq);
|
|
|
|
|
|
|
|
/* Search for an sd spanning us and the target CPU. */
|
|
|
|
for_each_domain(target_cpu, sd) {
|
|
|
|
if ((sd->flags & SD_LOAD_BALANCE) &&
|
|
|
|
cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (likely(sd)) {
|
|
|
|
schedstat_inc(sd, alb_count);
|
|
|
|
|
|
|
|
if (move_one_task(target_rq, target_cpu, busiest_rq,
|
|
|
|
sd, CPU_IDLE))
|
|
|
|
schedstat_inc(sd, alb_pushed);
|
|
|
|
else
|
|
|
|
schedstat_inc(sd, alb_failed);
|
|
|
|
}
|
|
|
|
double_unlock_balance(busiest_rq, target_rq);
|
2010-05-06 16:49:21 +00:00
|
|
|
out_unlock:
|
|
|
|
busiest_rq->active_balance = 0;
|
|
|
|
raw_spin_unlock_irq(&busiest_rq->lock);
|
|
|
|
return 0;
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_NO_HZ
|
2010-05-22 00:09:41 +00:00
|
|
|
|
|
|
|
static DEFINE_PER_CPU(struct call_single_data, remote_sched_softirq_cb);
|
|
|
|
|
|
|
|
static void trigger_sched_softirq(void *data)
|
|
|
|
{
|
|
|
|
raise_softirq_irqoff(SCHED_SOFTIRQ);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void init_sched_softirq_csd(struct call_single_data *csd)
|
|
|
|
{
|
|
|
|
csd->func = trigger_sched_softirq;
|
|
|
|
csd->info = NULL;
|
|
|
|
csd->flags = 0;
|
|
|
|
csd->priv = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* idle load balancing details
|
|
|
|
* - One of the idle CPUs nominates itself as idle load_balancer, while
|
|
|
|
* entering idle.
|
|
|
|
* - This idle load balancer CPU will also go into tickless mode when
|
|
|
|
* it is idle, just like all other idle CPUs
|
|
|
|
* - When one of the busy CPUs notice that there may be an idle rebalancing
|
|
|
|
* needed, they will kick the idle load balancer, which then does idle
|
|
|
|
* load balancing for all the idle CPUs.
|
|
|
|
*/
|
2009-12-17 16:00:43 +00:00
|
|
|
static struct {
|
|
|
|
atomic_t load_balancer;
|
2010-05-22 00:09:41 +00:00
|
|
|
atomic_t first_pick_cpu;
|
|
|
|
atomic_t second_pick_cpu;
|
|
|
|
cpumask_var_t idle_cpus_mask;
|
|
|
|
cpumask_var_t grp_idle_mask;
|
|
|
|
unsigned long next_balance; /* in jiffy units */
|
|
|
|
} nohz ____cacheline_aligned;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
int get_nohz_load_balancer(void)
|
|
|
|
{
|
|
|
|
return atomic_read(&nohz.load_balancer);
|
|
|
|
}
|
|
|
|
|
|
|
|
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
|
|
|
|
/**
|
|
|
|
* lowest_flag_domain - Return lowest sched_domain containing flag.
|
|
|
|
* @cpu: The cpu whose lowest level of sched domain is to
|
|
|
|
* be returned.
|
|
|
|
* @flag: The flag to check for the lowest sched_domain
|
|
|
|
* for the given cpu.
|
|
|
|
*
|
|
|
|
* Returns the lowest sched_domain of a cpu which contains the given flag.
|
|
|
|
*/
|
|
|
|
static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)
|
|
|
|
{
|
|
|
|
struct sched_domain *sd;
|
|
|
|
|
|
|
|
for_each_domain(cpu, sd)
|
|
|
|
if (sd && (sd->flags & flag))
|
|
|
|
break;
|
|
|
|
|
|
|
|
return sd;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* for_each_flag_domain - Iterates over sched_domains containing the flag.
|
|
|
|
* @cpu: The cpu whose domains we're iterating over.
|
|
|
|
* @sd: variable holding the value of the power_savings_sd
|
|
|
|
* for cpu.
|
|
|
|
* @flag: The flag to filter the sched_domains to be iterated.
|
|
|
|
*
|
|
|
|
* Iterates over all the scheduler domains for a given cpu that has the 'flag'
|
|
|
|
* set, starting from the lowest sched_domain to the highest.
|
|
|
|
*/
|
|
|
|
#define for_each_flag_domain(cpu, sd, flag) \
|
|
|
|
for (sd = lowest_flag_domain(cpu, flag); \
|
|
|
|
(sd && (sd->flags & flag)); sd = sd->parent)
|
|
|
|
|
|
|
|
/**
|
|
|
|
* is_semi_idle_group - Checks if the given sched_group is semi-idle.
|
|
|
|
* @ilb_group: group to be checked for semi-idleness
|
|
|
|
*
|
|
|
|
* Returns: 1 if the group is semi-idle. 0 otherwise.
|
|
|
|
*
|
|
|
|
* We define a sched_group to be semi idle if it has atleast one idle-CPU
|
|
|
|
* and atleast one non-idle CPU. This helper function checks if the given
|
|
|
|
* sched_group is semi-idle or not.
|
|
|
|
*/
|
|
|
|
static inline int is_semi_idle_group(struct sched_group *ilb_group)
|
|
|
|
{
|
2010-05-22 00:09:41 +00:00
|
|
|
cpumask_and(nohz.grp_idle_mask, nohz.idle_cpus_mask,
|
2009-12-17 16:00:43 +00:00
|
|
|
sched_group_cpus(ilb_group));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* A sched_group is semi-idle when it has atleast one busy cpu
|
|
|
|
* and atleast one idle cpu.
|
|
|
|
*/
|
2010-05-22 00:09:41 +00:00
|
|
|
if (cpumask_empty(nohz.grp_idle_mask))
|
2009-12-17 16:00:43 +00:00
|
|
|
return 0;
|
|
|
|
|
2010-05-22 00:09:41 +00:00
|
|
|
if (cpumask_equal(nohz.grp_idle_mask, sched_group_cpus(ilb_group)))
|
2009-12-17 16:00:43 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
/**
|
|
|
|
* find_new_ilb - Finds the optimum idle load balancer for nomination.
|
|
|
|
* @cpu: The cpu which is nominating a new idle_load_balancer.
|
|
|
|
*
|
|
|
|
* Returns: Returns the id of the idle load balancer if it exists,
|
|
|
|
* Else, returns >= nr_cpu_ids.
|
|
|
|
*
|
|
|
|
* This algorithm picks the idle load balancer such that it belongs to a
|
|
|
|
* semi-idle powersavings sched_domain. The idea is to try and avoid
|
|
|
|
* completely idle packages/cores just for the purpose of idle load balancing
|
|
|
|
* when there are other idle cpu's which are better suited for that job.
|
|
|
|
*/
|
|
|
|
static int find_new_ilb(int cpu)
|
|
|
|
{
|
|
|
|
struct sched_domain *sd;
|
|
|
|
struct sched_group *ilb_group;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Have idle load balancer selection from semi-idle packages only
|
|
|
|
* when power-aware load balancing is enabled
|
|
|
|
*/
|
|
|
|
if (!(sched_smt_power_savings || sched_mc_power_savings))
|
|
|
|
goto out_done;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Optimize for the case when we have no idle CPUs or only one
|
|
|
|
* idle CPU. Don't walk the sched_domain hierarchy in such cases
|
|
|
|
*/
|
2010-05-22 00:09:41 +00:00
|
|
|
if (cpumask_weight(nohz.idle_cpus_mask) < 2)
|
2009-12-17 16:00:43 +00:00
|
|
|
goto out_done;
|
|
|
|
|
|
|
|
for_each_flag_domain(cpu, sd, SD_POWERSAVINGS_BALANCE) {
|
|
|
|
ilb_group = sd->groups;
|
|
|
|
|
|
|
|
do {
|
|
|
|
if (is_semi_idle_group(ilb_group))
|
2010-05-22 00:09:41 +00:00
|
|
|
return cpumask_first(nohz.grp_idle_mask);
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
ilb_group = ilb_group->next;
|
|
|
|
|
|
|
|
} while (ilb_group != sd->groups);
|
|
|
|
}
|
|
|
|
|
|
|
|
out_done:
|
2010-05-22 00:09:41 +00:00
|
|
|
return nr_cpu_ids;
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
#else /* (CONFIG_SCHED_MC || CONFIG_SCHED_SMT) */
|
|
|
|
static inline int find_new_ilb(int call_cpu)
|
|
|
|
{
|
2010-05-22 00:09:41 +00:00
|
|
|
return nr_cpu_ids;
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2010-05-22 00:09:41 +00:00
|
|
|
/*
|
|
|
|
* Kick a CPU to do the nohz balancing, if it is time for it. We pick the
|
|
|
|
* nohz_load_balancer CPU (if there is one) otherwise fallback to any idle
|
|
|
|
* CPU (if there is one).
|
|
|
|
*/
|
|
|
|
static void nohz_balancer_kick(int cpu)
|
|
|
|
{
|
|
|
|
int ilb_cpu;
|
|
|
|
|
|
|
|
nohz.next_balance++;
|
|
|
|
|
|
|
|
ilb_cpu = get_nohz_load_balancer();
|
|
|
|
|
|
|
|
if (ilb_cpu >= nr_cpu_ids) {
|
|
|
|
ilb_cpu = cpumask_first(nohz.idle_cpus_mask);
|
|
|
|
if (ilb_cpu >= nr_cpu_ids)
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!cpu_rq(ilb_cpu)->nohz_balance_kick) {
|
|
|
|
struct call_single_data *cp;
|
|
|
|
|
|
|
|
cpu_rq(ilb_cpu)->nohz_balance_kick = 1;
|
|
|
|
cp = &per_cpu(remote_sched_softirq_cb, cpu);
|
|
|
|
__smp_call_function_single(ilb_cpu, cp, 0);
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
/*
|
|
|
|
* This routine will try to nominate the ilb (idle load balancing)
|
|
|
|
* owner among the cpus whose ticks are stopped. ilb owner will do the idle
|
2010-05-22 00:09:41 +00:00
|
|
|
* load balancing on behalf of all those cpus.
|
2009-12-17 16:00:43 +00:00
|
|
|
*
|
2010-05-22 00:09:41 +00:00
|
|
|
* When the ilb owner becomes busy, we will not have new ilb owner until some
|
|
|
|
* idle CPU wakes up and goes back to idle or some busy CPU tries to kick
|
|
|
|
* idle load balancing by kicking one of the idle CPUs.
|
2009-12-17 16:00:43 +00:00
|
|
|
*
|
2010-05-22 00:09:41 +00:00
|
|
|
* Ticks are stopped for the ilb owner as well, with busy CPU kicking this
|
|
|
|
* ilb owner CPU in future (when there is a need for idle load balancing on
|
|
|
|
* behalf of all idle CPUs).
|
2009-12-17 16:00:43 +00:00
|
|
|
*/
|
2010-05-22 00:09:41 +00:00
|
|
|
void select_nohz_load_balancer(int stop_tick)
|
2009-12-17 16:00:43 +00:00
|
|
|
{
|
|
|
|
int cpu = smp_processor_id();
|
|
|
|
|
|
|
|
if (stop_tick) {
|
|
|
|
if (!cpu_active(cpu)) {
|
|
|
|
if (atomic_read(&nohz.load_balancer) != cpu)
|
2010-05-22 00:09:41 +00:00
|
|
|
return;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If we are going offline and still the leader,
|
|
|
|
* give up!
|
|
|
|
*/
|
2010-05-22 00:09:41 +00:00
|
|
|
if (atomic_cmpxchg(&nohz.load_balancer, cpu,
|
|
|
|
nr_cpu_ids) != cpu)
|
2009-12-17 16:00:43 +00:00
|
|
|
BUG();
|
|
|
|
|
2010-05-22 00:09:41 +00:00
|
|
|
return;
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
|
2010-05-22 00:09:41 +00:00
|
|
|
cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
|
2009-12-17 16:00:43 +00:00
|
|
|
|
2010-05-22 00:09:41 +00:00
|
|
|
if (atomic_read(&nohz.first_pick_cpu) == cpu)
|
|
|
|
atomic_cmpxchg(&nohz.first_pick_cpu, cpu, nr_cpu_ids);
|
|
|
|
if (atomic_read(&nohz.second_pick_cpu) == cpu)
|
|
|
|
atomic_cmpxchg(&nohz.second_pick_cpu, cpu, nr_cpu_ids);
|
2009-12-17 16:00:43 +00:00
|
|
|
|
2010-05-22 00:09:41 +00:00
|
|
|
if (atomic_read(&nohz.load_balancer) >= nr_cpu_ids) {
|
2009-12-17 16:00:43 +00:00
|
|
|
int new_ilb;
|
|
|
|
|
2010-05-22 00:09:41 +00:00
|
|
|
/* make me the ilb owner */
|
|
|
|
if (atomic_cmpxchg(&nohz.load_balancer, nr_cpu_ids,
|
|
|
|
cpu) != nr_cpu_ids)
|
|
|
|
return;
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
/*
|
|
|
|
* Check to see if there is a more power-efficient
|
|
|
|
* ilb.
|
|
|
|
*/
|
|
|
|
new_ilb = find_new_ilb(cpu);
|
|
|
|
if (new_ilb < nr_cpu_ids && new_ilb != cpu) {
|
2010-05-22 00:09:41 +00:00
|
|
|
atomic_set(&nohz.load_balancer, nr_cpu_ids);
|
2009-12-17 16:00:43 +00:00
|
|
|
resched_cpu(new_ilb);
|
2010-05-22 00:09:41 +00:00
|
|
|
return;
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
2010-05-22 00:09:41 +00:00
|
|
|
return;
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
} else {
|
2010-05-22 00:09:41 +00:00
|
|
|
if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask))
|
|
|
|
return;
|
2009-12-17 16:00:43 +00:00
|
|
|
|
2010-05-22 00:09:41 +00:00
|
|
|
cpumask_clear_cpu(cpu, nohz.idle_cpus_mask);
|
2009-12-17 16:00:43 +00:00
|
|
|
|
|
|
|
if (atomic_read(&nohz.load_balancer) == cpu)
|
2010-05-22 00:09:41 +00:00
|
|
|
if (atomic_cmpxchg(&nohz.load_balancer, cpu,
|
|
|
|
nr_cpu_ids) != cpu)
|
2009-12-17 16:00:43 +00:00
|
|
|
BUG();
|
|
|
|
}
|
2010-05-22 00:09:41 +00:00
|
|
|
return;
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static DEFINE_SPINLOCK(balancing);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* It checks each scheduling domain to see if it is due to be balanced,
|
|
|
|
* and initiates a balancing operation if so.
|
|
|
|
*
|
|
|
|
* Balancing parameters are set up in arch_init_sched_domains.
|
|
|
|
*/
|
|
|
|
static void rebalance_domains(int cpu, enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
int balance = 1;
|
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
unsigned long interval;
|
|
|
|
struct sched_domain *sd;
|
|
|
|
/* Earliest time when we have to do rebalance again */
|
|
|
|
unsigned long next_balance = jiffies + 60*HZ;
|
|
|
|
int update_next_balance = 0;
|
|
|
|
int need_serialize;
|
|
|
|
|
|
|
|
for_each_domain(cpu, sd) {
|
|
|
|
if (!(sd->flags & SD_LOAD_BALANCE))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
interval = sd->balance_interval;
|
|
|
|
if (idle != CPU_IDLE)
|
|
|
|
interval *= sd->busy_factor;
|
|
|
|
|
|
|
|
/* scale ms to jiffies */
|
|
|
|
interval = msecs_to_jiffies(interval);
|
|
|
|
if (unlikely(!interval))
|
|
|
|
interval = 1;
|
|
|
|
if (interval > HZ*NR_CPUS/10)
|
|
|
|
interval = HZ*NR_CPUS/10;
|
|
|
|
|
|
|
|
need_serialize = sd->flags & SD_SERIALIZE;
|
|
|
|
|
|
|
|
if (need_serialize) {
|
|
|
|
if (!spin_trylock(&balancing))
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (time_after_eq(jiffies, sd->last_balance + interval)) {
|
|
|
|
if (load_balance(cpu, rq, sd, idle, &balance)) {
|
|
|
|
/*
|
|
|
|
* We've pulled tasks over so either we're no
|
|
|
|
* longer idle, or one of our SMT siblings is
|
|
|
|
* not idle.
|
|
|
|
*/
|
|
|
|
idle = CPU_NOT_IDLE;
|
|
|
|
}
|
|
|
|
sd->last_balance = jiffies;
|
|
|
|
}
|
|
|
|
if (need_serialize)
|
|
|
|
spin_unlock(&balancing);
|
|
|
|
out:
|
|
|
|
if (time_after(next_balance, sd->last_balance + interval)) {
|
|
|
|
next_balance = sd->last_balance + interval;
|
|
|
|
update_next_balance = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Stop the load balance at this level. There is another
|
|
|
|
* CPU in our sched group which is doing load balancing more
|
|
|
|
* actively.
|
|
|
|
*/
|
|
|
|
if (!balance)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* next_balance will be updated only when there is a need.
|
|
|
|
* When the cpu is attached to null domain for ex, it will not be
|
|
|
|
* updated.
|
|
|
|
*/
|
|
|
|
if (likely(update_next_balance))
|
|
|
|
rq->next_balance = next_balance;
|
|
|
|
}
|
|
|
|
|
2010-05-22 00:09:41 +00:00
|
|
|
#ifdef CONFIG_NO_HZ
|
2009-12-17 16:00:43 +00:00
|
|
|
/*
|
2010-05-22 00:09:41 +00:00
|
|
|
* In CONFIG_NO_HZ case, the idle balance kickee will do the
|
2009-12-17 16:00:43 +00:00
|
|
|
* rebalancing for all the cpus for whom scheduler ticks are stopped.
|
|
|
|
*/
|
2010-05-22 00:09:41 +00:00
|
|
|
static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
struct rq *this_rq = cpu_rq(this_cpu);
|
|
|
|
struct rq *rq;
|
|
|
|
int balance_cpu;
|
|
|
|
|
|
|
|
if (idle != CPU_IDLE || !this_rq->nohz_balance_kick)
|
|
|
|
return;
|
|
|
|
|
|
|
|
for_each_cpu(balance_cpu, nohz.idle_cpus_mask) {
|
|
|
|
if (balance_cpu == this_cpu)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If this cpu gets work to do, stop the load balancing
|
|
|
|
* work being done for other cpus. Next load
|
|
|
|
* balancing owner will pick it up.
|
|
|
|
*/
|
|
|
|
if (need_resched()) {
|
|
|
|
this_rq->nohz_balance_kick = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
raw_spin_lock_irq(&this_rq->lock);
|
2010-07-09 13:19:54 +00:00
|
|
|
update_rq_clock(this_rq);
|
2010-05-22 00:09:41 +00:00
|
|
|
update_cpu_load(this_rq);
|
|
|
|
raw_spin_unlock_irq(&this_rq->lock);
|
|
|
|
|
|
|
|
rebalance_domains(balance_cpu, CPU_IDLE);
|
|
|
|
|
|
|
|
rq = cpu_rq(balance_cpu);
|
|
|
|
if (time_after(this_rq->next_balance, rq->next_balance))
|
|
|
|
this_rq->next_balance = rq->next_balance;
|
|
|
|
}
|
|
|
|
nohz.next_balance = this_rq->next_balance;
|
|
|
|
this_rq->nohz_balance_kick = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Current heuristic for kicking the idle load balancer
|
|
|
|
* - first_pick_cpu is the one of the busy CPUs. It will kick
|
|
|
|
* idle load balancer when it has more than one process active. This
|
|
|
|
* eliminates the need for idle load balancing altogether when we have
|
|
|
|
* only one running process in the system (common case).
|
|
|
|
* - If there are more than one busy CPU, idle load balancer may have
|
|
|
|
* to run for active_load_balance to happen (i.e., two busy CPUs are
|
|
|
|
* SMT or core siblings and can run better if they move to different
|
|
|
|
* physical CPUs). So, second_pick_cpu is the second of the busy CPUs
|
|
|
|
* which will kick idle load balancer as soon as it has any load.
|
|
|
|
*/
|
|
|
|
static inline int nohz_kick_needed(struct rq *rq, int cpu)
|
|
|
|
{
|
|
|
|
unsigned long now = jiffies;
|
|
|
|
int ret;
|
|
|
|
int first_pick_cpu, second_pick_cpu;
|
|
|
|
|
|
|
|
if (time_before(now, nohz.next_balance))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (!rq->nr_running)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
first_pick_cpu = atomic_read(&nohz.first_pick_cpu);
|
|
|
|
second_pick_cpu = atomic_read(&nohz.second_pick_cpu);
|
|
|
|
|
|
|
|
if (first_pick_cpu < nr_cpu_ids && first_pick_cpu != cpu &&
|
|
|
|
second_pick_cpu < nr_cpu_ids && second_pick_cpu != cpu)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
ret = atomic_cmpxchg(&nohz.first_pick_cpu, nr_cpu_ids, cpu);
|
|
|
|
if (ret == nr_cpu_ids || ret == cpu) {
|
|
|
|
atomic_cmpxchg(&nohz.second_pick_cpu, cpu, nr_cpu_ids);
|
|
|
|
if (rq->nr_running > 1)
|
|
|
|
return 1;
|
|
|
|
} else {
|
|
|
|
ret = atomic_cmpxchg(&nohz.second_pick_cpu, nr_cpu_ids, cpu);
|
|
|
|
if (ret == nr_cpu_ids || ret == cpu) {
|
|
|
|
if (rq->nr_running)
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) { }
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* run_rebalance_domains is triggered when needed from the scheduler tick.
|
|
|
|
* Also triggered for nohz idle balancing (with nohz_balancing_kick set).
|
|
|
|
*/
|
2009-12-17 16:00:43 +00:00
|
|
|
static void run_rebalance_domains(struct softirq_action *h)
|
|
|
|
{
|
|
|
|
int this_cpu = smp_processor_id();
|
|
|
|
struct rq *this_rq = cpu_rq(this_cpu);
|
|
|
|
enum cpu_idle_type idle = this_rq->idle_at_tick ?
|
|
|
|
CPU_IDLE : CPU_NOT_IDLE;
|
|
|
|
|
|
|
|
rebalance_domains(this_cpu, idle);
|
|
|
|
|
|
|
|
/*
|
2010-05-22 00:09:41 +00:00
|
|
|
* If this cpu has a pending nohz_balance_kick, then do the
|
2009-12-17 16:00:43 +00:00
|
|
|
* balancing on behalf of the other idle cpus whose ticks are
|
|
|
|
* stopped.
|
|
|
|
*/
|
2010-05-22 00:09:41 +00:00
|
|
|
nohz_idle_balance(this_cpu, idle);
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline int on_null_domain(int cpu)
|
|
|
|
{
|
2010-02-28 16:32:18 +00:00
|
|
|
return !rcu_dereference_sched(cpu_rq(cpu)->sd);
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
|
|
|
|
*/
|
|
|
|
static inline void trigger_load_balance(struct rq *rq, int cpu)
|
|
|
|
{
|
|
|
|
/* Don't need to rebalance while attached to NULL domain */
|
|
|
|
if (time_after_eq(jiffies, rq->next_balance) &&
|
|
|
|
likely(!on_null_domain(cpu)))
|
|
|
|
raise_softirq(SCHED_SOFTIRQ);
|
2010-05-22 00:09:41 +00:00
|
|
|
#ifdef CONFIG_NO_HZ
|
|
|
|
else if (nohz_kick_needed(rq, cpu) && likely(!on_null_domain(cpu)))
|
|
|
|
nohz_balancer_kick(cpu);
|
|
|
|
#endif
|
2009-12-17 16:00:43 +00:00
|
|
|
}
|
|
|
|
|
2009-11-30 11:16:46 +00:00
|
|
|
static void rq_online_fair(struct rq *rq)
|
|
|
|
{
|
|
|
|
update_sysctl();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void rq_offline_fair(struct rq *rq)
|
|
|
|
{
|
|
|
|
update_sysctl();
|
|
|
|
}
|
|
|
|
|
2009-12-17 16:00:43 +00:00
|
|
|
#else /* CONFIG_SMP */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* on UP we do not need to balance between CPUs:
|
|
|
|
*/
|
|
|
|
static inline void idle_balance(int cpu, struct rq *rq)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2008-06-24 18:09:43 +00:00
|
|
|
#endif /* CONFIG_SMP */
|
2007-10-24 16:23:51 +00:00
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
/*
|
|
|
|
* scheduler tick hitting a task of our scheduling class:
|
|
|
|
*/
|
2008-01-25 20:08:29 +00:00
|
|
|
static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
struct sched_entity *se = &curr->se;
|
|
|
|
|
|
|
|
for_each_sched_entity(se) {
|
|
|
|
cfs_rq = cfs_rq_of(se);
|
2008-01-25 20:08:29 +00:00
|
|
|
entity_tick(cfs_rq, se, queued);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-11-27 16:32:46 +00:00
|
|
|
* called on fork with the child task as argument from the parent's context
|
|
|
|
* - child not yet on the tasklist
|
|
|
|
* - preemption disabled
|
2007-07-09 16:51:58 +00:00
|
|
|
*/
|
2009-11-27 16:32:46 +00:00
|
|
|
static void task_fork_fair(struct task_struct *p)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
2009-11-27 16:32:46 +00:00
|
|
|
struct cfs_rq *cfs_rq = task_cfs_rq(current);
|
2007-10-15 15:00:03 +00:00
|
|
|
struct sched_entity *se = &p->se, *curr = cfs_rq->curr;
|
2007-10-15 15:00:14 +00:00
|
|
|
int this_cpu = smp_processor_id();
|
2009-11-27 16:32:46 +00:00
|
|
|
struct rq *rq = this_rq();
|
|
|
|
unsigned long flags;
|
|
|
|
|
2009-11-17 13:28:38 +00:00
|
|
|
raw_spin_lock_irqsave(&rq->lock, flags);
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2009-11-27 16:32:46 +00:00
|
|
|
if (unlikely(task_cpu(p) != this_cpu))
|
|
|
|
__set_task_cpu(p, this_cpu);
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2007-08-28 10:53:24 +00:00
|
|
|
update_curr(cfs_rq);
|
2009-11-27 16:32:46 +00:00
|
|
|
|
sched: Ensure that a child can't gain time over it's parent after fork()
A fork/exec load is usually "pass the baton", so the child
should never be placed behind the parent. With START_DEBIT we
make room for the new task, but with child_runs_first, that
room comes out of the _parent's_ hide. There's nothing to say
that the parent wasn't ahead of min_vruntime at fork() time,
which means that the "baton carrier", who is essentially the
parent in drag, can gain time and increase scheduling latencies
for waiters.
With NEW_FAIR_SLEEPERS + START_DEBIT + child_runs_first
enabled, we essentially pass the sleeper fairness off to the
child, which is fine, but if we don't base placement on the
parent's updated vruntime, we can end up compounding latency
woes if the child itself then does fork/exec. The debit
incurred at fork doesn't hurt the parent who is then going to
sleep and maybe exit, but the child who acquires the error
harms all comers.
This improves latencies of make -j<n> kernel build workloads.
Reported-by: Jens Axboe <jens.axboe@oracle.com>
Signed-off-by: Mike Galbraith <efault@gmx.de>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-08 09:12:28 +00:00
|
|
|
if (curr)
|
|
|
|
se->vruntime = curr->vruntime;
|
2007-10-15 15:00:05 +00:00
|
|
|
place_entity(cfs_rq, se, 1);
|
2007-10-15 15:00:04 +00:00
|
|
|
|
2009-11-27 16:32:46 +00:00
|
|
|
if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) {
|
2007-10-15 15:00:08 +00:00
|
|
|
/*
|
2007-10-15 15:00:08 +00:00
|
|
|
* Upon rescheduling, sched_class::put_prev_task() will place
|
|
|
|
* 'current' within the tree based on its new key value.
|
|
|
|
*/
|
2007-10-15 15:00:04 +00:00
|
|
|
swap(curr->vruntime, se->vruntime);
|
2008-08-28 09:12:49 +00:00
|
|
|
resched_task(rq->curr);
|
2007-10-15 15:00:04 +00:00
|
|
|
}
|
2007-07-09 16:51:58 +00:00
|
|
|
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
se->vruntime -= cfs_rq->min_vruntime;
|
|
|
|
|
2009-11-17 13:28:38 +00:00
|
|
|
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
|
2008-01-25 20:08:22 +00:00
|
|
|
/*
|
|
|
|
* Priority of the task has changed. Check to see if we preempt
|
|
|
|
* the current task.
|
|
|
|
*/
|
|
|
|
static void prio_changed_fair(struct rq *rq, struct task_struct *p,
|
|
|
|
int oldprio, int running)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Reschedule if we are currently running on this runqueue and
|
|
|
|
* our priority decreased, or if we are not currently running on
|
|
|
|
* this runqueue and our priority is higher than the current's
|
|
|
|
*/
|
|
|
|
if (running) {
|
|
|
|
if (p->prio > oldprio)
|
|
|
|
resched_task(rq->curr);
|
|
|
|
} else
|
2008-09-20 21:38:02 +00:00
|
|
|
check_preempt_curr(rq, p, 0);
|
2008-01-25 20:08:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We switched to the sched_fair class.
|
|
|
|
*/
|
|
|
|
static void switched_to_fair(struct rq *rq, struct task_struct *p,
|
|
|
|
int running)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We were most likely switched from sched_rt, so
|
|
|
|
* kick off the schedule if running, otherwise just see
|
|
|
|
* if we can still preempt the current task.
|
|
|
|
*/
|
|
|
|
if (running)
|
|
|
|
resched_task(rq->curr);
|
|
|
|
else
|
2008-09-20 21:38:02 +00:00
|
|
|
check_preempt_curr(rq, p, 0);
|
2008-01-25 20:08:22 +00:00
|
|
|
}
|
|
|
|
|
2007-10-15 15:00:08 +00:00
|
|
|
/* Account for a task changing its policy or group.
|
|
|
|
*
|
|
|
|
* This routine is mostly called to set cfs_rq->curr field when a task
|
|
|
|
* migrates between groups/classes.
|
|
|
|
*/
|
|
|
|
static void set_curr_task_fair(struct rq *rq)
|
|
|
|
{
|
|
|
|
struct sched_entity *se = &rq->curr->se;
|
|
|
|
|
|
|
|
for_each_sched_entity(se)
|
|
|
|
set_next_entity(cfs_rq_of(se), se);
|
|
|
|
}
|
|
|
|
|
2008-02-29 20:21:01 +00:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
static void moved_group_fair(struct task_struct *p, int on_rq)
|
2008-02-29 20:21:01 +00:00
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq = task_cfs_rq(p);
|
|
|
|
|
|
|
|
update_curr(cfs_rq);
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
if (!on_rq)
|
|
|
|
place_entity(cfs_rq, &p->se, 1);
|
2008-02-29 20:21:01 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2010-01-14 03:21:52 +00:00
|
|
|
static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task)
|
2009-09-21 01:31:53 +00:00
|
|
|
{
|
|
|
|
struct sched_entity *se = &task->se;
|
|
|
|
unsigned int rr_interval = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Time slice is 0 for SCHED_OTHER tasks that are on an otherwise
|
|
|
|
* idle runqueue:
|
|
|
|
*/
|
|
|
|
if (rq->cfs.load.weight)
|
|
|
|
rr_interval = NS_TO_JIFFIES(sched_slice(&rq->cfs, se));
|
|
|
|
|
|
|
|
return rr_interval;
|
|
|
|
}
|
|
|
|
|
2007-07-09 16:51:58 +00:00
|
|
|
/*
|
|
|
|
* All the scheduling class methods:
|
|
|
|
*/
|
2007-10-15 15:00:12 +00:00
|
|
|
static const struct sched_class fair_sched_class = {
|
|
|
|
.next = &idle_sched_class,
|
2007-07-09 16:51:58 +00:00
|
|
|
.enqueue_task = enqueue_task_fair,
|
|
|
|
.dequeue_task = dequeue_task_fair,
|
|
|
|
.yield_task = yield_task_fair,
|
|
|
|
|
2007-10-15 15:00:05 +00:00
|
|
|
.check_preempt_curr = check_preempt_wakeup,
|
2007-07-09 16:51:58 +00:00
|
|
|
|
|
|
|
.pick_next_task = pick_next_task_fair,
|
|
|
|
.put_prev_task = put_prev_task_fair,
|
|
|
|
|
2007-10-24 16:23:51 +00:00
|
|
|
#ifdef CONFIG_SMP
|
2008-10-22 07:25:26 +00:00
|
|
|
.select_task_rq = select_task_rq_fair,
|
|
|
|
|
2009-11-30 11:16:46 +00:00
|
|
|
.rq_online = rq_online_fair,
|
|
|
|
.rq_offline = rq_offline_fair,
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 17:04:41 +00:00
|
|
|
|
|
|
|
.task_waking = task_waking_fair,
|
2007-10-24 16:23:51 +00:00
|
|
|
#endif
|
2007-07-09 16:51:58 +00:00
|
|
|
|
2007-10-15 15:00:08 +00:00
|
|
|
.set_curr_task = set_curr_task_fair,
|
2007-07-09 16:51:58 +00:00
|
|
|
.task_tick = task_tick_fair,
|
2009-11-27 16:32:46 +00:00
|
|
|
.task_fork = task_fork_fair,
|
2008-01-25 20:08:22 +00:00
|
|
|
|
|
|
|
.prio_changed = prio_changed_fair,
|
|
|
|
.switched_to = switched_to_fair,
|
2008-02-29 20:21:01 +00:00
|
|
|
|
2009-09-21 01:31:53 +00:00
|
|
|
.get_rr_interval = get_rr_interval_fair,
|
|
|
|
|
2008-02-29 20:21:01 +00:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
|
|
.moved_group = moved_group_fair,
|
|
|
|
#endif
|
2007-07-09 16:51:58 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
2007-08-09 09:16:47 +00:00
|
|
|
static void print_cfs_stats(struct seq_file *m, int cpu)
|
2007-07-09 16:51:58 +00:00
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
|
2008-01-25 20:08:34 +00:00
|
|
|
rcu_read_lock();
|
2007-08-09 09:16:51 +00:00
|
|
|
for_each_leaf_cfs_rq(cpu_rq(cpu), cfs_rq)
|
2007-08-09 09:16:47 +00:00
|
|
|
print_cfs_rq(m, cpu, cfs_rq);
|
2008-01-25 20:08:34 +00:00
|
|
|
rcu_read_unlock();
|
2007-07-09 16:51:58 +00:00
|
|
|
}
|
|
|
|
#endif
|