2019-06-01 08:08:55 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2008-05-12 19:21:01 +00:00
|
|
|
/*
|
2011-11-15 16:14:39 +00:00
|
|
|
* kernel/sched/cpupri.c
|
2008-05-12 19:21:01 +00:00
|
|
|
*
|
|
|
|
* CPU priority management
|
|
|
|
*
|
|
|
|
* Copyright (C) 2007-2008 Novell
|
|
|
|
*
|
|
|
|
* Author: Gregory Haskins <ghaskins@novell.com>
|
|
|
|
*
|
|
|
|
* This code tracks the priority of each CPU so that global migration
|
|
|
|
* decisions are easy to calculate. Each CPU can be in a state as follows:
|
|
|
|
*
|
2020-10-14 19:39:04 +00:00
|
|
|
* (INVALID), NORMAL, RT1, ... RT99, HIGHER
|
2008-05-12 19:21:01 +00:00
|
|
|
*
|
|
|
|
* going from the lowest priority to the highest. CPUs in the INVALID state
|
|
|
|
* are not eligible for routing. The system maintains this state with
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 13:01:12 +00:00
|
|
|
* a 2 dimensional bitmap (the first for priority class, the second for CPUs
|
2008-05-12 19:21:01 +00:00
|
|
|
* in that class). Therefore a typical application without affinity
|
|
|
|
* restrictions can find a suitable CPU with O(1) complexity (e.g. two bit
|
|
|
|
* searches). For tasks with affinity restrictions, the algorithm has a
|
2020-10-14 19:39:04 +00:00
|
|
|
* worst case complexity of O(min(101, nr_domcpus)), though the scenario that
|
2008-05-12 19:21:01 +00:00
|
|
|
* yields the worst case search is fairly contrived.
|
|
|
|
*/
|
|
|
|
|
2020-10-14 19:06:49 +00:00
|
|
|
/*
|
|
|
|
* p->rt_priority p->prio newpri cpupri
|
|
|
|
*
|
|
|
|
* -1 -1 (CPUPRI_INVALID)
|
|
|
|
*
|
|
|
|
* 99 0 (CPUPRI_NORMAL)
|
|
|
|
*
|
|
|
|
* 1 98 98 1
|
|
|
|
* ...
|
|
|
|
* 49 50 50 49
|
|
|
|
* 50 49 49 50
|
|
|
|
* ...
|
|
|
|
* 99 0 0 99
|
2020-10-14 19:39:04 +00:00
|
|
|
*
|
|
|
|
* 100 100 (CPUPRI_HIGHER)
|
2020-10-14 19:06:49 +00:00
|
|
|
*/
|
2008-05-12 19:21:01 +00:00
|
|
|
static int convert_prio(int prio)
|
|
|
|
{
|
|
|
|
int cpupri;
|
|
|
|
|
2020-10-14 19:06:49 +00:00
|
|
|
switch (prio) {
|
|
|
|
case CPUPRI_INVALID:
|
|
|
|
cpupri = CPUPRI_INVALID; /* -1 */
|
|
|
|
break;
|
|
|
|
|
|
|
|
case 0 ... 98:
|
|
|
|
cpupri = MAX_RT_PRIO-1 - prio; /* 1 ... 99 */
|
|
|
|
break;
|
|
|
|
|
|
|
|
case MAX_RT_PRIO-1:
|
|
|
|
cpupri = CPUPRI_NORMAL; /* 0 */
|
|
|
|
break;
|
2020-10-14 19:39:04 +00:00
|
|
|
|
|
|
|
case MAX_RT_PRIO:
|
|
|
|
cpupri = CPUPRI_HIGHER; /* 100 */
|
|
|
|
break;
|
2020-10-14 19:06:49 +00:00
|
|
|
}
|
2008-05-12 19:21:01 +00:00
|
|
|
|
|
|
|
return cpupri;
|
|
|
|
}
|
|
|
|
|
2020-03-02 13:27:16 +00:00
|
|
|
static inline int __cpupri_find(struct cpupri *cp, struct task_struct *p,
|
|
|
|
struct cpumask *lowest_mask, int idx)
|
|
|
|
{
|
|
|
|
struct cpupri_vec *vec = &cp->pri_to_cpu[idx];
|
|
|
|
int skip = 0;
|
|
|
|
|
|
|
|
if (!atomic_read(&(vec)->count))
|
|
|
|
skip = 1;
|
|
|
|
/*
|
|
|
|
* When looking at the vector, we need to read the counter,
|
|
|
|
* do a memory barrier, then read the mask.
|
|
|
|
*
|
2021-03-18 12:38:50 +00:00
|
|
|
* Note: This is still all racy, but we can deal with it.
|
2020-03-02 13:27:16 +00:00
|
|
|
* Ideally, we only want to look at masks that are set.
|
|
|
|
*
|
|
|
|
* If a mask is not set, then the only thing wrong is that we
|
|
|
|
* did a little more work than necessary.
|
|
|
|
*
|
|
|
|
* If we read a zero count but the mask is set, because of the
|
|
|
|
* memory barriers, that can only happen when the highest prio
|
|
|
|
* task for a run queue has left the run queue, in which case,
|
|
|
|
* it will be followed by a pull. If the task we are processing
|
|
|
|
* fails to find a proper place to go, that pull request will
|
|
|
|
* pull this task if the run queue is running at a lower
|
|
|
|
* priority.
|
|
|
|
*/
|
|
|
|
smp_rmb();
|
|
|
|
|
|
|
|
/* Need to do the rmb for every iteration */
|
|
|
|
if (skip)
|
|
|
|
return 0;
|
|
|
|
|
2020-10-01 14:05:39 +00:00
|
|
|
if (cpumask_any_and(&p->cpus_mask, vec->mask) >= nr_cpu_ids)
|
2020-03-02 13:27:16 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (lowest_mask) {
|
2020-10-01 14:05:39 +00:00
|
|
|
cpumask_and(lowest_mask, &p->cpus_mask, vec->mask);
|
2023-09-23 01:14:08 +00:00
|
|
|
cpumask_and(lowest_mask, lowest_mask, cpu_active_mask);
|
2020-03-02 13:27:16 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We have to ensure that we have at least one bit
|
|
|
|
* still set in the array, since the map could have
|
|
|
|
* been concurrently emptied between the first and
|
|
|
|
* second reads of vec->mask. If we hit this
|
|
|
|
* condition, simply act as though we never hit this
|
|
|
|
* priority level and continue on.
|
|
|
|
*/
|
|
|
|
if (cpumask_empty(lowest_mask))
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2020-03-02 13:27:18 +00:00
|
|
|
int cpupri_find(struct cpupri *cp, struct task_struct *p,
|
|
|
|
struct cpumask *lowest_mask)
|
|
|
|
{
|
|
|
|
return cpupri_find_fitness(cp, p, lowest_mask, NULL);
|
|
|
|
}
|
|
|
|
|
2008-05-12 19:21:01 +00:00
|
|
|
/**
|
2020-03-02 13:27:18 +00:00
|
|
|
* cpupri_find_fitness - find the best (lowest-pri) CPU in the system
|
2008-05-12 19:21:01 +00:00
|
|
|
* @cp: The cpupri context
|
|
|
|
* @p: The task
|
2009-03-25 04:31:22 +00:00
|
|
|
* @lowest_mask: A mask to fill in with selected CPUs (or NULL)
|
2019-10-09 10:46:11 +00:00
|
|
|
* @fitness_fn: A pointer to a function to do custom checks whether the CPU
|
|
|
|
* fits a specific criteria so that we only return those CPUs.
|
2008-05-12 19:21:01 +00:00
|
|
|
*
|
|
|
|
* Note: This function returns the recommended CPUs as calculated during the
|
2009-12-11 21:35:40 +00:00
|
|
|
* current invocation. By the time the call returns, the CPUs may have in
|
2008-05-12 19:21:01 +00:00
|
|
|
* fact changed priorities any number of times. While not ideal, it is not
|
|
|
|
* an issue of correctness since the normal rebalancer logic will correct
|
|
|
|
* any discrepancies created by racing against the uncertainty of the current
|
|
|
|
* priority configuration.
|
|
|
|
*
|
2013-07-12 18:45:47 +00:00
|
|
|
* Return: (int)bool - CPUs were found
|
2008-05-12 19:21:01 +00:00
|
|
|
*/
|
2020-03-02 13:27:18 +00:00
|
|
|
int cpupri_find_fitness(struct cpupri *cp, struct task_struct *p,
|
2019-10-09 10:46:11 +00:00
|
|
|
struct cpumask *lowest_mask,
|
|
|
|
bool (*fitness_fn)(struct task_struct *p, int cpu))
|
2008-05-12 19:21:01 +00:00
|
|
|
{
|
2012-07-12 07:03:42 +00:00
|
|
|
int task_pri = convert_prio(p->prio);
|
2020-03-05 10:24:50 +00:00
|
|
|
int idx, cpu;
|
2008-05-12 19:21:01 +00:00
|
|
|
|
2022-08-11 06:54:52 +00:00
|
|
|
WARN_ON_ONCE(task_pri >= CPUPRI_NR_PRIORITIES);
|
sched/cpupri: Remove the vec->lock
sched/cpupri: Remove the vec->lock
The cpupri vec->lock has been showing up as a top contention
lately. This is because of the RT push/pull logic takes an
agressive approach for migrating RT tasks. The cpupri logic is
in place to improve the performance of the push/pull when dealing
with large number CPU machines.
The problem though is a vec->lock is required, where a vec is a
global per RT priority structure. That is, if there are lots of
RT tasks at the same priority, every time they are added or removed
from the RT queue, this global vec->lock is taken. Now that more
kernel threads are becoming RT (RCU boost and threaded interrupts)
this is becoming much more of an issue.
There are two variables that are being synced by the vec->lock.
The cpupri bitmask, and the vec->counter. The cpupri bitmask
is one bit per priority. If a RT priority vec has a process queued,
then the vec->count is > 0 and the cpupri bitmask is set for that
RT priority.
If the cpupri bitmask gets out of sync with the vec->counter, we could
end up pushing a low proirity RT task to a high priority queue.
That RT task that could have run immediately could be queued on a
run queue with a higher priority task indefinitely.
The solution is not to use the cpupri bitmask and just look at the
vec->count directly when doing a pull. The cpupri bitmask is just
a fast way to scan the RT priorities when a pull is made. Instead
of using the bitmask, and just examine all RT priorities, and
look at the vec->counts, we could eliminate the vec->lock. The
scan of RT tasks is to find a run queue that we can push an RT task
to, and we do not push to a high priority queue, thus the scan only
needs to go from 1 to RT task->prio, and not all 100 RT priorities.
The push algorithm, which does the scan of RT priorities (and
scan of the bitmask) only happens when we have an overloaded RT run
queue (more than one RT task queued). The grabbing of the vec->lock
happens every time any RT task is queued or dequeued on the run
queue for that priority. The slowing down of the scan by not using
a bitmask is negligible by the speed up of removing the vec->lock
contention, and replacing it with an atomic counter and memory barrier.
To prove this, I wrote a patch that times both the loop and the code
that grabs the vec->locks. I passed the patches to various people
(and companies) to test and show the results. I let everyone choose
their own load to test, giving different loads on the system,
for various different setups.
Here's some of the results: (snipping to a few CPUs to not make
this change log huge, but the results were consistent across
the entire system).
System 1 (24 CPUs)
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
[...]
cpu 20: loop 3057 1.766 0.061 0.642 1963.170
vec 6782949 90.469 0.089 0.414 2811760.503
cpu 21: loop 2617 1.723 0.062 0.641 1679.074
vec 6782810 90.499 0.089 0.291 1978499.900
cpu 22: loop 2212 1.863 0.063 0.699 1547.160
vec 6767244 85.685 0.089 0.435 2949676.898
cpu 23: loop 2320 2.013 0.062 0.594 1380.265
vec 6781694 87.923 0.088 0.431 2928538.224
After patch:
cpu 20: loop 2078 1.579 0.061 0.533 1108.006
vec 6164555 5.704 0.060 0.143 885185.809
cpu 21: loop 2268 1.712 0.065 0.575 1305.248
vec 6153376 5.558 0.060 0.187 1154960.469
cpu 22: loop 1542 1.639 0.095 0.533 823.249
vec 6156510 5.720 0.060 0.190 1172727.232
cpu 23: loop 1650 1.733 0.068 0.545 900.781
vec 6170784 5.533 0.060 0.167 1034287.953
All times are in microseconds. The 'loop' is the amount of time spent
doing the loop across the priorities (before patch uses bitmask).
the 'vec' is the amount of time in the code that requires grabbing
the vec->lock. The second patch just does not have the vec lock, but
encompasses the same code.
Amazingly the loop code even went down on average. The vec code went
from .5 down to .18, that's more than half the time spent!
Note, more than one test was run, but they all had the same results.
System 2 (64 CPUs)
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 60: loop 0 0 0 0 0
vec 5410840 277.954 0.084 0.782 4232895.727
cpu 61: loop 0 0 0 0 0
vec 4915648 188.399 0.084 0.570 2803220.301
cpu 62: loop 0 0 0 0 0
vec 5356076 276.417 0.085 0.786 4214544.548
cpu 63: loop 0 0 0 0 0
vec 4891837 170.531 0.085 0.799 3910948.833
After patch:
cpu 60: loop 0 0 0 0 0
vec 5365118 5.080 0.021 0.063 340490.267
cpu 61: loop 0 0 0 0 0
vec 4898590 1.757 0.019 0.071 347903.615
cpu 62: loop 0 0 0 0 0
vec 5737130 3.067 0.021 0.119 687108.734
cpu 63: loop 0 0 0 0 0
vec 4903228 1.822 0.021 0.071 348506.477
The test run during the measurement did not have any (very few,
from other CPUs) RT tasks pushing. But this shows that it helped
out tremendously with the contention, as the contention happens
because the vec->lock is taken only on queuing at an RT priority,
and different CPUs that queue tasks at the same priority will
have contention.
I tested on my own 4 CPU machine with the following results:
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 0: loop 2377 1.489 0.158 0.588 1398.395
vec 4484 770.146 2.301 4.396 19711.755
cpu 1: loop 2169 1.962 0.160 0.576 1250.110
vec 4425 152.769 2.297 4.030 17834.228
cpu 2: loop 2324 1.749 0.155 0.559 1299.799
vec 4368 779.632 2.325 4.665 20379.268
cpu 3: loop 2325 1.629 0.157 0.561 1306.113
vec 4650 408.782 2.394 4.348 20222.577
After patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 0: loop 2121 1.616 0.113 0.636 1349.189
vec 4303 1.151 0.225 0.421 1811.966
cpu 1: loop 2130 1.638 0.178 0.644 1372.927
vec 4627 1.379 0.235 0.428 1983.648
cpu 2: loop 2056 1.464 0.165 0.637 1310.141
vec 4471 1.311 0.217 0.433 1937.927
cpu 3: loop 2154 1.481 0.162 0.601 1295.083
vec 4236 1.253 0.230 0.425 1803.008
This was running my migrate.c code that can be found at:
http://lwn.net/Articles/425763/
The migrate code does stress the RT tasks a bit. This shows that
the loop did increase a little after the patch, but not by much.
The vec code dropped dramatically. From 4.3us down to .42us.
That's a 10x improvement!
Tested-by: Mike Galbraith <mgalbraith@suse.de>
Tested-by: Luis Claudio R. Gonçalves <lgoncalv@redhat.com>
Tested-by: Matthew Hank Sabins<msabins@linux.vnet.ibm.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Gregory Haskins <gregory.haskins@gmail.com>
Acked-by: Hillf Danton <dhillf@gmail.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Chris Mason <chris.mason@oracle.com>
Link: http://lkml.kernel.org/r/1312317372.18583.101.camel@gandalf.stny.rr.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-08-02 20:36:12 +00:00
|
|
|
|
|
|
|
for (idx = 0; idx < task_pri; idx++) {
|
sched/cpupri: Fix memory barriers for vec updates to always be in order
[ This patch actually compiles. Thanks to Mike Galbraith for pointing
that out. I compiled and booted this patch with no issues. ]
Re-examining the cpupri patch, I see there's a possible race because the
update of the two priorities vec->counts are not protected by a memory
barrier.
When a RT runqueue is overloaded and wants to push an RT task to another
runqueue, it scans the RT priority vectors in a loop from lowest
priority to highest.
When we queue or dequeue an RT task that changes a runqueue's highest
priority task, we update the vectors to show that a runqueue is rated at
a different priority. To do this, we first set the new priority mask,
and increment the vec->count, and then set the old priority mask by
decrementing the vec->count.
If we are lowering the runqueue's RT priority rating, it will trigger a
RT pull, and we do not care if we miss pushing to this runqueue or not.
But if we raise the priority, but the priority is still lower than an RT
task that is looking to be pushed, we must make sure that this runqueue
is still seen by the push algorithm (the loop).
Because the loop reads from lowest to highest, and the new priority is
set before the old one is cleared, we will either see the new or old
priority set and the vector will be checked.
But! Since there's no memory barrier between the updates of the two, the
old count may be decremented first before the new count is incremented.
This means the loop may see the old count of zero and skip it, and also
the new count of zero before it was updated. A possible runqueue that
the RT task could move to could be missed.
A conditional memory barrier is placed between the vec->count updates
and is only called when both updates are done.
The smp_wmb() has also been changed to smp_mb__before_atomic_inc/dec(),
as they are not needed by archs that already synchronize
atomic_inc/dec().
The smp_rmb() has been moved to be called at every iteration of the loop
so that the race between seeing the two updates is visible by each
iteration of the loop, as an arch is free to optimize the reading of
memory of the counters in the loop.
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Nick Piggin <npiggin@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/1312547269.18583.194.camel@gandalf.stny.rr.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-08-05 12:27:49 +00:00
|
|
|
|
2020-03-02 13:27:16 +00:00
|
|
|
if (!__cpupri_find(cp, p, lowest_mask, idx))
|
2008-05-12 19:21:01 +00:00
|
|
|
continue;
|
|
|
|
|
2020-03-02 13:27:16 +00:00
|
|
|
if (!lowest_mask || !fitness_fn)
|
|
|
|
return 1;
|
2019-10-09 10:46:11 +00:00
|
|
|
|
2020-03-02 13:27:16 +00:00
|
|
|
/* Ensure the capacity of the CPUs fit the task */
|
|
|
|
for_each_cpu(cpu, lowest_mask) {
|
|
|
|
if (!fitness_fn(p, cpu))
|
|
|
|
cpumask_clear_cpu(cpu, lowest_mask);
|
|
|
|
}
|
2009-07-30 14:57:28 +00:00
|
|
|
|
2020-03-02 13:27:16 +00:00
|
|
|
/*
|
|
|
|
* If no CPU at the current priority can fit the task
|
|
|
|
* continue looking
|
|
|
|
*/
|
2020-03-05 10:24:50 +00:00
|
|
|
if (cpumask_empty(lowest_mask))
|
2020-03-02 13:27:16 +00:00
|
|
|
continue;
|
2009-07-30 14:57:28 +00:00
|
|
|
|
2008-05-12 19:21:01 +00:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2020-03-02 13:27:16 +00:00
|
|
|
/*
|
2020-03-05 10:24:50 +00:00
|
|
|
* If we failed to find a fitting lowest_mask, kick off a new search
|
|
|
|
* but without taking into account any fitness criteria this time.
|
2020-03-02 13:27:16 +00:00
|
|
|
*
|
|
|
|
* This rule favours honouring priority over fitting the task in the
|
|
|
|
* correct CPU (Capacity Awareness being the only user now).
|
|
|
|
* The idea is that if a higher priority task can run, then it should
|
|
|
|
* run even if this ends up being on unfitting CPU.
|
|
|
|
*
|
|
|
|
* The cost of this trade-off is not entirely clear and will probably
|
|
|
|
* be good for some workloads and bad for others.
|
|
|
|
*
|
2021-03-18 12:38:50 +00:00
|
|
|
* The main idea here is that if some CPUs were over-committed, we try
|
2020-03-02 13:27:16 +00:00
|
|
|
* to spread which is what the scheduler traditionally did. Sys admins
|
|
|
|
* must do proper RT planning to avoid overloading the system if they
|
|
|
|
* really care.
|
|
|
|
*/
|
2020-03-05 10:24:50 +00:00
|
|
|
if (fitness_fn)
|
|
|
|
return cpupri_find(cp, p, lowest_mask);
|
2020-03-02 13:27:16 +00:00
|
|
|
|
2008-05-12 19:21:01 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 13:01:12 +00:00
|
|
|
* cpupri_set - update the CPU priority setting
|
2008-05-12 19:21:01 +00:00
|
|
|
* @cp: The cpupri context
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 13:01:12 +00:00
|
|
|
* @cpu: The target CPU
|
2020-10-14 19:39:04 +00:00
|
|
|
* @newpri: The priority (INVALID,NORMAL,RT1-RT99,HIGHER) to assign to this CPU
|
2008-05-12 19:21:01 +00:00
|
|
|
*
|
|
|
|
* Note: Assumes cpu_rq(cpu)->lock is locked
|
|
|
|
*
|
|
|
|
* Returns: (void)
|
|
|
|
*/
|
|
|
|
void cpupri_set(struct cpupri *cp, int cpu, int newpri)
|
|
|
|
{
|
2012-07-12 07:03:42 +00:00
|
|
|
int *currpri = &cp->cpu_to_pri[cpu];
|
|
|
|
int oldpri = *currpri;
|
|
|
|
int do_mb = 0;
|
2008-05-12 19:21:01 +00:00
|
|
|
|
|
|
|
newpri = convert_prio(newpri);
|
|
|
|
|
|
|
|
BUG_ON(newpri >= CPUPRI_NR_PRIORITIES);
|
|
|
|
|
|
|
|
if (newpri == oldpri)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 13:01:12 +00:00
|
|
|
* If the CPU was currently mapped to a different value, we
|
2009-07-29 04:21:23 +00:00
|
|
|
* need to map it to the new value then remove the old value.
|
|
|
|
* Note, we must add the new value first, otherwise we risk the
|
2011-08-06 00:10:04 +00:00
|
|
|
* cpu being missed by the priority loop in cpupri_find.
|
2008-05-12 19:21:01 +00:00
|
|
|
*/
|
|
|
|
if (likely(newpri != CPUPRI_INVALID)) {
|
|
|
|
struct cpupri_vec *vec = &cp->pri_to_cpu[newpri];
|
|
|
|
|
2008-11-24 16:05:13 +00:00
|
|
|
cpumask_set_cpu(cpu, vec->mask);
|
sched/cpupri: Remove the vec->lock
sched/cpupri: Remove the vec->lock
The cpupri vec->lock has been showing up as a top contention
lately. This is because of the RT push/pull logic takes an
agressive approach for migrating RT tasks. The cpupri logic is
in place to improve the performance of the push/pull when dealing
with large number CPU machines.
The problem though is a vec->lock is required, where a vec is a
global per RT priority structure. That is, if there are lots of
RT tasks at the same priority, every time they are added or removed
from the RT queue, this global vec->lock is taken. Now that more
kernel threads are becoming RT (RCU boost and threaded interrupts)
this is becoming much more of an issue.
There are two variables that are being synced by the vec->lock.
The cpupri bitmask, and the vec->counter. The cpupri bitmask
is one bit per priority. If a RT priority vec has a process queued,
then the vec->count is > 0 and the cpupri bitmask is set for that
RT priority.
If the cpupri bitmask gets out of sync with the vec->counter, we could
end up pushing a low proirity RT task to a high priority queue.
That RT task that could have run immediately could be queued on a
run queue with a higher priority task indefinitely.
The solution is not to use the cpupri bitmask and just look at the
vec->count directly when doing a pull. The cpupri bitmask is just
a fast way to scan the RT priorities when a pull is made. Instead
of using the bitmask, and just examine all RT priorities, and
look at the vec->counts, we could eliminate the vec->lock. The
scan of RT tasks is to find a run queue that we can push an RT task
to, and we do not push to a high priority queue, thus the scan only
needs to go from 1 to RT task->prio, and not all 100 RT priorities.
The push algorithm, which does the scan of RT priorities (and
scan of the bitmask) only happens when we have an overloaded RT run
queue (more than one RT task queued). The grabbing of the vec->lock
happens every time any RT task is queued or dequeued on the run
queue for that priority. The slowing down of the scan by not using
a bitmask is negligible by the speed up of removing the vec->lock
contention, and replacing it with an atomic counter and memory barrier.
To prove this, I wrote a patch that times both the loop and the code
that grabs the vec->locks. I passed the patches to various people
(and companies) to test and show the results. I let everyone choose
their own load to test, giving different loads on the system,
for various different setups.
Here's some of the results: (snipping to a few CPUs to not make
this change log huge, but the results were consistent across
the entire system).
System 1 (24 CPUs)
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
[...]
cpu 20: loop 3057 1.766 0.061 0.642 1963.170
vec 6782949 90.469 0.089 0.414 2811760.503
cpu 21: loop 2617 1.723 0.062 0.641 1679.074
vec 6782810 90.499 0.089 0.291 1978499.900
cpu 22: loop 2212 1.863 0.063 0.699 1547.160
vec 6767244 85.685 0.089 0.435 2949676.898
cpu 23: loop 2320 2.013 0.062 0.594 1380.265
vec 6781694 87.923 0.088 0.431 2928538.224
After patch:
cpu 20: loop 2078 1.579 0.061 0.533 1108.006
vec 6164555 5.704 0.060 0.143 885185.809
cpu 21: loop 2268 1.712 0.065 0.575 1305.248
vec 6153376 5.558 0.060 0.187 1154960.469
cpu 22: loop 1542 1.639 0.095 0.533 823.249
vec 6156510 5.720 0.060 0.190 1172727.232
cpu 23: loop 1650 1.733 0.068 0.545 900.781
vec 6170784 5.533 0.060 0.167 1034287.953
All times are in microseconds. The 'loop' is the amount of time spent
doing the loop across the priorities (before patch uses bitmask).
the 'vec' is the amount of time in the code that requires grabbing
the vec->lock. The second patch just does not have the vec lock, but
encompasses the same code.
Amazingly the loop code even went down on average. The vec code went
from .5 down to .18, that's more than half the time spent!
Note, more than one test was run, but they all had the same results.
System 2 (64 CPUs)
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 60: loop 0 0 0 0 0
vec 5410840 277.954 0.084 0.782 4232895.727
cpu 61: loop 0 0 0 0 0
vec 4915648 188.399 0.084 0.570 2803220.301
cpu 62: loop 0 0 0 0 0
vec 5356076 276.417 0.085 0.786 4214544.548
cpu 63: loop 0 0 0 0 0
vec 4891837 170.531 0.085 0.799 3910948.833
After patch:
cpu 60: loop 0 0 0 0 0
vec 5365118 5.080 0.021 0.063 340490.267
cpu 61: loop 0 0 0 0 0
vec 4898590 1.757 0.019 0.071 347903.615
cpu 62: loop 0 0 0 0 0
vec 5737130 3.067 0.021 0.119 687108.734
cpu 63: loop 0 0 0 0 0
vec 4903228 1.822 0.021 0.071 348506.477
The test run during the measurement did not have any (very few,
from other CPUs) RT tasks pushing. But this shows that it helped
out tremendously with the contention, as the contention happens
because the vec->lock is taken only on queuing at an RT priority,
and different CPUs that queue tasks at the same priority will
have contention.
I tested on my own 4 CPU machine with the following results:
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 0: loop 2377 1.489 0.158 0.588 1398.395
vec 4484 770.146 2.301 4.396 19711.755
cpu 1: loop 2169 1.962 0.160 0.576 1250.110
vec 4425 152.769 2.297 4.030 17834.228
cpu 2: loop 2324 1.749 0.155 0.559 1299.799
vec 4368 779.632 2.325 4.665 20379.268
cpu 3: loop 2325 1.629 0.157 0.561 1306.113
vec 4650 408.782 2.394 4.348 20222.577
After patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 0: loop 2121 1.616 0.113 0.636 1349.189
vec 4303 1.151 0.225 0.421 1811.966
cpu 1: loop 2130 1.638 0.178 0.644 1372.927
vec 4627 1.379 0.235 0.428 1983.648
cpu 2: loop 2056 1.464 0.165 0.637 1310.141
vec 4471 1.311 0.217 0.433 1937.927
cpu 3: loop 2154 1.481 0.162 0.601 1295.083
vec 4236 1.253 0.230 0.425 1803.008
This was running my migrate.c code that can be found at:
http://lwn.net/Articles/425763/
The migrate code does stress the RT tasks a bit. This shows that
the loop did increase a little after the patch, but not by much.
The vec code dropped dramatically. From 4.3us down to .42us.
That's a 10x improvement!
Tested-by: Mike Galbraith <mgalbraith@suse.de>
Tested-by: Luis Claudio R. Gonçalves <lgoncalv@redhat.com>
Tested-by: Matthew Hank Sabins<msabins@linux.vnet.ibm.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Gregory Haskins <gregory.haskins@gmail.com>
Acked-by: Hillf Danton <dhillf@gmail.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Chris Mason <chris.mason@oracle.com>
Link: http://lkml.kernel.org/r/1312317372.18583.101.camel@gandalf.stny.rr.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-08-02 20:36:12 +00:00
|
|
|
/*
|
|
|
|
* When adding a new vector, we update the mask first,
|
|
|
|
* do a write memory barrier, and then update the count, to
|
|
|
|
* make sure the vector is visible when count is set.
|
|
|
|
*/
|
2014-03-17 17:06:10 +00:00
|
|
|
smp_mb__before_atomic();
|
sched/cpupri: Remove the vec->lock
sched/cpupri: Remove the vec->lock
The cpupri vec->lock has been showing up as a top contention
lately. This is because of the RT push/pull logic takes an
agressive approach for migrating RT tasks. The cpupri logic is
in place to improve the performance of the push/pull when dealing
with large number CPU machines.
The problem though is a vec->lock is required, where a vec is a
global per RT priority structure. That is, if there are lots of
RT tasks at the same priority, every time they are added or removed
from the RT queue, this global vec->lock is taken. Now that more
kernel threads are becoming RT (RCU boost and threaded interrupts)
this is becoming much more of an issue.
There are two variables that are being synced by the vec->lock.
The cpupri bitmask, and the vec->counter. The cpupri bitmask
is one bit per priority. If a RT priority vec has a process queued,
then the vec->count is > 0 and the cpupri bitmask is set for that
RT priority.
If the cpupri bitmask gets out of sync with the vec->counter, we could
end up pushing a low proirity RT task to a high priority queue.
That RT task that could have run immediately could be queued on a
run queue with a higher priority task indefinitely.
The solution is not to use the cpupri bitmask and just look at the
vec->count directly when doing a pull. The cpupri bitmask is just
a fast way to scan the RT priorities when a pull is made. Instead
of using the bitmask, and just examine all RT priorities, and
look at the vec->counts, we could eliminate the vec->lock. The
scan of RT tasks is to find a run queue that we can push an RT task
to, and we do not push to a high priority queue, thus the scan only
needs to go from 1 to RT task->prio, and not all 100 RT priorities.
The push algorithm, which does the scan of RT priorities (and
scan of the bitmask) only happens when we have an overloaded RT run
queue (more than one RT task queued). The grabbing of the vec->lock
happens every time any RT task is queued or dequeued on the run
queue for that priority. The slowing down of the scan by not using
a bitmask is negligible by the speed up of removing the vec->lock
contention, and replacing it with an atomic counter and memory barrier.
To prove this, I wrote a patch that times both the loop and the code
that grabs the vec->locks. I passed the patches to various people
(and companies) to test and show the results. I let everyone choose
their own load to test, giving different loads on the system,
for various different setups.
Here's some of the results: (snipping to a few CPUs to not make
this change log huge, but the results were consistent across
the entire system).
System 1 (24 CPUs)
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
[...]
cpu 20: loop 3057 1.766 0.061 0.642 1963.170
vec 6782949 90.469 0.089 0.414 2811760.503
cpu 21: loop 2617 1.723 0.062 0.641 1679.074
vec 6782810 90.499 0.089 0.291 1978499.900
cpu 22: loop 2212 1.863 0.063 0.699 1547.160
vec 6767244 85.685 0.089 0.435 2949676.898
cpu 23: loop 2320 2.013 0.062 0.594 1380.265
vec 6781694 87.923 0.088 0.431 2928538.224
After patch:
cpu 20: loop 2078 1.579 0.061 0.533 1108.006
vec 6164555 5.704 0.060 0.143 885185.809
cpu 21: loop 2268 1.712 0.065 0.575 1305.248
vec 6153376 5.558 0.060 0.187 1154960.469
cpu 22: loop 1542 1.639 0.095 0.533 823.249
vec 6156510 5.720 0.060 0.190 1172727.232
cpu 23: loop 1650 1.733 0.068 0.545 900.781
vec 6170784 5.533 0.060 0.167 1034287.953
All times are in microseconds. The 'loop' is the amount of time spent
doing the loop across the priorities (before patch uses bitmask).
the 'vec' is the amount of time in the code that requires grabbing
the vec->lock. The second patch just does not have the vec lock, but
encompasses the same code.
Amazingly the loop code even went down on average. The vec code went
from .5 down to .18, that's more than half the time spent!
Note, more than one test was run, but they all had the same results.
System 2 (64 CPUs)
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 60: loop 0 0 0 0 0
vec 5410840 277.954 0.084 0.782 4232895.727
cpu 61: loop 0 0 0 0 0
vec 4915648 188.399 0.084 0.570 2803220.301
cpu 62: loop 0 0 0 0 0
vec 5356076 276.417 0.085 0.786 4214544.548
cpu 63: loop 0 0 0 0 0
vec 4891837 170.531 0.085 0.799 3910948.833
After patch:
cpu 60: loop 0 0 0 0 0
vec 5365118 5.080 0.021 0.063 340490.267
cpu 61: loop 0 0 0 0 0
vec 4898590 1.757 0.019 0.071 347903.615
cpu 62: loop 0 0 0 0 0
vec 5737130 3.067 0.021 0.119 687108.734
cpu 63: loop 0 0 0 0 0
vec 4903228 1.822 0.021 0.071 348506.477
The test run during the measurement did not have any (very few,
from other CPUs) RT tasks pushing. But this shows that it helped
out tremendously with the contention, as the contention happens
because the vec->lock is taken only on queuing at an RT priority,
and different CPUs that queue tasks at the same priority will
have contention.
I tested on my own 4 CPU machine with the following results:
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 0: loop 2377 1.489 0.158 0.588 1398.395
vec 4484 770.146 2.301 4.396 19711.755
cpu 1: loop 2169 1.962 0.160 0.576 1250.110
vec 4425 152.769 2.297 4.030 17834.228
cpu 2: loop 2324 1.749 0.155 0.559 1299.799
vec 4368 779.632 2.325 4.665 20379.268
cpu 3: loop 2325 1.629 0.157 0.561 1306.113
vec 4650 408.782 2.394 4.348 20222.577
After patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 0: loop 2121 1.616 0.113 0.636 1349.189
vec 4303 1.151 0.225 0.421 1811.966
cpu 1: loop 2130 1.638 0.178 0.644 1372.927
vec 4627 1.379 0.235 0.428 1983.648
cpu 2: loop 2056 1.464 0.165 0.637 1310.141
vec 4471 1.311 0.217 0.433 1937.927
cpu 3: loop 2154 1.481 0.162 0.601 1295.083
vec 4236 1.253 0.230 0.425 1803.008
This was running my migrate.c code that can be found at:
http://lwn.net/Articles/425763/
The migrate code does stress the RT tasks a bit. This shows that
the loop did increase a little after the patch, but not by much.
The vec code dropped dramatically. From 4.3us down to .42us.
That's a 10x improvement!
Tested-by: Mike Galbraith <mgalbraith@suse.de>
Tested-by: Luis Claudio R. Gonçalves <lgoncalv@redhat.com>
Tested-by: Matthew Hank Sabins<msabins@linux.vnet.ibm.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Gregory Haskins <gregory.haskins@gmail.com>
Acked-by: Hillf Danton <dhillf@gmail.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Chris Mason <chris.mason@oracle.com>
Link: http://lkml.kernel.org/r/1312317372.18583.101.camel@gandalf.stny.rr.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-08-02 20:36:12 +00:00
|
|
|
atomic_inc(&(vec)->count);
|
sched/cpupri: Fix memory barriers for vec updates to always be in order
[ This patch actually compiles. Thanks to Mike Galbraith for pointing
that out. I compiled and booted this patch with no issues. ]
Re-examining the cpupri patch, I see there's a possible race because the
update of the two priorities vec->counts are not protected by a memory
barrier.
When a RT runqueue is overloaded and wants to push an RT task to another
runqueue, it scans the RT priority vectors in a loop from lowest
priority to highest.
When we queue or dequeue an RT task that changes a runqueue's highest
priority task, we update the vectors to show that a runqueue is rated at
a different priority. To do this, we first set the new priority mask,
and increment the vec->count, and then set the old priority mask by
decrementing the vec->count.
If we are lowering the runqueue's RT priority rating, it will trigger a
RT pull, and we do not care if we miss pushing to this runqueue or not.
But if we raise the priority, but the priority is still lower than an RT
task that is looking to be pushed, we must make sure that this runqueue
is still seen by the push algorithm (the loop).
Because the loop reads from lowest to highest, and the new priority is
set before the old one is cleared, we will either see the new or old
priority set and the vector will be checked.
But! Since there's no memory barrier between the updates of the two, the
old count may be decremented first before the new count is incremented.
This means the loop may see the old count of zero and skip it, and also
the new count of zero before it was updated. A possible runqueue that
the RT task could move to could be missed.
A conditional memory barrier is placed between the vec->count updates
and is only called when both updates are done.
The smp_wmb() has also been changed to smp_mb__before_atomic_inc/dec(),
as they are not needed by archs that already synchronize
atomic_inc/dec().
The smp_rmb() has been moved to be called at every iteration of the loop
so that the race between seeing the two updates is visible by each
iteration of the loop, as an arch is free to optimize the reading of
memory of the counters in the loop.
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Nick Piggin <npiggin@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/1312547269.18583.194.camel@gandalf.stny.rr.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-08-05 12:27:49 +00:00
|
|
|
do_mb = 1;
|
2008-05-12 19:21:01 +00:00
|
|
|
}
|
2009-07-29 04:21:23 +00:00
|
|
|
if (likely(oldpri != CPUPRI_INVALID)) {
|
|
|
|
struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri];
|
|
|
|
|
sched/cpupri: Fix memory barriers for vec updates to always be in order
[ This patch actually compiles. Thanks to Mike Galbraith for pointing
that out. I compiled and booted this patch with no issues. ]
Re-examining the cpupri patch, I see there's a possible race because the
update of the two priorities vec->counts are not protected by a memory
barrier.
When a RT runqueue is overloaded and wants to push an RT task to another
runqueue, it scans the RT priority vectors in a loop from lowest
priority to highest.
When we queue or dequeue an RT task that changes a runqueue's highest
priority task, we update the vectors to show that a runqueue is rated at
a different priority. To do this, we first set the new priority mask,
and increment the vec->count, and then set the old priority mask by
decrementing the vec->count.
If we are lowering the runqueue's RT priority rating, it will trigger a
RT pull, and we do not care if we miss pushing to this runqueue or not.
But if we raise the priority, but the priority is still lower than an RT
task that is looking to be pushed, we must make sure that this runqueue
is still seen by the push algorithm (the loop).
Because the loop reads from lowest to highest, and the new priority is
set before the old one is cleared, we will either see the new or old
priority set and the vector will be checked.
But! Since there's no memory barrier between the updates of the two, the
old count may be decremented first before the new count is incremented.
This means the loop may see the old count of zero and skip it, and also
the new count of zero before it was updated. A possible runqueue that
the RT task could move to could be missed.
A conditional memory barrier is placed between the vec->count updates
and is only called when both updates are done.
The smp_wmb() has also been changed to smp_mb__before_atomic_inc/dec(),
as they are not needed by archs that already synchronize
atomic_inc/dec().
The smp_rmb() has been moved to be called at every iteration of the loop
so that the race between seeing the two updates is visible by each
iteration of the loop, as an arch is free to optimize the reading of
memory of the counters in the loop.
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Nick Piggin <npiggin@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/1312547269.18583.194.camel@gandalf.stny.rr.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-08-05 12:27:49 +00:00
|
|
|
/*
|
|
|
|
* Because the order of modification of the vec->count
|
|
|
|
* is important, we must make sure that the update
|
|
|
|
* of the new prio is seen before we decrement the
|
|
|
|
* old prio. This makes sure that the loop sees
|
|
|
|
* one or the other when we raise the priority of
|
|
|
|
* the run queue. We don't care about when we lower the
|
|
|
|
* priority, as that will trigger an rt pull anyway.
|
|
|
|
*
|
|
|
|
* We only need to do a memory barrier if we updated
|
|
|
|
* the new priority vec.
|
|
|
|
*/
|
|
|
|
if (do_mb)
|
2014-03-17 17:06:10 +00:00
|
|
|
smp_mb__after_atomic();
|
sched/cpupri: Fix memory barriers for vec updates to always be in order
[ This patch actually compiles. Thanks to Mike Galbraith for pointing
that out. I compiled and booted this patch with no issues. ]
Re-examining the cpupri patch, I see there's a possible race because the
update of the two priorities vec->counts are not protected by a memory
barrier.
When a RT runqueue is overloaded and wants to push an RT task to another
runqueue, it scans the RT priority vectors in a loop from lowest
priority to highest.
When we queue or dequeue an RT task that changes a runqueue's highest
priority task, we update the vectors to show that a runqueue is rated at
a different priority. To do this, we first set the new priority mask,
and increment the vec->count, and then set the old priority mask by
decrementing the vec->count.
If we are lowering the runqueue's RT priority rating, it will trigger a
RT pull, and we do not care if we miss pushing to this runqueue or not.
But if we raise the priority, but the priority is still lower than an RT
task that is looking to be pushed, we must make sure that this runqueue
is still seen by the push algorithm (the loop).
Because the loop reads from lowest to highest, and the new priority is
set before the old one is cleared, we will either see the new or old
priority set and the vector will be checked.
But! Since there's no memory barrier between the updates of the two, the
old count may be decremented first before the new count is incremented.
This means the loop may see the old count of zero and skip it, and also
the new count of zero before it was updated. A possible runqueue that
the RT task could move to could be missed.
A conditional memory barrier is placed between the vec->count updates
and is only called when both updates are done.
The smp_wmb() has also been changed to smp_mb__before_atomic_inc/dec(),
as they are not needed by archs that already synchronize
atomic_inc/dec().
The smp_rmb() has been moved to be called at every iteration of the loop
so that the race between seeing the two updates is visible by each
iteration of the loop, as an arch is free to optimize the reading of
memory of the counters in the loop.
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Nick Piggin <npiggin@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/1312547269.18583.194.camel@gandalf.stny.rr.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-08-05 12:27:49 +00:00
|
|
|
|
sched/cpupri: Remove the vec->lock
sched/cpupri: Remove the vec->lock
The cpupri vec->lock has been showing up as a top contention
lately. This is because of the RT push/pull logic takes an
agressive approach for migrating RT tasks. The cpupri logic is
in place to improve the performance of the push/pull when dealing
with large number CPU machines.
The problem though is a vec->lock is required, where a vec is a
global per RT priority structure. That is, if there are lots of
RT tasks at the same priority, every time they are added or removed
from the RT queue, this global vec->lock is taken. Now that more
kernel threads are becoming RT (RCU boost and threaded interrupts)
this is becoming much more of an issue.
There are two variables that are being synced by the vec->lock.
The cpupri bitmask, and the vec->counter. The cpupri bitmask
is one bit per priority. If a RT priority vec has a process queued,
then the vec->count is > 0 and the cpupri bitmask is set for that
RT priority.
If the cpupri bitmask gets out of sync with the vec->counter, we could
end up pushing a low proirity RT task to a high priority queue.
That RT task that could have run immediately could be queued on a
run queue with a higher priority task indefinitely.
The solution is not to use the cpupri bitmask and just look at the
vec->count directly when doing a pull. The cpupri bitmask is just
a fast way to scan the RT priorities when a pull is made. Instead
of using the bitmask, and just examine all RT priorities, and
look at the vec->counts, we could eliminate the vec->lock. The
scan of RT tasks is to find a run queue that we can push an RT task
to, and we do not push to a high priority queue, thus the scan only
needs to go from 1 to RT task->prio, and not all 100 RT priorities.
The push algorithm, which does the scan of RT priorities (and
scan of the bitmask) only happens when we have an overloaded RT run
queue (more than one RT task queued). The grabbing of the vec->lock
happens every time any RT task is queued or dequeued on the run
queue for that priority. The slowing down of the scan by not using
a bitmask is negligible by the speed up of removing the vec->lock
contention, and replacing it with an atomic counter and memory barrier.
To prove this, I wrote a patch that times both the loop and the code
that grabs the vec->locks. I passed the patches to various people
(and companies) to test and show the results. I let everyone choose
their own load to test, giving different loads on the system,
for various different setups.
Here's some of the results: (snipping to a few CPUs to not make
this change log huge, but the results were consistent across
the entire system).
System 1 (24 CPUs)
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
[...]
cpu 20: loop 3057 1.766 0.061 0.642 1963.170
vec 6782949 90.469 0.089 0.414 2811760.503
cpu 21: loop 2617 1.723 0.062 0.641 1679.074
vec 6782810 90.499 0.089 0.291 1978499.900
cpu 22: loop 2212 1.863 0.063 0.699 1547.160
vec 6767244 85.685 0.089 0.435 2949676.898
cpu 23: loop 2320 2.013 0.062 0.594 1380.265
vec 6781694 87.923 0.088 0.431 2928538.224
After patch:
cpu 20: loop 2078 1.579 0.061 0.533 1108.006
vec 6164555 5.704 0.060 0.143 885185.809
cpu 21: loop 2268 1.712 0.065 0.575 1305.248
vec 6153376 5.558 0.060 0.187 1154960.469
cpu 22: loop 1542 1.639 0.095 0.533 823.249
vec 6156510 5.720 0.060 0.190 1172727.232
cpu 23: loop 1650 1.733 0.068 0.545 900.781
vec 6170784 5.533 0.060 0.167 1034287.953
All times are in microseconds. The 'loop' is the amount of time spent
doing the loop across the priorities (before patch uses bitmask).
the 'vec' is the amount of time in the code that requires grabbing
the vec->lock. The second patch just does not have the vec lock, but
encompasses the same code.
Amazingly the loop code even went down on average. The vec code went
from .5 down to .18, that's more than half the time spent!
Note, more than one test was run, but they all had the same results.
System 2 (64 CPUs)
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 60: loop 0 0 0 0 0
vec 5410840 277.954 0.084 0.782 4232895.727
cpu 61: loop 0 0 0 0 0
vec 4915648 188.399 0.084 0.570 2803220.301
cpu 62: loop 0 0 0 0 0
vec 5356076 276.417 0.085 0.786 4214544.548
cpu 63: loop 0 0 0 0 0
vec 4891837 170.531 0.085 0.799 3910948.833
After patch:
cpu 60: loop 0 0 0 0 0
vec 5365118 5.080 0.021 0.063 340490.267
cpu 61: loop 0 0 0 0 0
vec 4898590 1.757 0.019 0.071 347903.615
cpu 62: loop 0 0 0 0 0
vec 5737130 3.067 0.021 0.119 687108.734
cpu 63: loop 0 0 0 0 0
vec 4903228 1.822 0.021 0.071 348506.477
The test run during the measurement did not have any (very few,
from other CPUs) RT tasks pushing. But this shows that it helped
out tremendously with the contention, as the contention happens
because the vec->lock is taken only on queuing at an RT priority,
and different CPUs that queue tasks at the same priority will
have contention.
I tested on my own 4 CPU machine with the following results:
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 0: loop 2377 1.489 0.158 0.588 1398.395
vec 4484 770.146 2.301 4.396 19711.755
cpu 1: loop 2169 1.962 0.160 0.576 1250.110
vec 4425 152.769 2.297 4.030 17834.228
cpu 2: loop 2324 1.749 0.155 0.559 1299.799
vec 4368 779.632 2.325 4.665 20379.268
cpu 3: loop 2325 1.629 0.157 0.561 1306.113
vec 4650 408.782 2.394 4.348 20222.577
After patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 0: loop 2121 1.616 0.113 0.636 1349.189
vec 4303 1.151 0.225 0.421 1811.966
cpu 1: loop 2130 1.638 0.178 0.644 1372.927
vec 4627 1.379 0.235 0.428 1983.648
cpu 2: loop 2056 1.464 0.165 0.637 1310.141
vec 4471 1.311 0.217 0.433 1937.927
cpu 3: loop 2154 1.481 0.162 0.601 1295.083
vec 4236 1.253 0.230 0.425 1803.008
This was running my migrate.c code that can be found at:
http://lwn.net/Articles/425763/
The migrate code does stress the RT tasks a bit. This shows that
the loop did increase a little after the patch, but not by much.
The vec code dropped dramatically. From 4.3us down to .42us.
That's a 10x improvement!
Tested-by: Mike Galbraith <mgalbraith@suse.de>
Tested-by: Luis Claudio R. Gonçalves <lgoncalv@redhat.com>
Tested-by: Matthew Hank Sabins<msabins@linux.vnet.ibm.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Gregory Haskins <gregory.haskins@gmail.com>
Acked-by: Hillf Danton <dhillf@gmail.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Chris Mason <chris.mason@oracle.com>
Link: http://lkml.kernel.org/r/1312317372.18583.101.camel@gandalf.stny.rr.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-08-02 20:36:12 +00:00
|
|
|
/*
|
|
|
|
* When removing from the vector, we decrement the counter first
|
|
|
|
* do a memory barrier and then clear the mask.
|
|
|
|
*/
|
|
|
|
atomic_dec(&(vec)->count);
|
2014-03-17 17:06:10 +00:00
|
|
|
smp_mb__after_atomic();
|
2009-07-29 04:21:23 +00:00
|
|
|
cpumask_clear_cpu(cpu, vec->mask);
|
|
|
|
}
|
2008-05-12 19:21:01 +00:00
|
|
|
|
|
|
|
*currpri = newpri;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpupri_init - initialize the cpupri structure
|
|
|
|
* @cp: The cpupri context
|
|
|
|
*
|
2013-07-12 18:45:47 +00:00
|
|
|
* Return: -ENOMEM on memory allocation failure.
|
2008-05-12 19:21:01 +00:00
|
|
|
*/
|
2010-07-15 20:18:22 +00:00
|
|
|
int cpupri_init(struct cpupri *cp)
|
2008-05-12 19:21:01 +00:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) {
|
|
|
|
struct cpupri_vec *vec = &cp->pri_to_cpu[i];
|
|
|
|
|
sched/cpupri: Remove the vec->lock
sched/cpupri: Remove the vec->lock
The cpupri vec->lock has been showing up as a top contention
lately. This is because of the RT push/pull logic takes an
agressive approach for migrating RT tasks. The cpupri logic is
in place to improve the performance of the push/pull when dealing
with large number CPU machines.
The problem though is a vec->lock is required, where a vec is a
global per RT priority structure. That is, if there are lots of
RT tasks at the same priority, every time they are added or removed
from the RT queue, this global vec->lock is taken. Now that more
kernel threads are becoming RT (RCU boost and threaded interrupts)
this is becoming much more of an issue.
There are two variables that are being synced by the vec->lock.
The cpupri bitmask, and the vec->counter. The cpupri bitmask
is one bit per priority. If a RT priority vec has a process queued,
then the vec->count is > 0 and the cpupri bitmask is set for that
RT priority.
If the cpupri bitmask gets out of sync with the vec->counter, we could
end up pushing a low proirity RT task to a high priority queue.
That RT task that could have run immediately could be queued on a
run queue with a higher priority task indefinitely.
The solution is not to use the cpupri bitmask and just look at the
vec->count directly when doing a pull. The cpupri bitmask is just
a fast way to scan the RT priorities when a pull is made. Instead
of using the bitmask, and just examine all RT priorities, and
look at the vec->counts, we could eliminate the vec->lock. The
scan of RT tasks is to find a run queue that we can push an RT task
to, and we do not push to a high priority queue, thus the scan only
needs to go from 1 to RT task->prio, and not all 100 RT priorities.
The push algorithm, which does the scan of RT priorities (and
scan of the bitmask) only happens when we have an overloaded RT run
queue (more than one RT task queued). The grabbing of the vec->lock
happens every time any RT task is queued or dequeued on the run
queue for that priority. The slowing down of the scan by not using
a bitmask is negligible by the speed up of removing the vec->lock
contention, and replacing it with an atomic counter and memory barrier.
To prove this, I wrote a patch that times both the loop and the code
that grabs the vec->locks. I passed the patches to various people
(and companies) to test and show the results. I let everyone choose
their own load to test, giving different loads on the system,
for various different setups.
Here's some of the results: (snipping to a few CPUs to not make
this change log huge, but the results were consistent across
the entire system).
System 1 (24 CPUs)
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
[...]
cpu 20: loop 3057 1.766 0.061 0.642 1963.170
vec 6782949 90.469 0.089 0.414 2811760.503
cpu 21: loop 2617 1.723 0.062 0.641 1679.074
vec 6782810 90.499 0.089 0.291 1978499.900
cpu 22: loop 2212 1.863 0.063 0.699 1547.160
vec 6767244 85.685 0.089 0.435 2949676.898
cpu 23: loop 2320 2.013 0.062 0.594 1380.265
vec 6781694 87.923 0.088 0.431 2928538.224
After patch:
cpu 20: loop 2078 1.579 0.061 0.533 1108.006
vec 6164555 5.704 0.060 0.143 885185.809
cpu 21: loop 2268 1.712 0.065 0.575 1305.248
vec 6153376 5.558 0.060 0.187 1154960.469
cpu 22: loop 1542 1.639 0.095 0.533 823.249
vec 6156510 5.720 0.060 0.190 1172727.232
cpu 23: loop 1650 1.733 0.068 0.545 900.781
vec 6170784 5.533 0.060 0.167 1034287.953
All times are in microseconds. The 'loop' is the amount of time spent
doing the loop across the priorities (before patch uses bitmask).
the 'vec' is the amount of time in the code that requires grabbing
the vec->lock. The second patch just does not have the vec lock, but
encompasses the same code.
Amazingly the loop code even went down on average. The vec code went
from .5 down to .18, that's more than half the time spent!
Note, more than one test was run, but they all had the same results.
System 2 (64 CPUs)
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 60: loop 0 0 0 0 0
vec 5410840 277.954 0.084 0.782 4232895.727
cpu 61: loop 0 0 0 0 0
vec 4915648 188.399 0.084 0.570 2803220.301
cpu 62: loop 0 0 0 0 0
vec 5356076 276.417 0.085 0.786 4214544.548
cpu 63: loop 0 0 0 0 0
vec 4891837 170.531 0.085 0.799 3910948.833
After patch:
cpu 60: loop 0 0 0 0 0
vec 5365118 5.080 0.021 0.063 340490.267
cpu 61: loop 0 0 0 0 0
vec 4898590 1.757 0.019 0.071 347903.615
cpu 62: loop 0 0 0 0 0
vec 5737130 3.067 0.021 0.119 687108.734
cpu 63: loop 0 0 0 0 0
vec 4903228 1.822 0.021 0.071 348506.477
The test run during the measurement did not have any (very few,
from other CPUs) RT tasks pushing. But this shows that it helped
out tremendously with the contention, as the contention happens
because the vec->lock is taken only on queuing at an RT priority,
and different CPUs that queue tasks at the same priority will
have contention.
I tested on my own 4 CPU machine with the following results:
Before patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 0: loop 2377 1.489 0.158 0.588 1398.395
vec 4484 770.146 2.301 4.396 19711.755
cpu 1: loop 2169 1.962 0.160 0.576 1250.110
vec 4425 152.769 2.297 4.030 17834.228
cpu 2: loop 2324 1.749 0.155 0.559 1299.799
vec 4368 779.632 2.325 4.665 20379.268
cpu 3: loop 2325 1.629 0.157 0.561 1306.113
vec 4650 408.782 2.394 4.348 20222.577
After patch:
CPU: Name Count Max Min Average Total
---- ---- ----- --- --- ------- -----
cpu 0: loop 2121 1.616 0.113 0.636 1349.189
vec 4303 1.151 0.225 0.421 1811.966
cpu 1: loop 2130 1.638 0.178 0.644 1372.927
vec 4627 1.379 0.235 0.428 1983.648
cpu 2: loop 2056 1.464 0.165 0.637 1310.141
vec 4471 1.311 0.217 0.433 1937.927
cpu 3: loop 2154 1.481 0.162 0.601 1295.083
vec 4236 1.253 0.230 0.425 1803.008
This was running my migrate.c code that can be found at:
http://lwn.net/Articles/425763/
The migrate code does stress the RT tasks a bit. This shows that
the loop did increase a little after the patch, but not by much.
The vec code dropped dramatically. From 4.3us down to .42us.
That's a 10x improvement!
Tested-by: Mike Galbraith <mgalbraith@suse.de>
Tested-by: Luis Claudio R. Gonçalves <lgoncalv@redhat.com>
Tested-by: Matthew Hank Sabins<msabins@linux.vnet.ibm.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Gregory Haskins <gregory.haskins@gmail.com>
Acked-by: Hillf Danton <dhillf@gmail.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Chris Mason <chris.mason@oracle.com>
Link: http://lkml.kernel.org/r/1312317372.18583.101.camel@gandalf.stny.rr.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-08-02 20:36:12 +00:00
|
|
|
atomic_set(&vec->count, 0);
|
2010-07-15 20:18:22 +00:00
|
|
|
if (!zalloc_cpumask_var(&vec->mask, GFP_KERNEL))
|
2008-11-24 16:05:13 +00:00
|
|
|
goto cleanup;
|
2008-05-12 19:21:01 +00:00
|
|
|
}
|
|
|
|
|
2014-05-14 14:04:26 +00:00
|
|
|
cp->cpu_to_pri = kcalloc(nr_cpu_ids, sizeof(int), GFP_KERNEL);
|
|
|
|
if (!cp->cpu_to_pri)
|
|
|
|
goto cleanup;
|
|
|
|
|
2008-05-12 19:21:01 +00:00
|
|
|
for_each_possible_cpu(i)
|
|
|
|
cp->cpu_to_pri[i] = CPUPRI_INVALID;
|
2014-05-14 14:04:26 +00:00
|
|
|
|
2008-11-24 16:05:13 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
cleanup:
|
|
|
|
for (i--; i >= 0; i--)
|
|
|
|
free_cpumask_var(cp->pri_to_cpu[i].mask);
|
|
|
|
return -ENOMEM;
|
2008-05-12 19:21:01 +00:00
|
|
|
}
|
|
|
|
|
2008-11-24 16:05:13 +00:00
|
|
|
/**
|
|
|
|
* cpupri_cleanup - clean up the cpupri structure
|
|
|
|
* @cp: The cpupri context
|
|
|
|
*/
|
|
|
|
void cpupri_cleanup(struct cpupri *cp)
|
|
|
|
{
|
|
|
|
int i;
|
2008-05-12 19:21:01 +00:00
|
|
|
|
2014-05-14 14:04:26 +00:00
|
|
|
kfree(cp->cpu_to_pri);
|
2008-11-24 16:05:13 +00:00
|
|
|
for (i = 0; i < CPUPRI_NR_PRIORITIES; i++)
|
|
|
|
free_cpumask_var(cp->pri_to_cpu[i].mask);
|
|
|
|
}
|