2012-10-22 23:28:05 +00:00
|
|
|
/*
|
|
|
|
* drivers/cpufreq/cpufreq_governor.c
|
|
|
|
*
|
|
|
|
* CPUFREQ governors common code
|
|
|
|
*
|
2012-10-25 22:47:42 +00:00
|
|
|
* Copyright (C) 2001 Russell King
|
|
|
|
* (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
|
|
|
|
* (C) 2003 Jun Nakajima <jun.nakajima@intel.com>
|
|
|
|
* (C) 2009 Alexander Clouter <alex@digriz.org.uk>
|
|
|
|
* (c) 2012 Viresh Kumar <viresh.kumar@linaro.org>
|
|
|
|
*
|
2012-10-22 23:28:05 +00:00
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*/
|
|
|
|
|
2012-10-25 22:47:42 +00:00
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
|
2012-10-22 23:28:05 +00:00
|
|
|
#include <linux/export.h>
|
|
|
|
#include <linux/kernel_stat.h>
|
2013-03-27 15:58:58 +00:00
|
|
|
#include <linux/slab.h>
|
2012-10-25 22:47:42 +00:00
|
|
|
|
|
|
|
#include "cpufreq_governor.h"
|
|
|
|
|
2017-12-18 01:15:32 +00:00
|
|
|
#define CPUFREQ_DBS_MIN_SAMPLING_INTERVAL (2 * TICK_NSEC / NSEC_PER_USEC)
|
|
|
|
|
2016-02-20 23:51:27 +00:00
|
|
|
static DEFINE_PER_CPU(struct cpu_dbs_info, cpu_dbs);
|
|
|
|
|
2016-02-20 23:53:06 +00:00
|
|
|
static DEFINE_MUTEX(gov_dbs_data_mutex);
|
2016-02-07 15:01:31 +00:00
|
|
|
|
2016-02-11 12:01:15 +00:00
|
|
|
/* Common sysfs tunables */
|
|
|
|
/**
|
|
|
|
* store_sampling_rate - update sampling rate effective immediately if needed.
|
|
|
|
*
|
|
|
|
* If new rate is smaller than the old, simply updating
|
|
|
|
* dbs.sampling_rate might not be appropriate. For example, if the
|
|
|
|
* original sampling_rate was 1 second and the requested new sampling rate is 10
|
|
|
|
* ms because the user needs immediate reaction from ondemand governor, but not
|
|
|
|
* sure if higher frequency will be required or not, then, the governor may
|
|
|
|
* change the sampling rate too late; up to 1 second later. Thus, if we are
|
|
|
|
* reducing the sampling rate, we need to make the new value effective
|
|
|
|
* immediately.
|
|
|
|
*
|
|
|
|
* This must be called with dbs_data->mutex held, otherwise traversing
|
|
|
|
* policy_dbs_list isn't safe.
|
|
|
|
*/
|
2016-03-22 01:47:51 +00:00
|
|
|
ssize_t store_sampling_rate(struct gov_attr_set *attr_set, const char *buf,
|
2016-02-11 12:01:15 +00:00
|
|
|
size_t count)
|
|
|
|
{
|
2016-03-22 01:47:51 +00:00
|
|
|
struct dbs_data *dbs_data = to_dbs_data(attr_set);
|
2016-02-11 12:01:15 +00:00
|
|
|
struct policy_dbs_info *policy_dbs;
|
2017-12-18 01:15:32 +00:00
|
|
|
unsigned int sampling_interval;
|
2016-02-11 12:01:15 +00:00
|
|
|
int ret;
|
2017-12-18 01:15:32 +00:00
|
|
|
|
|
|
|
ret = sscanf(buf, "%u", &sampling_interval);
|
|
|
|
if (ret != 1 || sampling_interval < CPUFREQ_DBS_MIN_SAMPLING_INTERVAL)
|
2016-02-11 12:01:15 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
2017-12-18 01:15:32 +00:00
|
|
|
dbs_data->sampling_rate = sampling_interval;
|
|
|
|
|
2016-02-11 12:01:15 +00:00
|
|
|
/*
|
|
|
|
* We are operating under dbs_data->mutex and so the list and its
|
|
|
|
* entries can't be freed concurrently.
|
|
|
|
*/
|
2016-03-22 01:47:51 +00:00
|
|
|
list_for_each_entry(policy_dbs, &attr_set->policy_list, list) {
|
2016-11-08 05:36:33 +00:00
|
|
|
mutex_lock(&policy_dbs->update_mutex);
|
2016-02-11 12:01:15 +00:00
|
|
|
/*
|
|
|
|
* On 32-bit architectures this may race with the
|
|
|
|
* sample_delay_ns read in dbs_update_util_handler(), but that
|
|
|
|
* really doesn't matter. If the read returns a value that's
|
|
|
|
* too big, the sample will be skipped, but the next invocation
|
|
|
|
* of dbs_update_util_handler() (when the update has been
|
2016-02-15 01:20:11 +00:00
|
|
|
* completed) will take a sample.
|
2016-02-11 12:01:15 +00:00
|
|
|
*
|
|
|
|
* If this runs in parallel with dbs_work_handler(), we may end
|
|
|
|
* up overwriting the sample_delay_ns value that it has just
|
2016-02-15 01:20:11 +00:00
|
|
|
* written, but it will be corrected next time a sample is
|
|
|
|
* taken, so it shouldn't be significant.
|
2016-02-11 12:01:15 +00:00
|
|
|
*/
|
2016-02-15 01:20:11 +00:00
|
|
|
gov_update_sample_delay(policy_dbs, 0);
|
2016-11-08 05:36:33 +00:00
|
|
|
mutex_unlock(&policy_dbs->update_mutex);
|
2016-02-11 12:01:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(store_sampling_rate);
|
|
|
|
|
2016-02-18 01:26:55 +00:00
|
|
|
/**
|
|
|
|
* gov_update_cpu_data - Update CPU load data.
|
|
|
|
* @dbs_data: Top-level governor data pointer.
|
|
|
|
*
|
|
|
|
* Update CPU load data for all CPUs in the domain governed by @dbs_data
|
|
|
|
* (that may be a single policy or a bunch of them if governor tunables are
|
|
|
|
* system-wide).
|
|
|
|
*
|
|
|
|
* Call under the @dbs_data mutex.
|
|
|
|
*/
|
2016-02-20 23:51:27 +00:00
|
|
|
void gov_update_cpu_data(struct dbs_data *dbs_data)
|
2016-02-18 01:26:55 +00:00
|
|
|
{
|
|
|
|
struct policy_dbs_info *policy_dbs;
|
|
|
|
|
2016-03-22 01:47:51 +00:00
|
|
|
list_for_each_entry(policy_dbs, &dbs_data->attr_set.policy_list, list) {
|
2016-02-18 01:26:55 +00:00
|
|
|
unsigned int j;
|
|
|
|
|
|
|
|
for_each_cpu(j, policy_dbs->policy->cpus) {
|
2016-02-20 23:51:27 +00:00
|
|
|
struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
|
2016-02-18 01:26:55 +00:00
|
|
|
|
2016-04-27 23:19:03 +00:00
|
|
|
j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_update_time,
|
2016-02-18 01:26:55 +00:00
|
|
|
dbs_data->io_is_busy);
|
|
|
|
if (dbs_data->ignore_nice_load)
|
|
|
|
j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(gov_update_cpu_data);
|
|
|
|
|
2016-02-15 01:19:31 +00:00
|
|
|
unsigned int dbs_update(struct cpufreq_policy *policy)
|
2012-10-25 22:47:42 +00:00
|
|
|
{
|
2016-02-07 15:24:26 +00:00
|
|
|
struct policy_dbs_info *policy_dbs = policy->governor_data;
|
|
|
|
struct dbs_data *dbs_data = policy_dbs->dbs_data;
|
2016-02-09 03:31:32 +00:00
|
|
|
unsigned int ignore_nice = dbs_data->ignore_nice_load;
|
2016-11-16 17:26:29 +00:00
|
|
|
unsigned int max_load = 0, idle_periods = UINT_MAX;
|
2016-02-18 01:20:13 +00:00
|
|
|
unsigned int sampling_rate, io_busy, j;
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2016-02-15 01:20:51 +00:00
|
|
|
/*
|
|
|
|
* Sometimes governors may use an additional multiplier to increase
|
|
|
|
* sample delays temporarily. Apply that multiplier to sampling_rate
|
|
|
|
* so as to keep the wake-up-from-idle detection logic a bit
|
|
|
|
* conservative.
|
|
|
|
*/
|
|
|
|
sampling_rate = dbs_data->sampling_rate * policy_dbs->rate_mult;
|
2016-02-18 01:20:13 +00:00
|
|
|
/*
|
|
|
|
* For the purpose of ondemand, waiting for disk IO is an indication
|
|
|
|
* that you're performance critical, and not that the system is actually
|
|
|
|
* idle, so do not add the iowait time to the CPU idle time then.
|
|
|
|
*/
|
|
|
|
io_busy = dbs_data->io_is_busy;
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2013-06-05 16:01:25 +00:00
|
|
|
/* Get Absolute Load */
|
2012-10-25 22:47:42 +00:00
|
|
|
for_each_cpu(j, policy->cpus) {
|
2016-02-20 23:51:27 +00:00
|
|
|
struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
|
2016-04-27 23:19:03 +00:00
|
|
|
u64 update_time, cur_idle_time;
|
|
|
|
unsigned int idle_time, time_elapsed;
|
2012-10-25 22:47:42 +00:00
|
|
|
unsigned int load;
|
|
|
|
|
2016-04-27 23:19:03 +00:00
|
|
|
cur_idle_time = get_cpu_idle_time(j, &update_time, io_busy);
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2016-04-27 23:19:03 +00:00
|
|
|
time_elapsed = update_time - j_cdbs->prev_update_time;
|
|
|
|
j_cdbs->prev_update_time = update_time;
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2016-04-21 18:57:47 +00:00
|
|
|
idle_time = cur_idle_time - j_cdbs->prev_cpu_idle;
|
|
|
|
j_cdbs->prev_cpu_idle = cur_idle_time;
|
2012-10-25 22:47:42 +00:00
|
|
|
|
|
|
|
if (ignore_nice) {
|
2016-02-15 01:15:50 +00:00
|
|
|
u64 cur_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];
|
|
|
|
|
2017-01-31 03:09:19 +00:00
|
|
|
idle_time += div_u64(cur_nice - j_cdbs->prev_cpu_nice, NSEC_PER_USEC);
|
2016-02-15 01:15:50 +00:00
|
|
|
j_cdbs->prev_cpu_nice = cur_nice;
|
2012-10-25 22:47:42 +00:00
|
|
|
}
|
|
|
|
|
2016-05-05 23:30:37 +00:00
|
|
|
if (unlikely(!time_elapsed)) {
|
|
|
|
/*
|
|
|
|
* That can only happen when this function is called
|
|
|
|
* twice in a row with a very short interval between the
|
|
|
|
* calls, so the previous load value can be used then.
|
|
|
|
*/
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 20:41:43 +00:00
|
|
|
load = j_cdbs->prev_load;
|
2016-05-05 23:30:37 +00:00
|
|
|
} else if (unlikely(time_elapsed > 2 * sampling_rate &&
|
|
|
|
j_cdbs->prev_load)) {
|
2014-06-09 08:51:24 +00:00
|
|
|
/*
|
2016-05-05 23:30:37 +00:00
|
|
|
* If the CPU had gone completely idle and a task has
|
|
|
|
* just woken up on this CPU now, it would be unfair to
|
|
|
|
* calculate 'load' the usual way for this elapsed
|
|
|
|
* time-window, because it would show near-zero load,
|
|
|
|
* irrespective of how CPU intensive that task actually
|
|
|
|
* was. This is undesirable for latency-sensitive bursty
|
|
|
|
* workloads.
|
|
|
|
*
|
|
|
|
* To avoid this, reuse the 'load' from the previous
|
|
|
|
* time-window and give this task a chance to start with
|
|
|
|
* a reasonably high CPU frequency. However, that
|
|
|
|
* shouldn't be over-done, lest we get stuck at a high
|
|
|
|
* load (high frequency) for too long, even when the
|
|
|
|
* current system load has actually dropped down, so
|
|
|
|
* clear prev_load to guarantee that the load will be
|
|
|
|
* computed again next time.
|
|
|
|
*
|
|
|
|
* Detecting this situation is easy: the governor's
|
|
|
|
* utilization update handler would not have run during
|
|
|
|
* CPU-idle periods. Hence, an unusually large
|
|
|
|
* 'time_elapsed' (as compared to the sampling rate)
|
|
|
|
* indicates this scenario.
|
2014-06-09 08:51:24 +00:00
|
|
|
*/
|
2016-05-05 23:30:37 +00:00
|
|
|
load = j_cdbs->prev_load;
|
2014-06-09 08:51:24 +00:00
|
|
|
j_cdbs->prev_load = 0;
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 20:41:43 +00:00
|
|
|
} else {
|
2016-05-05 23:30:37 +00:00
|
|
|
if (time_elapsed >= idle_time) {
|
|
|
|
load = 100 * (time_elapsed - idle_time) / time_elapsed;
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* That can happen if idle_time is returned by
|
|
|
|
* get_cpu_idle_time_jiffy(). In that case
|
|
|
|
* idle_time is roughly equal to the difference
|
|
|
|
* between time_elapsed and "busy time" obtained
|
|
|
|
* from CPU statistics. Then, the "busy time"
|
|
|
|
* can end up being greater than time_elapsed
|
|
|
|
* (for example, if jiffies_64 and the CPU
|
|
|
|
* statistics are updated by different CPUs),
|
|
|
|
* so idle_time may in fact be negative. That
|
|
|
|
* means, though, that the CPU was busy all
|
|
|
|
* the time (on the rough average) during the
|
|
|
|
* last sampling interval and 100 can be
|
|
|
|
* returned as the load.
|
|
|
|
*/
|
|
|
|
load = (int)idle_time < 0 ? 100 : 0;
|
|
|
|
}
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 20:41:43 +00:00
|
|
|
j_cdbs->prev_load = load;
|
|
|
|
}
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2016-11-16 17:26:29 +00:00
|
|
|
if (time_elapsed > 2 * sampling_rate) {
|
|
|
|
unsigned int periods = time_elapsed / sampling_rate;
|
|
|
|
|
|
|
|
if (periods < idle_periods)
|
|
|
|
idle_periods = periods;
|
|
|
|
}
|
|
|
|
|
2012-10-25 22:47:42 +00:00
|
|
|
if (load > max_load)
|
|
|
|
max_load = load;
|
|
|
|
}
|
2016-11-16 17:26:29 +00:00
|
|
|
|
|
|
|
policy_dbs->idle_periods = idle_periods;
|
|
|
|
|
2016-02-15 01:19:31 +00:00
|
|
|
return max_load;
|
2012-10-25 22:47:42 +00:00
|
|
|
}
|
2016-02-15 01:19:31 +00:00
|
|
|
EXPORT_SYMBOL_GPL(dbs_update);
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2015-12-09 02:04:42 +00:00
|
|
|
static void dbs_work_handler(struct work_struct *work)
|
2015-07-18 06:01:00 +00:00
|
|
|
{
|
2016-02-10 16:07:44 +00:00
|
|
|
struct policy_dbs_info *policy_dbs;
|
2015-10-29 02:38:38 +00:00
|
|
|
struct cpufreq_policy *policy;
|
2016-02-07 15:09:51 +00:00
|
|
|
struct dbs_governor *gov;
|
2015-07-18 06:01:00 +00:00
|
|
|
|
2016-02-10 16:07:44 +00:00
|
|
|
policy_dbs = container_of(work, struct policy_dbs_info, work);
|
|
|
|
policy = policy_dbs->policy;
|
2016-02-07 15:09:51 +00:00
|
|
|
gov = dbs_governor_of(policy);
|
2015-10-29 02:38:38 +00:00
|
|
|
|
2015-12-09 02:04:42 +00:00
|
|
|
/*
|
2016-02-10 15:53:50 +00:00
|
|
|
* Make sure cpufreq_governor_limits() isn't evaluating load or the
|
|
|
|
* ondemand governor isn't updating the sampling rate in parallel.
|
2015-12-09 02:04:42 +00:00
|
|
|
*/
|
2016-11-08 05:36:33 +00:00
|
|
|
mutex_lock(&policy_dbs->update_mutex);
|
|
|
|
gov_update_sample_delay(policy_dbs, gov->gov_dbs_update(policy));
|
|
|
|
mutex_unlock(&policy_dbs->update_mutex);
|
2015-12-09 02:04:42 +00:00
|
|
|
|
2016-02-15 01:13:42 +00:00
|
|
|
/* Allow the utilization update handler to queue up more work. */
|
|
|
|
atomic_set(&policy_dbs->work_count, 0);
|
2016-02-10 15:53:50 +00:00
|
|
|
/*
|
2016-02-15 01:13:42 +00:00
|
|
|
* If the update below is reordered with respect to the sample delay
|
|
|
|
* modification, the utilization update handler may end up using a stale
|
|
|
|
* sample delay value.
|
2016-02-10 15:53:50 +00:00
|
|
|
*/
|
2016-02-15 01:13:42 +00:00
|
|
|
smp_wmb();
|
|
|
|
policy_dbs->work_in_progress = false;
|
2016-02-10 15:53:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void dbs_irq_work(struct irq_work *irq_work)
|
|
|
|
{
|
2016-02-10 16:07:44 +00:00
|
|
|
struct policy_dbs_info *policy_dbs;
|
2015-12-09 02:04:42 +00:00
|
|
|
|
2016-02-10 16:07:44 +00:00
|
|
|
policy_dbs = container_of(irq_work, struct policy_dbs_info, irq_work);
|
2016-03-22 00:17:43 +00:00
|
|
|
schedule_work_on(smp_processor_id(), &policy_dbs->work);
|
2015-12-09 02:04:42 +00:00
|
|
|
}
|
|
|
|
|
2016-02-10 15:53:50 +00:00
|
|
|
static void dbs_update_util_handler(struct update_util_data *data, u64 time,
|
2016-08-16 20:14:55 +00:00
|
|
|
unsigned int flags)
|
2016-02-10 15:53:50 +00:00
|
|
|
{
|
|
|
|
struct cpu_dbs_info *cdbs = container_of(data, struct cpu_dbs_info, update_util);
|
2016-02-10 16:07:44 +00:00
|
|
|
struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;
|
2016-02-22 13:14:34 +00:00
|
|
|
u64 delta_ns, lst;
|
2015-12-09 02:04:42 +00:00
|
|
|
|
sched: cpufreq: Allow remote cpufreq callbacks
With Android UI and benchmarks the latency of cpufreq response to
certain scheduling events can become very critical. Currently, callbacks
into cpufreq governors are only made from the scheduler if the target
CPU of the event is the same as the current CPU. This means there are
certain situations where a target CPU may not run the cpufreq governor
for some time.
One testcase to show this behavior is where a task starts running on
CPU0, then a new task is also spawned on CPU0 by a task on CPU1. If the
system is configured such that the new tasks should receive maximum
demand initially, this should result in CPU0 increasing frequency
immediately. But because of the above mentioned limitation though, this
does not occur.
This patch updates the scheduler core to call the cpufreq callbacks for
remote CPUs as well.
The schedutil, ondemand and conservative governors are updated to
process cpufreq utilization update hooks called for remote CPUs where
the remote CPU is managed by the cpufreq policy of the local CPU.
The intel_pstate driver is updated to always reject remote callbacks.
This is tested with couple of usecases (Android: hackbench, recentfling,
galleryfling, vellamo, Ubuntu: hackbench) on ARM hikey board (64 bit
octa-core, single policy). Only galleryfling showed minor improvements,
while others didn't had much deviation.
The reason being that this patch only targets a corner case, where
following are required to be true to improve performance and that
doesn't happen too often with these tests:
- Task is migrated to another CPU.
- The task has high demand, and should take the target CPU to higher
OPPs.
- And the target CPU doesn't call into the cpufreq governor until the
next tick.
Based on initial work from Steve Muckle.
Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org>
Acked-by: Saravana Kannan <skannan@codeaurora.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2017-07-28 06:46:38 +00:00
|
|
|
if (!cpufreq_can_do_remote_dvfs(policy_dbs->policy))
|
|
|
|
return;
|
|
|
|
|
2015-12-09 02:04:42 +00:00
|
|
|
/*
|
2016-02-10 15:53:50 +00:00
|
|
|
* The work may not be allowed to be queued up right now.
|
|
|
|
* Possible reasons:
|
|
|
|
* - Work has already been queued up or is in progress.
|
|
|
|
* - It is too early (too little time from the previous sample).
|
2015-12-09 02:04:42 +00:00
|
|
|
*/
|
2016-02-15 01:13:42 +00:00
|
|
|
if (policy_dbs->work_in_progress)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the reads below are reordered before the check above, the value
|
|
|
|
* of sample_delay_ns used in the computation may be stale.
|
|
|
|
*/
|
|
|
|
smp_rmb();
|
2016-02-22 13:14:34 +00:00
|
|
|
lst = READ_ONCE(policy_dbs->last_sample_time);
|
|
|
|
delta_ns = time - lst;
|
2016-02-15 01:13:42 +00:00
|
|
|
if ((s64)delta_ns < policy_dbs->sample_delay_ns)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the policy is not shared, the irq_work may be queued up right away
|
|
|
|
* at this point. Otherwise, we need to ensure that only one of the
|
|
|
|
* CPUs sharing the policy will do that.
|
|
|
|
*/
|
2016-02-22 13:14:34 +00:00
|
|
|
if (policy_dbs->is_shared) {
|
|
|
|
if (!atomic_add_unless(&policy_dbs->work_count, 1, 1))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If another CPU updated last_sample_time in the meantime, we
|
|
|
|
* shouldn't be here, so clear the work counter and bail out.
|
|
|
|
*/
|
|
|
|
if (unlikely(lst != READ_ONCE(policy_dbs->last_sample_time))) {
|
|
|
|
atomic_set(&policy_dbs->work_count, 0);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
2016-02-15 01:13:42 +00:00
|
|
|
|
|
|
|
policy_dbs->last_sample_time = time;
|
|
|
|
policy_dbs->work_in_progress = true;
|
|
|
|
irq_work_queue(&policy_dbs->irq_work);
|
2015-07-18 06:01:00 +00:00
|
|
|
}
|
2013-01-31 17:28:02 +00:00
|
|
|
|
2016-04-01 23:08:43 +00:00
|
|
|
static void gov_set_update_util(struct policy_dbs_info *policy_dbs,
|
|
|
|
unsigned int delay_us)
|
|
|
|
{
|
|
|
|
struct cpufreq_policy *policy = policy_dbs->policy;
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
gov_update_sample_delay(policy_dbs, delay_us);
|
|
|
|
policy_dbs->last_sample_time = 0;
|
|
|
|
|
|
|
|
for_each_cpu(cpu, policy->cpus) {
|
|
|
|
struct cpu_dbs_info *cdbs = &per_cpu(cpu_dbs, cpu);
|
|
|
|
|
|
|
|
cpufreq_add_update_util_hook(cpu, &cdbs->update_util,
|
|
|
|
dbs_update_util_handler);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void gov_clear_update_util(struct cpufreq_policy *policy)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_cpu(i, policy->cpus)
|
|
|
|
cpufreq_remove_update_util_hook(i);
|
|
|
|
|
|
|
|
synchronize_sched();
|
|
|
|
}
|
|
|
|
|
2016-02-07 15:24:26 +00:00
|
|
|
static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *policy,
|
|
|
|
struct dbs_governor *gov)
|
2015-07-18 06:00:59 +00:00
|
|
|
{
|
2016-02-10 16:07:44 +00:00
|
|
|
struct policy_dbs_info *policy_dbs;
|
2015-07-18 06:00:59 +00:00
|
|
|
int j;
|
|
|
|
|
2016-02-18 17:40:14 +00:00
|
|
|
/* Allocate memory for per-policy governor data. */
|
|
|
|
policy_dbs = gov->alloc();
|
2016-02-10 16:07:44 +00:00
|
|
|
if (!policy_dbs)
|
2016-02-07 15:24:26 +00:00
|
|
|
return NULL;
|
2015-07-18 06:00:59 +00:00
|
|
|
|
2016-02-11 12:01:14 +00:00
|
|
|
policy_dbs->policy = policy;
|
2016-11-08 05:36:33 +00:00
|
|
|
mutex_init(&policy_dbs->update_mutex);
|
2016-02-08 22:41:10 +00:00
|
|
|
atomic_set(&policy_dbs->work_count, 0);
|
2016-02-10 16:07:44 +00:00
|
|
|
init_irq_work(&policy_dbs->irq_work, dbs_irq_work);
|
|
|
|
INIT_WORK(&policy_dbs->work, dbs_work_handler);
|
2016-02-07 15:25:02 +00:00
|
|
|
|
|
|
|
/* Set policy_dbs for all CPUs, online+offline */
|
|
|
|
for_each_cpu(j, policy->related_cpus) {
|
2016-02-20 23:51:27 +00:00
|
|
|
struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
|
2016-02-07 15:25:02 +00:00
|
|
|
|
|
|
|
j_cdbs->policy_dbs = policy_dbs;
|
|
|
|
}
|
2016-02-07 15:24:26 +00:00
|
|
|
return policy_dbs;
|
2015-07-18 06:00:59 +00:00
|
|
|
}
|
|
|
|
|
2016-02-20 23:51:27 +00:00
|
|
|
static void free_policy_dbs_info(struct policy_dbs_info *policy_dbs,
|
2016-02-07 15:05:07 +00:00
|
|
|
struct dbs_governor *gov)
|
2015-07-18 06:00:59 +00:00
|
|
|
{
|
|
|
|
int j;
|
|
|
|
|
2016-11-08 05:36:33 +00:00
|
|
|
mutex_destroy(&policy_dbs->update_mutex);
|
2015-12-03 04:07:52 +00:00
|
|
|
|
2016-02-20 23:51:27 +00:00
|
|
|
for_each_cpu(j, policy_dbs->policy->related_cpus) {
|
|
|
|
struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
|
2015-07-18 06:00:59 +00:00
|
|
|
|
2016-02-07 15:25:02 +00:00
|
|
|
j_cdbs->policy_dbs = NULL;
|
|
|
|
j_cdbs->update_util.func = NULL;
|
|
|
|
}
|
2016-02-18 17:40:14 +00:00
|
|
|
gov->free(policy_dbs);
|
2015-07-18 06:00:59 +00:00
|
|
|
}
|
|
|
|
|
2016-06-02 21:24:15 +00:00
|
|
|
int cpufreq_dbs_governor_init(struct cpufreq_policy *policy)
|
2012-10-25 22:47:42 +00:00
|
|
|
{
|
2016-02-07 15:09:51 +00:00
|
|
|
struct dbs_governor *gov = dbs_governor_of(policy);
|
2016-02-20 23:53:06 +00:00
|
|
|
struct dbs_data *dbs_data;
|
2016-02-07 15:24:26 +00:00
|
|
|
struct policy_dbs_info *policy_dbs;
|
2016-02-20 23:53:06 +00:00
|
|
|
int ret = 0;
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2015-07-18 06:01:01 +00:00
|
|
|
/* State should be equivalent to EXIT */
|
|
|
|
if (policy->governor_data)
|
|
|
|
return -EBUSY;
|
|
|
|
|
2016-02-07 15:24:26 +00:00
|
|
|
policy_dbs = alloc_policy_dbs_info(policy, gov);
|
|
|
|
if (!policy_dbs)
|
|
|
|
return -ENOMEM;
|
2015-07-18 06:00:59 +00:00
|
|
|
|
2016-02-20 23:53:06 +00:00
|
|
|
/* Protect gov->gdbs_data against concurrent updates. */
|
|
|
|
mutex_lock(&gov_dbs_data_mutex);
|
|
|
|
|
|
|
|
dbs_data = gov->gdbs_data;
|
2016-02-07 15:24:26 +00:00
|
|
|
if (dbs_data) {
|
|
|
|
if (WARN_ON(have_governor_per_policy())) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto free_policy_dbs_info;
|
|
|
|
}
|
|
|
|
policy_dbs->dbs_data = dbs_data;
|
|
|
|
policy->governor_data = policy_dbs;
|
2016-02-10 05:30:25 +00:00
|
|
|
|
2016-03-22 01:47:51 +00:00
|
|
|
gov_attr_set_get(&dbs_data->attr_set, &policy_dbs->list);
|
2016-02-20 23:53:06 +00:00
|
|
|
goto out;
|
2015-06-04 11:13:27 +00:00
|
|
|
}
|
2013-03-27 15:58:58 +00:00
|
|
|
|
2015-06-04 11:13:27 +00:00
|
|
|
dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL);
|
2016-02-07 15:24:26 +00:00
|
|
|
if (!dbs_data) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto free_policy_dbs_info;
|
|
|
|
}
|
2015-07-18 06:00:59 +00:00
|
|
|
|
2016-03-22 01:47:51 +00:00
|
|
|
gov_attr_set_init(&dbs_data->attr_set, &policy_dbs->list);
|
2013-03-27 15:58:58 +00:00
|
|
|
|
2016-05-18 20:59:49 +00:00
|
|
|
ret = gov->init(dbs_data);
|
2015-06-04 11:13:27 +00:00
|
|
|
if (ret)
|
2016-02-10 16:07:44 +00:00
|
|
|
goto free_policy_dbs_info;
|
2013-03-27 15:58:58 +00:00
|
|
|
|
2017-12-18 01:15:32 +00:00
|
|
|
/*
|
|
|
|
* The sampling interval should not be less than the transition latency
|
|
|
|
* of the CPU and it also cannot be too small for dbs_update() to work
|
|
|
|
* correctly.
|
|
|
|
*/
|
|
|
|
dbs_data->sampling_rate = max_t(unsigned int,
|
|
|
|
CPUFREQ_DBS_MIN_SAMPLING_INTERVAL,
|
|
|
|
cpufreq_policy_transition_delay_us(policy));
|
2013-05-17 10:39:09 +00:00
|
|
|
|
2015-10-15 16:05:22 +00:00
|
|
|
if (!have_governor_per_policy())
|
2016-02-07 15:05:07 +00:00
|
|
|
gov->gdbs_data = dbs_data;
|
2013-03-27 15:58:58 +00:00
|
|
|
|
2016-02-10 05:30:25 +00:00
|
|
|
policy_dbs->dbs_data = dbs_data;
|
2016-03-22 01:47:51 +00:00
|
|
|
policy->governor_data = policy_dbs;
|
2016-02-10 05:30:25 +00:00
|
|
|
|
cpufreq: governor: New sysfs show/store callbacks for governor tunables
The ondemand and conservative governors use the global-attr or freq-attr
structures to represent sysfs attributes corresponding to their tunables
(which of them is actually used depends on whether or not different
policy objects can use the same governor with different tunables at the
same time and, consequently, on where those attributes are located in
sysfs).
Unfortunately, in the freq-attr case, the standard cpufreq show/store
sysfs attribute callbacks are applied to the governor tunable attributes
and they always acquire the policy->rwsem lock before carrying out the
operation. That may lead to an ABBA deadlock if governor tunable
attributes are removed under policy->rwsem while one of them is being
accessed concurrently (if sysfs attributes removal wins the race, it
will wait for the access to complete with policy->rwsem held while the
attribute callback will block on policy->rwsem indefinitely).
We attempted to address this issue by dropping policy->rwsem around
governor tunable attributes removal (that is, around invocations of the
->governor callback with the event arg equal to CPUFREQ_GOV_POLICY_EXIT)
in cpufreq_set_policy(), but that opened up race conditions that had not
been possible with policy->rwsem held all the time. Therefore
policy->rwsem cannot be dropped in cpufreq_set_policy() at any point,
but the deadlock situation described above must be avoided too.
To that end, use the observation that in principle governor tunables may
be represented by the same data type regardless of whether the governor
is system-wide or per-policy and introduce a new structure, struct
governor_attr, for representing them and new corresponding macros for
creating show/store sysfs callbacks for them. Also make their parent
kobject use a new kobject type whose default show/store callbacks are
not related to the standard core cpufreq ones in any way (and they don't
acquire policy->rwsem in particular).
Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org>
Tested-by: Juri Lelli <juri.lelli@arm.com>
Tested-by: Shilpasri G Bhat <shilpa.bhat@linux.vnet.ibm.com>
[ rjw: Subject & changelog + rebase ]
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2016-02-09 03:31:33 +00:00
|
|
|
gov->kobj_type.sysfs_ops = &governor_sysfs_ops;
|
2016-03-22 01:47:51 +00:00
|
|
|
ret = kobject_init_and_add(&dbs_data->attr_set.kobj, &gov->kobj_type,
|
cpufreq: governor: New sysfs show/store callbacks for governor tunables
The ondemand and conservative governors use the global-attr or freq-attr
structures to represent sysfs attributes corresponding to their tunables
(which of them is actually used depends on whether or not different
policy objects can use the same governor with different tunables at the
same time and, consequently, on where those attributes are located in
sysfs).
Unfortunately, in the freq-attr case, the standard cpufreq show/store
sysfs attribute callbacks are applied to the governor tunable attributes
and they always acquire the policy->rwsem lock before carrying out the
operation. That may lead to an ABBA deadlock if governor tunable
attributes are removed under policy->rwsem while one of them is being
accessed concurrently (if sysfs attributes removal wins the race, it
will wait for the access to complete with policy->rwsem held while the
attribute callback will block on policy->rwsem indefinitely).
We attempted to address this issue by dropping policy->rwsem around
governor tunable attributes removal (that is, around invocations of the
->governor callback with the event arg equal to CPUFREQ_GOV_POLICY_EXIT)
in cpufreq_set_policy(), but that opened up race conditions that had not
been possible with policy->rwsem held all the time. Therefore
policy->rwsem cannot be dropped in cpufreq_set_policy() at any point,
but the deadlock situation described above must be avoided too.
To that end, use the observation that in principle governor tunables may
be represented by the same data type regardless of whether the governor
is system-wide or per-policy and introduce a new structure, struct
governor_attr, for representing them and new corresponding macros for
creating show/store sysfs callbacks for them. Also make their parent
kobject use a new kobject type whose default show/store callbacks are
not related to the standard core cpufreq ones in any way (and they don't
acquire policy->rwsem in particular).
Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org>
Tested-by: Juri Lelli <juri.lelli@arm.com>
Tested-by: Shilpasri G Bhat <shilpa.bhat@linux.vnet.ibm.com>
[ rjw: Subject & changelog + rebase ]
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2016-02-09 03:31:33 +00:00
|
|
|
get_governor_parent_kobj(policy),
|
|
|
|
"%s", gov->gov.name);
|
2016-02-08 22:57:22 +00:00
|
|
|
if (!ret)
|
2016-02-20 23:53:06 +00:00
|
|
|
goto out;
|
2013-03-27 15:58:58 +00:00
|
|
|
|
2016-02-08 22:57:22 +00:00
|
|
|
/* Failure, so roll back. */
|
2016-05-18 12:25:27 +00:00
|
|
|
pr_err("initialization failed (dbs_data kobject init error %d)\n", ret);
|
2013-03-27 15:58:58 +00:00
|
|
|
|
2016-01-25 17:03:46 +00:00
|
|
|
policy->governor_data = NULL;
|
|
|
|
|
2015-10-15 16:05:22 +00:00
|
|
|
if (!have_governor_per_policy())
|
2016-02-07 15:05:07 +00:00
|
|
|
gov->gdbs_data = NULL;
|
2016-05-18 20:59:49 +00:00
|
|
|
gov->exit(dbs_data);
|
2016-02-07 15:24:26 +00:00
|
|
|
kfree(dbs_data);
|
|
|
|
|
2016-02-10 16:07:44 +00:00
|
|
|
free_policy_dbs_info:
|
2016-02-20 23:51:27 +00:00
|
|
|
free_policy_dbs_info(policy_dbs, gov);
|
2016-02-20 23:53:06 +00:00
|
|
|
|
|
|
|
out:
|
|
|
|
mutex_unlock(&gov_dbs_data_mutex);
|
2015-06-04 11:13:27 +00:00
|
|
|
return ret;
|
|
|
|
}
|
2016-06-02 21:24:15 +00:00
|
|
|
EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_init);
|
2013-03-27 15:58:58 +00:00
|
|
|
|
2016-06-02 21:24:15 +00:00
|
|
|
void cpufreq_dbs_governor_exit(struct cpufreq_policy *policy)
|
2015-06-04 11:13:27 +00:00
|
|
|
{
|
2016-02-07 15:09:51 +00:00
|
|
|
struct dbs_governor *gov = dbs_governor_of(policy);
|
2016-02-07 15:24:26 +00:00
|
|
|
struct policy_dbs_info *policy_dbs = policy->governor_data;
|
|
|
|
struct dbs_data *dbs_data = policy_dbs->dbs_data;
|
2016-03-22 01:47:51 +00:00
|
|
|
unsigned int count;
|
2015-07-18 06:01:01 +00:00
|
|
|
|
2016-02-20 23:53:06 +00:00
|
|
|
/* Protect gov->gdbs_data against concurrent updates. */
|
|
|
|
mutex_lock(&gov_dbs_data_mutex);
|
|
|
|
|
2016-03-22 01:47:51 +00:00
|
|
|
count = gov_attr_set_put(&dbs_data->attr_set, &policy_dbs->list);
|
2013-05-17 10:39:09 +00:00
|
|
|
|
2016-03-22 01:47:51 +00:00
|
|
|
policy->governor_data = NULL;
|
2016-01-25 17:03:46 +00:00
|
|
|
|
2016-03-22 01:47:51 +00:00
|
|
|
if (!count) {
|
2015-10-15 16:05:22 +00:00
|
|
|
if (!have_governor_per_policy())
|
2016-02-07 15:05:07 +00:00
|
|
|
gov->gdbs_data = NULL;
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2016-05-18 20:59:49 +00:00
|
|
|
gov->exit(dbs_data);
|
2015-06-04 11:13:27 +00:00
|
|
|
kfree(dbs_data);
|
2013-03-27 15:58:58 +00:00
|
|
|
}
|
2015-07-18 06:00:59 +00:00
|
|
|
|
2016-02-20 23:51:27 +00:00
|
|
|
free_policy_dbs_info(policy_dbs, gov);
|
2016-02-20 23:53:06 +00:00
|
|
|
|
|
|
|
mutex_unlock(&gov_dbs_data_mutex);
|
2015-06-04 11:13:27 +00:00
|
|
|
}
|
2016-06-02 21:24:15 +00:00
|
|
|
EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_exit);
|
2013-03-27 15:58:58 +00:00
|
|
|
|
2016-06-02 21:24:15 +00:00
|
|
|
int cpufreq_dbs_governor_start(struct cpufreq_policy *policy)
|
2015-06-04 11:13:27 +00:00
|
|
|
{
|
2016-02-07 15:09:51 +00:00
|
|
|
struct dbs_governor *gov = dbs_governor_of(policy);
|
2016-02-07 15:24:26 +00:00
|
|
|
struct policy_dbs_info *policy_dbs = policy->governor_data;
|
|
|
|
struct dbs_data *dbs_data = policy_dbs->dbs_data;
|
2016-02-18 01:21:21 +00:00
|
|
|
unsigned int sampling_rate, ignore_nice, j;
|
2016-02-18 01:20:13 +00:00
|
|
|
unsigned int io_busy;
|
2015-06-04 11:13:27 +00:00
|
|
|
|
|
|
|
if (!policy->cur)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2016-02-15 01:13:42 +00:00
|
|
|
policy_dbs->is_shared = policy_is_shared(policy);
|
2016-02-15 01:20:51 +00:00
|
|
|
policy_dbs->rate_mult = 1;
|
2016-02-15 01:13:42 +00:00
|
|
|
|
2016-02-09 03:31:32 +00:00
|
|
|
sampling_rate = dbs_data->sampling_rate;
|
|
|
|
ignore_nice = dbs_data->ignore_nice_load;
|
2016-02-18 01:20:13 +00:00
|
|
|
io_busy = dbs_data->io_is_busy;
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2015-06-04 11:13:27 +00:00
|
|
|
for_each_cpu(j, policy->cpus) {
|
2016-02-20 23:51:27 +00:00
|
|
|
struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2016-04-27 23:19:03 +00:00
|
|
|
j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_update_time, io_busy);
|
2016-04-25 14:21:34 +00:00
|
|
|
/*
|
|
|
|
* Make the first invocation of dbs_update() compute the load.
|
|
|
|
*/
|
|
|
|
j_cdbs->prev_load = 0;
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 20:41:43 +00:00
|
|
|
|
2015-06-04 11:13:27 +00:00
|
|
|
if (ignore_nice)
|
|
|
|
j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];
|
|
|
|
}
|
2012-12-27 14:55:38 +00:00
|
|
|
|
2016-02-18 01:21:21 +00:00
|
|
|
gov->start(policy);
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2016-02-10 16:07:44 +00:00
|
|
|
gov_set_update_util(policy_dbs, sampling_rate);
|
2015-06-04 11:13:27 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2016-06-02 21:24:15 +00:00
|
|
|
EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_start);
|
2015-06-04 11:13:27 +00:00
|
|
|
|
2016-06-02 21:24:15 +00:00
|
|
|
void cpufreq_dbs_governor_stop(struct cpufreq_policy *policy)
|
2015-06-04 11:13:27 +00:00
|
|
|
{
|
2016-06-08 23:45:32 +00:00
|
|
|
struct policy_dbs_info *policy_dbs = policy->governor_data;
|
|
|
|
|
|
|
|
gov_clear_update_util(policy_dbs->policy);
|
|
|
|
irq_work_sync(&policy_dbs->irq_work);
|
|
|
|
cancel_work_sync(&policy_dbs->work);
|
|
|
|
atomic_set(&policy_dbs->work_count, 0);
|
|
|
|
policy_dbs->work_in_progress = false;
|
2015-06-04 11:13:27 +00:00
|
|
|
}
|
2016-06-02 21:24:15 +00:00
|
|
|
EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_stop);
|
2012-10-25 22:47:42 +00:00
|
|
|
|
2016-06-02 21:24:15 +00:00
|
|
|
void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy)
|
2015-06-04 11:13:27 +00:00
|
|
|
{
|
2016-02-07 15:24:26 +00:00
|
|
|
struct policy_dbs_info *policy_dbs = policy->governor_data;
|
2013-01-31 17:28:01 +00:00
|
|
|
|
2016-11-08 05:36:33 +00:00
|
|
|
mutex_lock(&policy_dbs->update_mutex);
|
2016-05-18 12:25:31 +00:00
|
|
|
cpufreq_policy_apply_limits(policy);
|
2016-02-15 01:19:31 +00:00
|
|
|
gov_update_sample_delay(policy_dbs, 0);
|
|
|
|
|
2016-11-08 05:36:33 +00:00
|
|
|
mutex_unlock(&policy_dbs->update_mutex);
|
2012-10-25 22:47:42 +00:00
|
|
|
}
|
2016-06-02 21:24:15 +00:00
|
|
|
EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_limits);
|