Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-26 22:21:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

rcu: Drive expedited grace periods from workqueue

Browse Source 
The current implementation of expedited grace periods has the user
task drive the grace period.  This works, but has downsides: (1) The
user task must awaken tasks piggybacking on this grace period, which
can result in latencies rivaling that of the grace period itself, and
(2) User tasks can receive signals, which interfere with RCU CPU stall
warnings.

This commit therefore uses workqueues to drive the grace periods, so
that the user task need not do the awakening.  A subsequent commit
will remove the now-unnecessary code allowing for signals.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
Paul E. McKenney 2016-06-29 13:46:25 -07:00
parent f7b8eb847e
commit 8b355e3bc1
3 changed files with 46 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
kernel/rcu/tree.h
View File

@ -400,6 +400,7 @@ struct rcu_data {
#ifdef CONFIG_RCU_FAST_NO_HZ #ifdef CONFIG_RCU_FAST_NO_HZ
struct rcu_head oom_head; struct rcu_head oom_head;
#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
atomic_long_t exp_workdone0; /* # done by workqueue. */
atomic_long_t exp_workdone1; /* # done by others #1. */ atomic_long_t exp_workdone1; /* # done by others #1. */
atomic_long_t exp_workdone2; /* # done by others #2. */ atomic_long_t exp_workdone2; /* # done by others #2. */
atomic_long_t exp_workdone3; /* # done by others #3. */ atomic_long_t exp_workdone3; /* # done by others #3. */

46
kernel/rcu/tree_exp.h
View File

@ -500,7 +500,6 @@ static void rcu_exp_wait_wake(struct rcu_state *rsp, unsigned long s)
* next GP, to proceed. * next GP, to proceed.
*/ */
mutex_lock(&rsp->exp_wake_mutex); mutex_lock(&rsp->exp_wake_mutex);
mutex_unlock(&rsp->exp_mutex);
rcu_for_each_node_breadth_first(rsp, rnp) { rcu_for_each_node_breadth_first(rsp, rnp) {
if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) { if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) {
@ -516,6 +515,29 @@ static void rcu_exp_wait_wake(struct rcu_state *rsp, unsigned long s)
mutex_unlock(&rsp->exp_wake_mutex); mutex_unlock(&rsp->exp_wake_mutex);
} }
/* Let the workqueue handler know what it is supposed to do. */
struct rcu_exp_work {
smp_call_func_t rew_func;
struct rcu_state *rew_rsp;
unsigned long rew_s;
struct work_struct rew_work;
};
/*
* Work-queue handler to drive an expedited grace period forward.
*/
static void wait_rcu_exp_gp(struct work_struct *wp)
{
struct rcu_exp_work *rewp;
/* Initialize the rcu_node tree in preparation for the wait. */
rewp = container_of(wp, struct rcu_exp_work, rew_work);
sync_rcu_exp_select_cpus(rewp->rew_rsp, rewp->rew_func);
/* Wait and clean up, including waking everyone. */
rcu_exp_wait_wake(rewp->rew_rsp, rewp->rew_s);
}
/* /*
* Given an rcu_state pointer and a smp_call_function() handler, kick * Given an rcu_state pointer and a smp_call_function() handler, kick
* off the specified flavor of expedited grace period. * off the specified flavor of expedited grace period.
@ -523,6 +545,9 @@ static void rcu_exp_wait_wake(struct rcu_state *rsp, unsigned long s)
static void _synchronize_rcu_expedited(struct rcu_state *rsp, static void _synchronize_rcu_expedited(struct rcu_state *rsp,
smp_call_func_t func) smp_call_func_t func)
{ {
struct rcu_data *rdp;
struct rcu_exp_work rew;
struct rcu_node *rnp;
unsigned long s; unsigned long s;
/* If expedited grace periods are prohibited, fall back to normal. */ /* If expedited grace periods are prohibited, fall back to normal. */
@ -536,11 +561,22 @@ static void _synchronize_rcu_expedited(struct rcu_state *rsp,
if (exp_funnel_lock(rsp, s)) if (exp_funnel_lock(rsp, s))
return; /* Someone else did our work for us. */ return; /* Someone else did our work for us. */
/* Initialize the rcu_node tree in preparation for the wait. */ /* Marshall arguments and schedule the expedited grace period. */
sync_rcu_exp_select_cpus(rsp, func); rew.rew_func = func;
rew.rew_rsp = rsp;
rew.rew_s = s;
INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp);
schedule_work(&rew.rew_work);
/* Wait and clean up, including waking everyone. */ /* Wait for expedited grace period to complete. */
rcu_exp_wait_wake(rsp, s); rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id());
rnp = rcu_get_root(rsp);
wait_event(rnp->exp_wq[(s >> 1) & 0x3],
sync_exp_work_done(rsp,
&rdp->exp_workdone0, s));
/* Let the next expedited grace period start. */
mutex_unlock(&rsp->exp_mutex);
} }
/** /**

7
kernel/rcu/tree_trace.c
View File

@ -185,16 +185,17 @@ static int show_rcuexp(struct seq_file *m, void *v)
int cpu; int cpu;
struct rcu_state *rsp = (struct rcu_state *)m->private; struct rcu_state *rsp = (struct rcu_state *)m->private;
struct rcu_data *rdp; struct rcu_data *rdp;
unsigned long s1 = 0, s2 = 0, s3 = 0; unsigned long s0 = 0, s1 = 0, s2 = 0, s3 = 0;
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
rdp = per_cpu_ptr(rsp->rda, cpu); rdp = per_cpu_ptr(rsp->rda, cpu);
s0 += atomic_long_read(&rdp->exp_workdone0);
s1 += atomic_long_read(&rdp->exp_workdone1); s1 += atomic_long_read(&rdp->exp_workdone1);
s2 += atomic_long_read(&rdp->exp_workdone2); s2 += atomic_long_read(&rdp->exp_workdone2);
s3 += atomic_long_read(&rdp->exp_workdone3); s3 += atomic_long_read(&rdp->exp_workdone3);
} }
seq_printf(m, "s=%lu wd1=%lu wd2=%lu wd3=%lu n=%lu enq=%d sc=%lu\n", seq_printf(m, "s=%lu wd0=%lu wd1=%lu wd2=%lu wd3=%lu n=%lu enq=%d sc=%lu\n",
rsp->expedited_sequence, s1, s2, s3, rsp->expedited_sequence, s0, s1, s2, s3,
atomic_long_read(&rsp->expedited_normal), atomic_long_read(&rsp->expedited_normal),
atomic_read(&rsp->expedited_need_qs), atomic_read(&rsp->expedited_need_qs),
rsp->expedited_sequence / 2); rsp->expedited_sequence / 2);