leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

srcu: Add contention check to call_srcu() srcu_data ->lock acquisition

Browse Source 
This commit increases the sensitivity of contention detection by adding
checks to the acquisition of the srcu_data structure's lock on the
call_srcu() code path.

Co-developed-by: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
Paul E. McKenney 2022-01-31 13:27:15 -08:00
parent a57ffb3c6b
commit c2445d3878
1 changed files with 36 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
kernel/rcu/srcutree.c
View File

@ -330,18 +330,13 @@ static void srcu_transition_to_big(struct srcu_struct *ssp)
} }
/* /*
* Acquire the specified srcu_struct structure's ->lock, but check for * Check to see if the just-encountered contention event justifies
* excessive contention, which results in initiation of a transition * a transition to SRCU_SIZE_BIG.
* to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module
* parameter permits this.
*/ */
static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags) static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp)
{ {
unsigned long j; unsigned long j;
if (spin_trylock_irqsave_rcu_node(ssp, *flags))
return;
spin_lock_irqsave_rcu_node(ssp, *flags);
if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_size_state) if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_size_state)
return; return;
j = jiffies; j = jiffies;
@ -354,6 +349,38 @@ static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned l
__srcu_transition_to_big(ssp); __srcu_transition_to_big(ssp);
} }
/*
* Acquire the specified srcu_data structure's ->lock, but check for
* excessive contention, which results in initiation of a transition
* to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module
* parameter permits this.
*/
static void spin_lock_irqsave_sdp_contention(struct srcu_data *sdp, unsigned long *flags)
{
struct srcu_struct *ssp = sdp->ssp;
if (spin_trylock_irqsave_rcu_node(sdp, *flags))
return;
spin_lock_irqsave_rcu_node(ssp, *flags);
spin_lock_irqsave_check_contention(ssp);
spin_unlock_irqrestore_rcu_node(ssp, *flags);
spin_lock_irqsave_rcu_node(sdp, *flags);
}
/*
* Acquire the specified srcu_struct structure's ->lock, but check for
* excessive contention, which results in initiation of a transition
* to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module
* parameter permits this.
*/
static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags)
{
if (spin_trylock_irqsave_rcu_node(ssp, *flags))
return;
spin_lock_irqsave_rcu_node(ssp, *flags);
spin_lock_irqsave_check_contention(ssp);
}
/* /*
* First-use initialization of statically allocated srcu_struct * First-use initialization of statically allocated srcu_struct
* structure. Wiring up the combining tree is more than can be * structure. Wiring up the combining tree is more than can be
@ -1010,7 +1037,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,
sdp = per_cpu_ptr(ssp->sda, 0); sdp = per_cpu_ptr(ssp->sda, 0);
else else
sdp = raw_cpu_ptr(ssp->sda); sdp = raw_cpu_ptr(ssp->sda);
spin_lock_irqsave_rcu_node(sdp, flags); spin_lock_irqsave_sdp_contention(sdp, &flags);
if (rhp) if (rhp)
rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp);
rcu_segcblist_advance(&sdp->srcu_cblist, rcu_segcblist_advance(&sdp->srcu_cblist,