mirror of
https://github.com/Fishwaldo/Star64_linux.git
synced 2025-07-23 23:32:14 +00:00
Merge branches 'fixes.2015.10.06a' and 'exp.2015.10.07a' into HEAD
exp.2015.10.07a: Reduce OS jitter of RCU-sched expedited grace periods. fixes.2015.10.06a: Miscellaneous fixes.
This commit is contained in:
Paul E. McKenney
commit
d2856b046d
8 changed files with 651 additions and 344 deletions
|
|
@ -71,7 +71,6 @@ MODULE_ALIAS("rcutree");
|
|||
static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
|
||||
static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
|
||||
static struct lock_class_key rcu_exp_class[RCU_NUM_LVLS];
|
||||
static struct lock_class_key rcu_exp_sched_class[RCU_NUM_LVLS];
|
||||
|
||||
/*
|
||||
* In order to export the rcu_state name to the tracing tools, it
|
||||
|
|
@ -161,6 +160,8 @@ static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);
|
|||
static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
|
||||
static void invoke_rcu_core(void);
|
||||
static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
|
||||
static void rcu_report_exp_rdp(struct rcu_state *rsp,
|
||||
struct rcu_data *rdp, bool wake);
|
||||
|
||||
/* rcuc/rcub kthread realtime priority */
|
||||
#ifdef CONFIG_RCU_KTHREAD_PRIO
|
||||
|
|
@ -245,21 +246,33 @@ static int rcu_gp_in_progress(struct rcu_state *rsp)
|
|||
*/
|
||||
void rcu_sched_qs(void)
|
||||
{
|
||||
if (!__this_cpu_read(rcu_sched_data.passed_quiesce)) {
|
||||
unsigned long flags;
|
||||
|
||||
if (__this_cpu_read(rcu_sched_data.cpu_no_qs.s)) {
|
||||
trace_rcu_grace_period(TPS("rcu_sched"),
|
||||
__this_cpu_read(rcu_sched_data.gpnum),
|
||||
TPS("cpuqs"));
|
||||
__this_cpu_write(rcu_sched_data.passed_quiesce, 1);
|
||||
__this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);
|
||||
if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
|
||||
return;
|
||||
local_irq_save(flags);
|
||||
if (__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp)) {
|
||||
__this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, false);
|
||||
rcu_report_exp_rdp(&rcu_sched_state,
|
||||
this_cpu_ptr(&rcu_sched_data),
|
||||
true);
|
||||
}
|
||||
local_irq_restore(flags);
|
||||
}
|
||||
}
|
||||
|
||||
void rcu_bh_qs(void)
|
||||
{
|
||||
if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) {
|
||||
if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {
|
||||
trace_rcu_grace_period(TPS("rcu_bh"),
|
||||
__this_cpu_read(rcu_bh_data.gpnum),
|
||||
TPS("cpuqs"));
|
||||
__this_cpu_write(rcu_bh_data.passed_quiesce, 1);
|
||||
__this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1753,9 +1766,9 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
|
|||
*/
|
||||
rdp->gpnum = rnp->gpnum;
|
||||
trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
|
||||
rdp->passed_quiesce = 0;
|
||||
rdp->cpu_no_qs.b.norm = true;
|
||||
rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
|
||||
rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
|
||||
rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);
|
||||
zero_cpu_stall_ticks(rdp);
|
||||
WRITE_ONCE(rdp->gpwrap, false);
|
||||
}
|
||||
|
|
@ -2344,7 +2357,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
|
|||
rnp = rdp->mynode;
|
||||
raw_spin_lock_irqsave(&rnp->lock, flags);
|
||||
smp_mb__after_unlock_lock();
|
||||
if ((rdp->passed_quiesce == 0 &&
|
||||
if ((rdp->cpu_no_qs.b.norm &&
|
||||
rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||
|
||||
rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
|
||||
rdp->gpwrap) {
|
||||
|
|
@ -2355,7 +2368,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
|
|||
* We will instead need a new quiescent state that lies
|
||||
* within the current grace period.
|
||||
*/
|
||||
rdp->passed_quiesce = 0; /* need qs for new gp. */
|
||||
rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */
|
||||
rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
return;
|
||||
|
|
@ -2364,7 +2377,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
|
|||
if ((rnp->qsmask & mask) == 0) {
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
} else {
|
||||
rdp->qs_pending = 0;
|
||||
rdp->core_needs_qs = 0;
|
||||
|
||||
/*
|
||||
* This GP can't end until cpu checks in, so all of our
|
||||
|
|
@ -2395,14 +2408,14 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
|
|||
* Does this CPU still need to do its part for current grace period?
|
||||
* If no, return and let the other CPUs do their part as well.
|
||||
*/
|
||||
if (!rdp->qs_pending)
|
||||
if (!rdp->core_needs_qs)
|
||||
return;
|
||||
|
||||
/*
|
||||
* Was there a quiescent state since the beginning of the grace
|
||||
* period? If no, then exit and wait for the next call.
|
||||
*/
|
||||
if (!rdp->passed_quiesce &&
|
||||
if (rdp->cpu_no_qs.b.norm &&
|
||||
rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))
|
||||
return;
|
||||
|
||||
|
|
@ -3386,6 +3399,191 @@ static bool rcu_exp_gp_seq_done(struct rcu_state *rsp, unsigned long s)
|
|||
return rcu_seq_done(&rsp->expedited_sequence, s);
|
||||
}
|
||||
|
||||
/*
|
||||
* Reset the ->expmaskinit values in the rcu_node tree to reflect any
|
||||
* recent CPU-online activity. Note that these masks are not cleared
|
||||
* when CPUs go offline, so they reflect the union of all CPUs that have
|
||||
* ever been online. This means that this function normally takes its
|
||||
* no-work-to-do fastpath.
|
||||
*/
|
||||
static void sync_exp_reset_tree_hotplug(struct rcu_state *rsp)
|
||||
{
|
||||
bool done;
|
||||
unsigned long flags;
|
||||
unsigned long mask;
|
||||
unsigned long oldmask;
|
||||
int ncpus = READ_ONCE(rsp->ncpus);
|
||||
struct rcu_node *rnp;
|
||||
struct rcu_node *rnp_up;
|
||||
|
||||
/* If no new CPUs onlined since last time, nothing to do. */
|
||||
if (likely(ncpus == rsp->ncpus_snap))
|
||||
return;
|
||||
rsp->ncpus_snap = ncpus;
|
||||
|
||||
/*
|
||||
* Each pass through the following loop propagates newly onlined
|
||||
* CPUs for the current rcu_node structure up the rcu_node tree.
|
||||
*/
|
||||
rcu_for_each_leaf_node(rsp, rnp) {
|
||||
raw_spin_lock_irqsave(&rnp->lock, flags);
|
||||
smp_mb__after_unlock_lock();
|
||||
if (rnp->expmaskinit == rnp->expmaskinitnext) {
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
continue; /* No new CPUs, nothing to do. */
|
||||
}
|
||||
|
||||
/* Update this node's mask, track old value for propagation. */
|
||||
oldmask = rnp->expmaskinit;
|
||||
rnp->expmaskinit = rnp->expmaskinitnext;
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
|
||||
/* If was already nonzero, nothing to propagate. */
|
||||
if (oldmask)
|
||||
continue;
|
||||
|
||||
/* Propagate the new CPU up the tree. */
|
||||
mask = rnp->grpmask;
|
||||
rnp_up = rnp->parent;
|
||||
done = false;
|
||||
while (rnp_up) {
|
||||
raw_spin_lock_irqsave(&rnp_up->lock, flags);
|
||||
smp_mb__after_unlock_lock();
|
||||
if (rnp_up->expmaskinit)
|
||||
done = true;
|
||||
rnp_up->expmaskinit |= mask;
|
||||
raw_spin_unlock_irqrestore(&rnp_up->lock, flags);
|
||||
if (done)
|
||||
break;
|
||||
mask = rnp_up->grpmask;
|
||||
rnp_up = rnp_up->parent;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Reset the ->expmask values in the rcu_node tree in preparation for
|
||||
* a new expedited grace period.
|
||||
*/
|
||||
static void __maybe_unused sync_exp_reset_tree(struct rcu_state *rsp)
|
||||
{
|
||||
unsigned long flags;
|
||||
struct rcu_node *rnp;
|
||||
|
||||
sync_exp_reset_tree_hotplug(rsp);
|
||||
rcu_for_each_node_breadth_first(rsp, rnp) {
|
||||
raw_spin_lock_irqsave(&rnp->lock, flags);
|
||||
smp_mb__after_unlock_lock();
|
||||
WARN_ON_ONCE(rnp->expmask);
|
||||
rnp->expmask = rnp->expmaskinit;
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Return non-zero if there is no RCU expedited grace period in progress
|
||||
* for the specified rcu_node structure, in other words, if all CPUs and
|
||||
* tasks covered by the specified rcu_node structure have done their bit
|
||||
* for the current expedited grace period. Works only for preemptible
|
||||
* RCU -- other RCU implementation use other means.
|
||||
*
|
||||
* Caller must hold the root rcu_node's exp_funnel_mutex.
|
||||
*/
|
||||
static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
|
||||
{
|
||||
return rnp->exp_tasks == NULL &&
|
||||
READ_ONCE(rnp->expmask) == 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Report the exit from RCU read-side critical section for the last task
|
||||
* that queued itself during or before the current expedited preemptible-RCU
|
||||
* grace period. This event is reported either to the rcu_node structure on
|
||||
* which the task was queued or to one of that rcu_node structure's ancestors,
|
||||
* recursively up the tree. (Calm down, calm down, we do the recursion
|
||||
* iteratively!)
|
||||
*
|
||||
* Caller must hold the root rcu_node's exp_funnel_mutex and the
|
||||
* specified rcu_node structure's ->lock.
|
||||
*/
|
||||
static void __rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
|
||||
bool wake, unsigned long flags)
|
||||
__releases(rnp->lock)
|
||||
{
|
||||
unsigned long mask;
|
||||
|
||||
for (;;) {
|
||||
if (!sync_rcu_preempt_exp_done(rnp)) {
|
||||
if (!rnp->expmask)
|
||||
rcu_initiate_boost(rnp, flags);
|
||||
else
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
break;
|
||||
}
|
||||
if (rnp->parent == NULL) {
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
if (wake) {
|
||||
smp_mb(); /* EGP done before wake_up(). */
|
||||
wake_up(&rsp->expedited_wq);
|
||||
}
|
||||
break;
|
||||
}
|
||||
mask = rnp->grpmask;
|
||||
raw_spin_unlock(&rnp->lock); /* irqs remain disabled */
|
||||
rnp = rnp->parent;
|
||||
raw_spin_lock(&rnp->lock); /* irqs already disabled */
|
||||
smp_mb__after_unlock_lock();
|
||||
WARN_ON_ONCE(!(rnp->expmask & mask));
|
||||
rnp->expmask &= ~mask;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Report expedited quiescent state for specified node. This is a
|
||||
* lock-acquisition wrapper function for __rcu_report_exp_rnp().
|
||||
*
|
||||
* Caller must hold the root rcu_node's exp_funnel_mutex.
|
||||
*/
|
||||
static void __maybe_unused rcu_report_exp_rnp(struct rcu_state *rsp,
|
||||
struct rcu_node *rnp, bool wake)
|
||||
{
|
||||
unsigned long flags;
|
||||
|
||||
raw_spin_lock_irqsave(&rnp->lock, flags);
|
||||
smp_mb__after_unlock_lock();
|
||||
__rcu_report_exp_rnp(rsp, rnp, wake, flags);
|
||||
}
|
||||
|
||||
/*
|
||||
* Report expedited quiescent state for multiple CPUs, all covered by the
|
||||
* specified leaf rcu_node structure. Caller must hold the root
|
||||
* rcu_node's exp_funnel_mutex.
|
||||
*/
|
||||
static void rcu_report_exp_cpu_mult(struct rcu_state *rsp, struct rcu_node *rnp,
|
||||
unsigned long mask, bool wake)
|
||||
{
|
||||
unsigned long flags;
|
||||
|
||||
raw_spin_lock_irqsave(&rnp->lock, flags);
|
||||
smp_mb__after_unlock_lock();
|
||||
if (!(rnp->expmask & mask)) {
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
return;
|
||||
}
|
||||
rnp->expmask &= ~mask;
|
||||
__rcu_report_exp_rnp(rsp, rnp, wake, flags); /* Releases rnp->lock. */
|
||||
}
|
||||
|
||||
/*
|
||||
* Report expedited quiescent state for specified rcu_data (CPU).
|
||||
* Caller must hold the root rcu_node's exp_funnel_mutex.
|
||||
*/
|
||||
static void rcu_report_exp_rdp(struct rcu_state *rsp, struct rcu_data *rdp,
|
||||
bool wake)
|
||||
{
|
||||
rcu_report_exp_cpu_mult(rsp, rdp->mynode, rdp->grpmask, wake);
|
||||
}
|
||||
|
||||
/* Common code for synchronize_{rcu,sched}_expedited() work-done checking. */
|
||||
static bool sync_exp_work_done(struct rcu_state *rsp, struct rcu_node *rnp,
|
||||
struct rcu_data *rdp,
|
||||
|
|
@ -3462,16 +3660,111 @@ static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s)
|
|||
}
|
||||
|
||||
/* Invoked on each online non-idle CPU for expedited quiescent state. */
|
||||
static int synchronize_sched_expedited_cpu_stop(void *data)
|
||||
static void sync_sched_exp_handler(void *data)
|
||||
{
|
||||
struct rcu_data *rdp = data;
|
||||
struct rcu_state *rsp = rdp->rsp;
|
||||
struct rcu_data *rdp;
|
||||
struct rcu_node *rnp;
|
||||
struct rcu_state *rsp = data;
|
||||
|
||||
/* We are here: If we are last, do the wakeup. */
|
||||
rdp->exp_done = true;
|
||||
if (atomic_dec_and_test(&rsp->expedited_need_qs))
|
||||
wake_up(&rsp->expedited_wq);
|
||||
return 0;
|
||||
rdp = this_cpu_ptr(rsp->rda);
|
||||
rnp = rdp->mynode;
|
||||
if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) ||
|
||||
__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
|
||||
return;
|
||||
__this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, true);
|
||||
resched_cpu(smp_processor_id());
|
||||
}
|
||||
|
||||
/* Send IPI for expedited cleanup if needed at end of CPU-hotplug operation. */
|
||||
static void sync_sched_exp_online_cleanup(int cpu)
|
||||
{
|
||||
struct rcu_data *rdp;
|
||||
int ret;
|
||||
struct rcu_node *rnp;
|
||||
struct rcu_state *rsp = &rcu_sched_state;
|
||||
|
||||
rdp = per_cpu_ptr(rsp->rda, cpu);
|
||||
rnp = rdp->mynode;
|
||||
if (!(READ_ONCE(rnp->expmask) & rdp->grpmask))
|
||||
return;
|
||||
ret = smp_call_function_single(cpu, sync_sched_exp_handler, rsp, 0);
|
||||
WARN_ON_ONCE(ret);
|
||||
}
|
||||
|
||||
/*
|
||||
* Select the nodes that the upcoming expedited grace period needs
|
||||
* to wait for.
|
||||
*/
|
||||
static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
|
||||
smp_call_func_t func)
|
||||
{
|
||||
int cpu;
|
||||
unsigned long flags;
|
||||
unsigned long mask;
|
||||
unsigned long mask_ofl_test;
|
||||
unsigned long mask_ofl_ipi;
|
||||
int ret;
|
||||
struct rcu_node *rnp;
|
||||
|
||||
sync_exp_reset_tree(rsp);
|
||||
rcu_for_each_leaf_node(rsp, rnp) {
|
||||
raw_spin_lock_irqsave(&rnp->lock, flags);
|
||||
smp_mb__after_unlock_lock();
|
||||
|
||||
/* Each pass checks a CPU for identity, offline, and idle. */
|
||||
mask_ofl_test = 0;
|
||||
for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++) {
|
||||
struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
|
||||
struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
|
||||
|
||||
if (raw_smp_processor_id() == cpu ||
|
||||
!(atomic_add_return(0, &rdtp->dynticks) & 0x1))
|
||||
mask_ofl_test |= rdp->grpmask;
|
||||
}
|
||||
mask_ofl_ipi = rnp->expmask & ~mask_ofl_test;
|
||||
|
||||
/*
|
||||
* Need to wait for any blocked tasks as well. Note that
|
||||
* additional blocking tasks will also block the expedited
|
||||
* GP until such time as the ->expmask bits are cleared.
|
||||
*/
|
||||
if (rcu_preempt_has_tasks(rnp))
|
||||
rnp->exp_tasks = rnp->blkd_tasks.next;
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
|
||||
/* IPI the remaining CPUs for expedited quiescent state. */
|
||||
mask = 1;
|
||||
for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask <<= 1) {
|
||||
if (!(mask_ofl_ipi & mask))
|
||||
continue;
|
||||
retry_ipi:
|
||||
ret = smp_call_function_single(cpu, func, rsp, 0);
|
||||
if (!ret) {
|
||||
mask_ofl_ipi &= ~mask;
|
||||
} else {
|
||||
/* Failed, raced with offline. */
|
||||
raw_spin_lock_irqsave(&rnp->lock, flags);
|
||||
if (cpu_online(cpu) &&
|
||||
(rnp->expmask & mask)) {
|
||||
raw_spin_unlock_irqrestore(&rnp->lock,
|
||||
flags);
|
||||
schedule_timeout_uninterruptible(1);
|
||||
if (cpu_online(cpu) &&
|
||||
(rnp->expmask & mask))
|
||||
goto retry_ipi;
|
||||
raw_spin_lock_irqsave(&rnp->lock,
|
||||
flags);
|
||||
}
|
||||
if (!(rnp->expmask & mask))
|
||||
mask_ofl_ipi &= ~mask;
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
}
|
||||
}
|
||||
/* Report quiescent states for those that went offline. */
|
||||
mask_ofl_test |= mask_ofl_ipi;
|
||||
if (mask_ofl_test)
|
||||
rcu_report_exp_cpu_mult(rsp, rnp, mask_ofl_test, false);
|
||||
}
|
||||
}
|
||||
|
||||
static void synchronize_sched_expedited_wait(struct rcu_state *rsp)
|
||||
|
|
@ -3479,7 +3772,9 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp)
|
|||
int cpu;
|
||||
unsigned long jiffies_stall;
|
||||
unsigned long jiffies_start;
|
||||
struct rcu_data *rdp;
|
||||
unsigned long mask;
|
||||
struct rcu_node *rnp;
|
||||
struct rcu_node *rnp_root = rcu_get_root(rsp);
|
||||
int ret;
|
||||
|
||||
jiffies_stall = rcu_jiffies_till_stall_check();
|
||||
|
|
@ -3488,33 +3783,43 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp)
|
|||
for (;;) {
|
||||
ret = wait_event_interruptible_timeout(
|
||||
rsp->expedited_wq,
|
||||
!atomic_read(&rsp->expedited_need_qs),
|
||||
sync_rcu_preempt_exp_done(rnp_root),
|
||||
jiffies_stall);
|
||||
if (ret > 0)
|
||||
return;
|
||||
if (ret < 0) {
|
||||
/* Hit a signal, disable CPU stall warnings. */
|
||||
wait_event(rsp->expedited_wq,
|
||||
!atomic_read(&rsp->expedited_need_qs));
|
||||
sync_rcu_preempt_exp_done(rnp_root));
|
||||
return;
|
||||
}
|
||||
pr_err("INFO: %s detected expedited stalls on CPUs: {",
|
||||
pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {",
|
||||
rsp->name);
|
||||
for_each_online_cpu(cpu) {
|
||||
rdp = per_cpu_ptr(rsp->rda, cpu);
|
||||
rcu_for_each_leaf_node(rsp, rnp) {
|
||||
(void)rcu_print_task_exp_stall(rnp);
|
||||
mask = 1;
|
||||
for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask <<= 1) {
|
||||
struct rcu_data *rdp;
|
||||
|
||||
if (rdp->exp_done)
|
||||
continue;
|
||||
pr_cont(" %d", cpu);
|
||||
if (!(rnp->expmask & mask))
|
||||
continue;
|
||||
rdp = per_cpu_ptr(rsp->rda, cpu);
|
||||
pr_cont(" %d-%c%c%c", cpu,
|
||||
"O."[cpu_online(cpu)],
|
||||
"o."[!!(rdp->grpmask & rnp->expmaskinit)],
|
||||
"N."[!!(rdp->grpmask & rnp->expmaskinitnext)]);
|
||||
}
|
||||
mask <<= 1;
|
||||
}
|
||||
pr_cont(" } %lu jiffies s: %lu\n",
|
||||
jiffies - jiffies_start, rsp->expedited_sequence);
|
||||
for_each_online_cpu(cpu) {
|
||||
rdp = per_cpu_ptr(rsp->rda, cpu);
|
||||
|
||||
if (rdp->exp_done)
|
||||
continue;
|
||||
dump_cpu_task(cpu);
|
||||
rcu_for_each_leaf_node(rsp, rnp) {
|
||||
mask = 1;
|
||||
for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask <<= 1) {
|
||||
if (!(rnp->expmask & mask))
|
||||
continue;
|
||||
dump_cpu_task(cpu);
|
||||
}
|
||||
}
|
||||
jiffies_stall = 3 * rcu_jiffies_till_stall_check() + 3;
|
||||
}
|
||||
|
|
@ -3538,7 +3843,6 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp)
|
|||
*/
|
||||
void synchronize_sched_expedited(void)
|
||||
{
|
||||
int cpu;
|
||||
unsigned long s;
|
||||
struct rcu_node *rnp;
|
||||
struct rcu_state *rsp = &rcu_sched_state;
|
||||
|
|
@ -3546,48 +3850,16 @@ void synchronize_sched_expedited(void)
|
|||
/* Take a snapshot of the sequence number. */
|
||||
s = rcu_exp_gp_seq_snap(rsp);
|
||||
|
||||
if (!try_get_online_cpus()) {
|
||||
/* CPU hotplug operation in flight, fall back to normal GP. */
|
||||
wait_rcu_gp(call_rcu_sched);
|
||||
atomic_long_inc(&rsp->expedited_normal);
|
||||
return;
|
||||
}
|
||||
WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id()));
|
||||
|
||||
rnp = exp_funnel_lock(rsp, s);
|
||||
if (rnp == NULL) {
|
||||
put_online_cpus();
|
||||
if (rnp == NULL)
|
||||
return; /* Someone else did our work for us. */
|
||||
}
|
||||
|
||||
rcu_exp_gp_seq_start(rsp);
|
||||
|
||||
/* Stop each CPU that is online, non-idle, and not us. */
|
||||
init_waitqueue_head(&rsp->expedited_wq);
|
||||
atomic_set(&rsp->expedited_need_qs, 1); /* Extra count avoids race. */
|
||||
for_each_online_cpu(cpu) {
|
||||
struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
|
||||
struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
|
||||
|
||||
rdp->exp_done = false;
|
||||
|
||||
/* Skip our CPU and any idle CPUs. */
|
||||
if (raw_smp_processor_id() == cpu ||
|
||||
!(atomic_add_return(0, &rdtp->dynticks) & 0x1))
|
||||
continue;
|
||||
atomic_inc(&rsp->expedited_need_qs);
|
||||
stop_one_cpu_nowait(cpu, synchronize_sched_expedited_cpu_stop,
|
||||
rdp, &rdp->exp_stop_work);
|
||||
}
|
||||
|
||||
/* Remove extra count and, if necessary, wait for CPUs to stop. */
|
||||
if (!atomic_dec_and_test(&rsp->expedited_need_qs))
|
||||
synchronize_sched_expedited_wait(rsp);
|
||||
sync_rcu_exp_select_cpus(rsp, sync_sched_exp_handler);
|
||||
synchronize_sched_expedited_wait(rsp);
|
||||
|
||||
rcu_exp_gp_seq_end(rsp);
|
||||
mutex_unlock(&rnp->exp_funnel_mutex);
|
||||
|
||||
put_online_cpus();
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
|
||||
|
||||
|
|
@ -3613,11 +3885,11 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
|
|||
|
||||
/* Is the RCU core waiting for a quiescent state from this CPU? */
|
||||
if (rcu_scheduler_fully_active &&
|
||||
rdp->qs_pending && !rdp->passed_quiesce &&
|
||||
rdp->core_needs_qs && rdp->cpu_no_qs.b.norm &&
|
||||
rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
|
||||
rdp->n_rp_qs_pending++;
|
||||
} else if (rdp->qs_pending &&
|
||||
(rdp->passed_quiesce ||
|
||||
rdp->n_rp_core_needs_qs++;
|
||||
} else if (rdp->core_needs_qs &&
|
||||
(!rdp->cpu_no_qs.b.norm ||
|
||||
rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {
|
||||
rdp->n_rp_report_qs++;
|
||||
return 1;
|
||||
|
|
@ -3875,7 +4147,6 @@ static void rcu_init_new_rnp(struct rcu_node *rnp_leaf)
|
|||
static void __init
|
||||
rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
|
||||
{
|
||||
static struct lock_class_key rcu_exp_sched_rdp_class;
|
||||
unsigned long flags;
|
||||
struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
|
||||
struct rcu_node *rnp = rcu_get_root(rsp);
|
||||
|
|
@ -3891,10 +4162,6 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
|
|||
mutex_init(&rdp->exp_funnel_mutex);
|
||||
rcu_boot_init_nocb_percpu_data(rdp);
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
if (rsp == &rcu_sched_state)
|
||||
lockdep_set_class_and_name(&rdp->exp_funnel_mutex,
|
||||
&rcu_exp_sched_rdp_class,
|
||||
"rcu_data_exp_sched");
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -3913,7 +4180,6 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
|
|||
|
||||
/* Set up local state, ensuring consistent view of global state. */
|
||||
raw_spin_lock_irqsave(&rnp->lock, flags);
|
||||
rdp->beenonline = 1; /* We have now been online. */
|
||||
rdp->qlen_last_fqs_check = 0;
|
||||
rdp->n_force_qs_snap = rsp->n_force_qs;
|
||||
rdp->blimit = blimit;
|
||||
|
|
@ -3935,11 +4201,15 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
|
|||
raw_spin_lock(&rnp->lock); /* irqs already disabled. */
|
||||
smp_mb__after_unlock_lock();
|
||||
rnp->qsmaskinitnext |= mask;
|
||||
rnp->expmaskinitnext |= mask;
|
||||
if (!rdp->beenonline)
|
||||
WRITE_ONCE(rsp->ncpus, READ_ONCE(rsp->ncpus) + 1);
|
||||
rdp->beenonline = true; /* We have now been online. */
|
||||
rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */
|
||||
rdp->completed = rnp->completed;
|
||||
rdp->passed_quiesce = false;
|
||||
rdp->cpu_no_qs.b.norm = true;
|
||||
rdp->rcu_qs_ctr_snap = per_cpu(rcu_qs_ctr, cpu);
|
||||
rdp->qs_pending = false;
|
||||
rdp->core_needs_qs = false;
|
||||
trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));
|
||||
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
||||
}
|
||||
|
|
@ -3972,6 +4242,7 @@ int rcu_cpu_notify(struct notifier_block *self,
|
|||
break;
|
||||
case CPU_ONLINE:
|
||||
case CPU_DOWN_FAILED:
|
||||
sync_sched_exp_online_cleanup(cpu);
|
||||
rcu_boost_kthread_setaffinity(rnp, -1);
|
||||
break;
|
||||
case CPU_DOWN_PREPARE:
|
||||
|
|
@ -3983,6 +4254,12 @@ int rcu_cpu_notify(struct notifier_block *self,
|
|||
rcu_cleanup_dying_cpu(rsp);
|
||||
break;
|
||||
case CPU_DYING_IDLE:
|
||||
/* QS for any half-done expedited RCU-sched GP. */
|
||||
preempt_disable();
|
||||
rcu_report_exp_rdp(&rcu_sched_state,
|
||||
this_cpu_ptr(rcu_sched_state.rda), true);
|
||||
preempt_enable();
|
||||
|
||||
for_each_rcu_flavor(rsp) {
|
||||
rcu_cleanup_dying_idle_cpu(cpu, rsp);
|
||||
}
|
||||
|
|
@ -4114,7 +4391,6 @@ static void __init rcu_init_one(struct rcu_state *rsp,
|
|||
static const char * const buf[] = RCU_NODE_NAME_INIT;
|
||||
static const char * const fqs[] = RCU_FQS_NAME_INIT;
|
||||
static const char * const exp[] = RCU_EXP_NAME_INIT;
|
||||
static const char * const exp_sched[] = RCU_EXP_SCHED_NAME_INIT;
|
||||
static u8 fl_mask = 0x1;
|
||||
|
||||
int levelcnt[RCU_NUM_LVLS]; /* # nodes in each level. */
|
||||
|
|
@ -4174,18 +4450,13 @@ static void __init rcu_init_one(struct rcu_state *rsp,
|
|||
INIT_LIST_HEAD(&rnp->blkd_tasks);
|
||||
rcu_init_one_nocb(rnp);
|
||||
mutex_init(&rnp->exp_funnel_mutex);
|
||||
if (rsp == &rcu_sched_state)
|
||||
lockdep_set_class_and_name(
|
||||
&rnp->exp_funnel_mutex,
|
||||
&rcu_exp_sched_class[i], exp_sched[i]);
|
||||
else
|
||||
lockdep_set_class_and_name(
|
||||
&rnp->exp_funnel_mutex,
|
||||
&rcu_exp_class[i], exp[i]);
|
||||
lockdep_set_class_and_name(&rnp->exp_funnel_mutex,
|
||||
&rcu_exp_class[i], exp[i]);
|
||||
}
|
||||
}
|
||||
|
||||
init_waitqueue_head(&rsp->gp_wq);
|
||||
init_waitqueue_head(&rsp->expedited_wq);
|
||||
rnp = rsp->level[rcu_num_lvls - 1];
|
||||
for_each_possible_cpu(i) {
|
||||
while (i > rnp->grphi)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue