mirror of
https://github.com/Fishwaldo/linux-bl808.git
synced 2025-06-06 06:35:12 +00:00
sched/fair: Move idle_balance()
We're going to want to call nohz_idle_balance() or parts thereof from idle_balance(). Since we already have a forward declaration of idle_balance() move it down such that it's below nohz_idle_balance() avoiding the need for a forward declaration for that. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Peter Zijlstra
Ingo Molnar
parent
dd707247ab
commit
47ea54121e
1 changed files with 114 additions and 114 deletions
|
|
@ -8916,120 +8916,6 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance)
|
||||||
*next_balance = next;
|
*next_balance = next;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* idle_balance is called by schedule() if this_cpu is about to become
|
|
||||||
* idle. Attempts to pull tasks from other CPUs.
|
|
||||||
*/
|
|
||||||
static int idle_balance(struct rq *this_rq, struct rq_flags *rf)
|
|
||||||
{
|
|
||||||
unsigned long next_balance = jiffies + HZ;
|
|
||||||
int this_cpu = this_rq->cpu;
|
|
||||||
struct sched_domain *sd;
|
|
||||||
int pulled_task = 0;
|
|
||||||
u64 curr_cost = 0;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* We must set idle_stamp _before_ calling idle_balance(), such that we
|
|
||||||
* measure the duration of idle_balance() as idle time.
|
|
||||||
*/
|
|
||||||
this_rq->idle_stamp = rq_clock(this_rq);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Do not pull tasks towards !active CPUs...
|
|
||||||
*/
|
|
||||||
if (!cpu_active(this_cpu))
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* This is OK, because current is on_cpu, which avoids it being picked
|
|
||||||
* for load-balance and preemption/IRQs are still disabled avoiding
|
|
||||||
* further scheduler activity on it and we're being very careful to
|
|
||||||
* re-start the picking loop.
|
|
||||||
*/
|
|
||||||
rq_unpin_lock(this_rq, rf);
|
|
||||||
|
|
||||||
if (this_rq->avg_idle < sysctl_sched_migration_cost ||
|
|
||||||
!this_rq->rd->overload) {
|
|
||||||
rcu_read_lock();
|
|
||||||
sd = rcu_dereference_check_sched_domain(this_rq->sd);
|
|
||||||
if (sd)
|
|
||||||
update_next_balance(sd, &next_balance);
|
|
||||||
rcu_read_unlock();
|
|
||||||
|
|
||||||
goto out;
|
|
||||||
}
|
|
||||||
|
|
||||||
raw_spin_unlock(&this_rq->lock);
|
|
||||||
|
|
||||||
update_blocked_averages(this_cpu);
|
|
||||||
rcu_read_lock();
|
|
||||||
for_each_domain(this_cpu, sd) {
|
|
||||||
int continue_balancing = 1;
|
|
||||||
u64 t0, domain_cost;
|
|
||||||
|
|
||||||
if (!(sd->flags & SD_LOAD_BALANCE))
|
|
||||||
continue;
|
|
||||||
|
|
||||||
if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) {
|
|
||||||
update_next_balance(sd, &next_balance);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (sd->flags & SD_BALANCE_NEWIDLE) {
|
|
||||||
t0 = sched_clock_cpu(this_cpu);
|
|
||||||
|
|
||||||
pulled_task = load_balance(this_cpu, this_rq,
|
|
||||||
sd, CPU_NEWLY_IDLE,
|
|
||||||
&continue_balancing);
|
|
||||||
|
|
||||||
domain_cost = sched_clock_cpu(this_cpu) - t0;
|
|
||||||
if (domain_cost > sd->max_newidle_lb_cost)
|
|
||||||
sd->max_newidle_lb_cost = domain_cost;
|
|
||||||
|
|
||||||
curr_cost += domain_cost;
|
|
||||||
}
|
|
||||||
|
|
||||||
update_next_balance(sd, &next_balance);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Stop searching for tasks to pull if there are
|
|
||||||
* now runnable tasks on this rq.
|
|
||||||
*/
|
|
||||||
if (pulled_task || this_rq->nr_running > 0)
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
rcu_read_unlock();
|
|
||||||
|
|
||||||
raw_spin_lock(&this_rq->lock);
|
|
||||||
|
|
||||||
if (curr_cost > this_rq->max_idle_balance_cost)
|
|
||||||
this_rq->max_idle_balance_cost = curr_cost;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* While browsing the domains, we released the rq lock, a task could
|
|
||||||
* have been enqueued in the meantime. Since we're not going idle,
|
|
||||||
* pretend we pulled a task.
|
|
||||||
*/
|
|
||||||
if (this_rq->cfs.h_nr_running && !pulled_task)
|
|
||||||
pulled_task = 1;
|
|
||||||
|
|
||||||
out:
|
|
||||||
/* Move the next balance forward */
|
|
||||||
if (time_after(this_rq->next_balance, next_balance))
|
|
||||||
this_rq->next_balance = next_balance;
|
|
||||||
|
|
||||||
/* Is there a task of a high priority class? */
|
|
||||||
if (this_rq->nr_running != this_rq->cfs.h_nr_running)
|
|
||||||
pulled_task = -1;
|
|
||||||
|
|
||||||
if (pulled_task)
|
|
||||||
this_rq->idle_stamp = 0;
|
|
||||||
|
|
||||||
rq_repin_lock(this_rq, rf);
|
|
||||||
|
|
||||||
return pulled_task;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* active_load_balance_cpu_stop is run by the CPU stopper. It pushes
|
* active_load_balance_cpu_stop is run by the CPU stopper. It pushes
|
||||||
* running tasks off the busiest CPU onto idle CPUs. It requires at
|
* running tasks off the busiest CPU onto idle CPUs. It requires at
|
||||||
|
|
@ -9603,6 +9489,120 @@ static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
|
||||||
}
|
}
|
||||||
#endif /* CONFIG_NO_HZ_COMMON */
|
#endif /* CONFIG_NO_HZ_COMMON */
|
||||||
|
|
||||||
|
/*
|
||||||
|
* idle_balance is called by schedule() if this_cpu is about to become
|
||||||
|
* idle. Attempts to pull tasks from other CPUs.
|
||||||
|
*/
|
||||||
|
static int idle_balance(struct rq *this_rq, struct rq_flags *rf)
|
||||||
|
{
|
||||||
|
unsigned long next_balance = jiffies + HZ;
|
||||||
|
int this_cpu = this_rq->cpu;
|
||||||
|
struct sched_domain *sd;
|
||||||
|
int pulled_task = 0;
|
||||||
|
u64 curr_cost = 0;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* We must set idle_stamp _before_ calling idle_balance(), such that we
|
||||||
|
* measure the duration of idle_balance() as idle time.
|
||||||
|
*/
|
||||||
|
this_rq->idle_stamp = rq_clock(this_rq);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Do not pull tasks towards !active CPUs...
|
||||||
|
*/
|
||||||
|
if (!cpu_active(this_cpu))
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This is OK, because current is on_cpu, which avoids it being picked
|
||||||
|
* for load-balance and preemption/IRQs are still disabled avoiding
|
||||||
|
* further scheduler activity on it and we're being very careful to
|
||||||
|
* re-start the picking loop.
|
||||||
|
*/
|
||||||
|
rq_unpin_lock(this_rq, rf);
|
||||||
|
|
||||||
|
if (this_rq->avg_idle < sysctl_sched_migration_cost ||
|
||||||
|
!this_rq->rd->overload) {
|
||||||
|
rcu_read_lock();
|
||||||
|
sd = rcu_dereference_check_sched_domain(this_rq->sd);
|
||||||
|
if (sd)
|
||||||
|
update_next_balance(sd, &next_balance);
|
||||||
|
rcu_read_unlock();
|
||||||
|
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
raw_spin_unlock(&this_rq->lock);
|
||||||
|
|
||||||
|
update_blocked_averages(this_cpu);
|
||||||
|
rcu_read_lock();
|
||||||
|
for_each_domain(this_cpu, sd) {
|
||||||
|
int continue_balancing = 1;
|
||||||
|
u64 t0, domain_cost;
|
||||||
|
|
||||||
|
if (!(sd->flags & SD_LOAD_BALANCE))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) {
|
||||||
|
update_next_balance(sd, &next_balance);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sd->flags & SD_BALANCE_NEWIDLE) {
|
||||||
|
t0 = sched_clock_cpu(this_cpu);
|
||||||
|
|
||||||
|
pulled_task = load_balance(this_cpu, this_rq,
|
||||||
|
sd, CPU_NEWLY_IDLE,
|
||||||
|
&continue_balancing);
|
||||||
|
|
||||||
|
domain_cost = sched_clock_cpu(this_cpu) - t0;
|
||||||
|
if (domain_cost > sd->max_newidle_lb_cost)
|
||||||
|
sd->max_newidle_lb_cost = domain_cost;
|
||||||
|
|
||||||
|
curr_cost += domain_cost;
|
||||||
|
}
|
||||||
|
|
||||||
|
update_next_balance(sd, &next_balance);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Stop searching for tasks to pull if there are
|
||||||
|
* now runnable tasks on this rq.
|
||||||
|
*/
|
||||||
|
if (pulled_task || this_rq->nr_running > 0)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
rcu_read_unlock();
|
||||||
|
|
||||||
|
raw_spin_lock(&this_rq->lock);
|
||||||
|
|
||||||
|
if (curr_cost > this_rq->max_idle_balance_cost)
|
||||||
|
this_rq->max_idle_balance_cost = curr_cost;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* While browsing the domains, we released the rq lock, a task could
|
||||||
|
* have been enqueued in the meantime. Since we're not going idle,
|
||||||
|
* pretend we pulled a task.
|
||||||
|
*/
|
||||||
|
if (this_rq->cfs.h_nr_running && !pulled_task)
|
||||||
|
pulled_task = 1;
|
||||||
|
|
||||||
|
out:
|
||||||
|
/* Move the next balance forward */
|
||||||
|
if (time_after(this_rq->next_balance, next_balance))
|
||||||
|
this_rq->next_balance = next_balance;
|
||||||
|
|
||||||
|
/* Is there a task of a high priority class? */
|
||||||
|
if (this_rq->nr_running != this_rq->cfs.h_nr_running)
|
||||||
|
pulled_task = -1;
|
||||||
|
|
||||||
|
if (pulled_task)
|
||||||
|
this_rq->idle_stamp = 0;
|
||||||
|
|
||||||
|
rq_repin_lock(this_rq, rf);
|
||||||
|
|
||||||
|
return pulled_task;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* run_rebalance_domains is triggered when needed from the scheduler tick.
|
* run_rebalance_domains is triggered when needed from the scheduler tick.
|
||||||
* Also triggered for nohz idle balancing (with nohz_balancing_kick set).
|
* Also triggered for nohz idle balancing (with nohz_balancing_kick set).
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue