/*

* kernel/sched/core.c

*

* Kernel scheduler and related syscalls

*

* Copyright (C) 1991-2002 Linus Torvalds

*

* 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and

* make semaphores SMP safe

* 1998-11-19 Implemented schedule_timeout() and related stuff

* by Andrea Arcangeli

* 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar:

* hybrid priority-list and round-robin design with

* an array-switch method of distributing timeslices

* and per-CPU runqueues. Cleanups and useful suggestions

* by Davide Libenzi, preemptible kernel bits by Robert Love.

* 2003-09-03 Interactivity tuning by Con Kolivas.

* 2004-04-02 Scheduler domains code by Nick Piggin

* 2007-04-15 Work begun on replacing all interactivity tuning with a

* fair scheduling design by Con Kolivas.

* 2007-05-05 Load balancing (smp-nice) and other improvements

* by Peter Williams

* 2007-05-06 Interactivity improvements to CFS by Mike Galbraith

* 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri

* 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins,

* Thomas Gleixner, Mike Kravetz

*/

#include <linux/kasan.h>

#include <linux/mm.h>

#include <linux/module.h>

#include <linux/nmi.h>

#include <linux/init.h>

#include <linux/uaccess.h>

#include <linux/highmem.h>

#include <linux/mmu_context.h>

#include <linux/interrupt.h>

#include <linux/capability.h>

#include <linux/completion.h>

#include <linux/kernel_stat.h>

#include <linux/debug_locks.h>

#include <linux/perf_event.h>

#include <linux/security.h>

#include <linux/notifier.h>

#include <linux/profile.h>

#include <linux/freezer.h>

#include <linux/vmalloc.h>

#include <linux/blkdev.h>

#include <linux/delay.h>

#include <linux/pid_namespace.h>

#include <linux/smp.h>

#include <linux/threads.h>

#include <linux/timer.h>

#include <linux/rcupdate.h>

#include <linux/cpu.h>

#include <linux/cpuset.h>

#include <linux/percpu.h>

#include <linux/proc_fs.h>

#include <linux/seq_file.h>

#include <linux/sysctl.h>

#include <linux/syscalls.h>

#include <linux/times.h>

#include <linux/tsacct_kern.h>

#include <linux/kprobes.h>

#include <linux/delayacct.h>

#include <linux/unistd.h>

#include <linux/pagemap.h>

#include <linux/hrtimer.h>

#include <linux/tick.h>

#include <linux/ctype.h>

#include <linux/ftrace.h>

#include <linux/slab.h>

#include <linux/init_task.h>

#include <linux/context_tracking.h>

#include <linux/compiler.h>

#include <linux/frame.h>

#include <linux/prefetch.h>

#include <asm/switch_to.h>

#include <asm/tlb.h>

#include <asm/irq_regs.h>

#include <asm/mutex.h>

#ifdef CONFIG_PARAVIRT

#include <asm/paravirt.h>

#endif

#include "sched.h"

#include "../workqueue_internal.h"

#include "../smpboot.h"

#define CREATE_TRACE_POINTS

#include <trace/events/sched.h>

DEFINE_MUTEX(sched_domains_mutex);

DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);

static void update_rq_clock_task(struct rq *rq, s64 delta);

void update_rq_clock(struct rq *rq)

{

s64 delta;

lockdep_assert_held(&rq->lock);

if (rq->clock_skip_update & RQCF_ACT_SKIP)

return;

delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;

if (delta < 0)

return;

rq->clock += delta;

update_rq_clock_task(rq, delta);

}

/*

* Debugging: various feature bits

*/

#define SCHED_FEAT(name, enabled) \

(1UL << __SCHED_FEAT_##name) * enabled |

const_debug unsigned int sysctl_sched_features =

#include "features.h"

0;

#undef SCHED_FEAT

/*

* Number of tasks to iterate in a single balance run.

* Limited because this is done with IRQs disabled.

*/

const_debug unsigned int sysctl_sched_nr_migrate = 32;

/*

* period over which we average the RT time consumption, measured

* in ms.

*

* default: 1s

*/

const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;

/*

* period over which we measure -rt task cpu usage in us.

* default: 1s

*/

unsigned int sysctl_sched_rt_period = 1000000;

__read_mostly int scheduler_running;

/*

* part of the period that we allow rt tasks to run in us.

* default: 0.95s

*/

int sysctl_sched_rt_runtime = 950000;

/* cpus with isolated domains */

cpumask_var_t cpu_isolated_map;

/*

* this_rq_lock - lock this runqueue and disable interrupts.

*/

static struct rq *this_rq_lock(void)

__acquires(rq->lock)

{

struct rq *rq;

local_irq_disable();

rq = this_rq();

raw_spin_lock(&rq->lock);

return rq;

}

/*

* __task_rq_lock - lock the rq @p resides on.

*/

struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)

__acquires(rq->lock)

{

struct rq *rq;

lockdep_assert_held(&p->pi_lock);

for (;;) {

rq = task_rq(p);

raw_spin_lock(&rq->lock);

if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {

rf->cookie = lockdep_pin_lock(&rq->lock);

return rq;

}

raw_spin_unlock(&rq->lock);

while (unlikely(task_on_rq_migrating(p)))

cpu_relax();

}

}

/*

* task_rq_lock - lock p->pi_lock and lock the rq @p resides on.

*/

struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)

__acquires(p->pi_lock)

__acquires(rq->lock)

{

struct rq *rq;

for (;;) {

raw_spin_lock_irqsave(&p->pi_lock, rf->flags);

rq = task_rq(p);

raw_spin_lock(&rq->lock);

/*

* move_queued_task() task_rq_lock()

*

* ACQUIRE (rq->lock)

* [S] ->on_rq = MIGRATING [L] rq = task_rq()

* WMB (__set_task_cpu()) ACQUIRE (rq->lock);

* [S] ->cpu = new_cpu [L] task_rq()

* [L] ->on_rq

* RELEASE (rq->lock)

*

* If we observe the old cpu in task_rq_lock, the acquire of

* the old rq->lock will fully serialize against the stores.

*

* If we observe the new cpu in task_rq_lock, the acquire will

* pair with the WMB to ensure we must then also see migrating.

*/

if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {

rf->cookie = lockdep_pin_lock(&rq->lock);

return rq;

}

raw_spin_unlock(&rq->lock);

raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);

while (unlikely(task_on_rq_migrating(p)))

cpu_relax();

}

}

#ifdef CONFIG_SCHED_HRTICK

/*

* Use HR-timers to deliver accurate preemption points.

*/

static void hrtick_clear(struct rq *rq)

{

if (hrtimer_active(&rq->hrtick_timer))

hrtimer_cancel(&rq->hrtick_timer);

}

/*

* High-resolution timer tick.

* Runs from hardirq context with interrupts disabled.

*/

static enum hrtimer_restart hrtick(struct hrtimer *timer)

{

struct rq *rq = container_of(timer, struct rq, hrtick_timer);

WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());

raw_spin_lock(&rq->lock);

update_rq_clock(rq);

rq->curr->sched_class->task_tick(rq, rq->curr, 1);

raw_spin_unlock(&rq->lock);

return HRTIMER_NORESTART;

}

#ifdef CONFIG_SMP

static void __hrtick_restart(struct rq *rq)

{

struct hrtimer *timer = &rq->hrtick_timer;

hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);

}

/*

* called from hardirq (IPI) context

*/

static void __hrtick_start(void *arg)

{

struct rq *rq = arg;

raw_spin_lock(&rq->lock);

__hrtick_restart(rq);

rq->hrtick_csd_pending = 0;

raw_spin_unlock(&rq->lock);

}

/*

* Called to set the hrtick timer state.

*

* called with rq->lock held and irqs disabled

*/

void hrtick_start(struct rq *rq, u64 delay)

{

struct hrtimer *timer = &rq->hrtick_timer;

ktime_t time;

s64 delta;

/*

* Don't schedule slices shorter than 10000ns, that just

* doesn't make sense and can cause timer DoS.

*/

delta = max_t(s64, delay, 10000LL);

time = ktime_add_ns(timer->base->get_time(), delta);

hrtimer_set_expires(timer, time);

if (rq == this_rq()) {

__hrtick_restart(rq);

} else if (!rq->hrtick_csd_pending) {

smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);

rq->hrtick_csd_pending = 1;

}

}

#else

/*

* Called to set the hrtick timer state.

*

* called with rq->lock held and irqs disabled

*/

void hrtick_start(struct rq *rq, u64 delay)

{

/*

* Don't schedule slices shorter than 10000ns, that just

* doesn't make sense. Rely on vruntime for fairness.

*/

delay = max_t(u64, delay, 10000LL);

hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),

HRTIMER_MODE_REL_PINNED);

}

#endif /* CONFIG_SMP */

static void init_rq_hrtick(struct rq *rq)

{

#ifdef CONFIG_SMP

rq->hrtick_csd_pending = 0;

rq->hrtick_csd.flags = 0;

rq->hrtick_csd.func = __hrtick_start;

rq->hrtick_csd.info = rq;

#endif

hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);

rq->hrtick_timer.function = hrtick;

}

#else /* CONFIG_SCHED_HRTICK */

static inline void hrtick_clear(struct rq *rq)

{

}

static inline void init_rq_hrtick(struct rq *rq)

{

}

#endif /* CONFIG_SCHED_HRTICK */

/*

* cmpxchg based fetch_or, macro so it works for different integer types

*/

#define fetch_or(ptr, mask) \

({ \

typeof(ptr) _ptr = (ptr); \

typeof(mask) _mask = (mask); \

typeof(*_ptr) _old, _val = *_ptr; \

\

for (;;) { \

_old = cmpxchg(_ptr, _val, _val | _mask); \

if (_old == _val) \

break; \

_val = _old; \

} \

_old; \

})

#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)

/*

* Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,

* this avoids any races wrt polling state changes and thereby avoids

* spurious IPIs.

*/

static bool set_nr_and_not_polling(struct task_struct *p)

{

struct thread_info *ti = task_thread_info(p);

return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);

}

/*

* Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.

*

* If this returns true, then the idle task promises to call

* sched_ttwu_pending() and reschedule soon.

*/

static bool set_nr_if_polling(struct task_struct *p)

{

struct thread_info *ti = task_thread_info(p);

typeof(ti->flags) old, val = READ_ONCE(ti->flags);

for (;;) {

if (!(val & _TIF_POLLING_NRFLAG))

return false;

if (val & _TIF_NEED_RESCHED)

return true;

old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);

if (old == val)

break;

val = old;

}

return true;

}

#else

static bool set_nr_and_not_polling(struct task_struct *p)

{

set_tsk_need_resched(p);

return true;

}

#ifdef CONFIG_SMP

static bool set_nr_if_polling(struct task_struct *p)

{

return false;

}

#endif

#endif

void wake_q_add(struct wake_q_head *head, struct task_struct *task)

{

struct wake_q_node *node = &task->wake_q;

/*

* Atomically grab the task, if ->wake_q is !nil already it means

* its already queued (either by us or someone else) and will get the

* wakeup due to that.

*

* This cmpxchg() implies a full barrier, which pairs with the write

* barrier implied by the wakeup in wake_up_q().

*/

if (cmpxchg(&node->next, NULL, WAKE_Q_TAIL))

return;

get_task_struct(task);

/*

* The head is context local, there can be no concurrency.

*/

*head->lastp = node;

head->lastp = &node->next;

}

void wake_up_q(struct wake_q_head *head)

{

struct wake_q_node *node = head->first;

while (node != WAKE_Q_TAIL) {

struct task_struct *task;

task = container_of(node, struct task_struct, wake_q);

BUG_ON(!task);

/* task can safely be re-inserted now */

node = node->next;

task->wake_q.next = NULL;

/*

* wake_up_process() implies a wmb() to pair with the queueing

* in wake_q_add() so as not to miss wakeups.

*/

wake_up_process(task);

put_task_struct(task);

}

}

/*

* resched_curr - mark rq's current task 'to be rescheduled now'.

*

* On UP this means the setting of the need_resched flag, on SMP it

* might also involve a cross-CPU call to trigger the scheduler on

* the target CPU.

*/

void resched_curr(struct rq *rq)

{

struct task_struct *curr = rq->curr;

int cpu;

lockdep_assert_held(&rq->lock);

if (test_tsk_need_resched(curr))

return;

cpu = cpu_of(rq);

if (cpu == smp_processor_id()) {

set_tsk_need_resched(curr);

set_preempt_need_resched();

return;

}

if (set_nr_and_not_polling(curr))

smp_send_reschedule(cpu);

else

trace_sched_wake_idle_without_ipi(cpu);

}

void resched_cpu(int cpu)

{

struct rq *rq = cpu_rq(cpu);

unsigned long flags;

if (!raw_spin_trylock_irqsave(&rq->lock, flags))

return;

resched_curr(rq);

raw_spin_unlock_irqrestore(&rq->lock, flags);

}

#ifdef CONFIG_SMP

#ifdef CONFIG_NO_HZ_COMMON

/*

* In the semi idle case, use the nearest busy cpu for migrating timers

* from an idle cpu. This is good for power-savings.

*

* We don't do similar optimization for completely idle system, as

* selecting an idle cpu will add more delays to the timers than intended

* (as that cpu's timer base may not be uptodate wrt jiffies etc).

*/

int get_nohz_timer_target(void)

{

int i, cpu = smp_processor_id();

struct sched_domain *sd;

if (!idle_cpu(cpu) && is_housekeeping_cpu(cpu))

return cpu;

rcu_read_lock();

for_each_domain(cpu, sd) {

for_each_cpu(i, sched_domain_span(sd)) {

if (cpu == i)

continue;

if (!idle_cpu(i) && is_housekeeping_cpu(i)) {

cpu = i;

goto unlock;

}

}

}

if (!is_housekeeping_cpu(cpu))

cpu = housekeeping_any_cpu();

unlock:

rcu_read_unlock();

return cpu;

}

/*

* When add_timer_on() enqueues a timer into the timer wheel of an

* idle CPU then this timer might expire before the next timer event

* which is scheduled to wake up that CPU. In case of a completely

* idle system the next event might even be infinite time into the

* future. wake_up_idle_cpu() ensures that the CPU is woken up and

* leaves the inner idle loop so the newly added timer is taken into

* account when the CPU goes back to idle and evaluates the timer

* wheel for the next timer event.

*/

static void wake_up_idle_cpu(int cpu)

{

struct rq *rq = cpu_rq(cpu);

if (cpu == smp_processor_id())

return;

if (set_nr_and_not_polling(rq->idle))

smp_send_reschedule(cpu);

else

trace_sched_wake_idle_without_ipi(cpu);

}

static bool wake_up_full_nohz_cpu(int cpu)

{

/*

* We just need the target to call irq_exit() and re-evaluate

* the next tick. The nohz full kick at least implies that.

* If needed we can still optimize that later with an

* empty IRQ.

*/

if (tick_nohz_full_cpu(cpu)) {

if (cpu != smp_processor_id() ||

tick_nohz_tick_stopped())

tick_nohz_full_kick_cpu(cpu);

return true;

}

return false;

}

void wake_up_nohz_cpu(int cpu)

{

if (!wake_up_full_nohz_cpu(cpu))

wake_up_idle_cpu(cpu);

}

static inline bool got_nohz_idle_kick(void)

{

int cpu = smp_processor_id();

if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)))

return false;

if (idle_cpu(cpu) && !need_resched())

return true;

/*

* We can't run Idle Load Balance on this CPU for this time so we

* cancel it and clear NOHZ_BALANCE_KICK

*/

clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu));

return false;

}

#else /* CONFIG_NO_HZ_COMMON */

static inline bool got_nohz_idle_kick(void)

{

return false;

}

#endif /* CONFIG_NO_HZ_COMMON */

#ifdef CONFIG_NO_HZ_FULL

bool sched_can_stop_tick(struct rq *rq)

{

int fifo_nr_running;

/* Deadline tasks, even if single, need the tick */

if (rq->dl.dl_nr_running)

return false;

/*

* If there are more than one RR tasks, we need the tick to effect the

* actual RR behaviour.

*/

if (rq->rt.rr_nr_running) {

if (rq->rt.rr_nr_running == 1)

return true;

else

return false;

}

/*

* If there's no RR tasks, but FIFO tasks, we can skip the tick, no

* forced preemption between FIFO tasks.

*/

fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running;

if (fifo_nr_running)

return true;

/*

* If there are no DL,RR/FIFO tasks, there must only be CFS tasks left;

* if there's more than one we need the tick for involuntary

* preemption.

*/

if (rq->nr_running > 1)

return false;

return true;

}

#endif /* CONFIG_NO_HZ_FULL */

void sched_avg_update(struct rq *rq)

{

s64 period = sched_avg_period();

while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {

/*

* Inline assembly required to prevent the compiler

* optimising this loop into a divmod call.

* See __iter_div_u64_rem() for another example of this.

*/

asm("" : "+rm" (rq->age_stamp));

rq->age_stamp += period;

rq->rt_avg /= 2;

}

}

#endif /* CONFIG_SMP */

#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \

(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))

/*

* Iterate task_group tree rooted at *from, calling @down when first entering a

* node and @up when leaving it for the final time.

*

* Caller must hold rcu_lock or sufficient equivalent.

*/

int walk_tg_tree_from(struct task_group *from,

tg_visitor down, tg_visitor up, void *data)

{

struct task_group *parent, *child;

int ret;

parent = from;

down:

ret = (*down)(parent, data);

if (ret)

goto out;

list_for_each_entry_rcu(child, &parent->children, siblings) {

parent = child;

goto down;

up:

continue;

}

ret = (*up)(parent, data);

if (ret || parent == from)

goto out;

child = parent;

parent = parent->parent;

if (parent)

goto up;

out:

return ret;

}

int tg_nop(struct task_group *tg, void *data)

{

return 0;

}

#endif

static void set_load_weight(struct task_struct *p)

{

int prio = p->static_prio - MAX_RT_PRIO;

struct load_weight *load = &p->se.load;

/*

* SCHED_IDLE tasks get minimal weight:

*/

if (idle_policy(p->policy)) {

load->weight = scale_load(WEIGHT_IDLEPRIO);

load->inv_weight = WMULT_IDLEPRIO;

return;

}

load->weight = scale_load(sched_prio_to_weight[prio]);

load->inv_weight = sched_prio_to_wmult[prio];

}

static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)

{

update_rq_clock(rq);

if (!(flags & ENQUEUE_RESTORE))

sched_info_queued(rq, p);

p->sched_class->enqueue_task(rq, p, flags);

}

static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)

{

update_rq_clock(rq);

if (!(flags & DEQUEUE_SAVE))

sched_info_dequeued(rq, p);

p->sched_class->dequeue_task(rq, p, flags);

}

void activate_task(struct rq *rq, struct task_struct *p, int flags)

{

if (task_contributes_to_load(p))

rq->nr_uninterruptible--;

enqueue_task(rq, p, flags);

}

void deactivate_task(struct rq *rq, struct task_struct *p, int flags)

{

if (task_contributes_to_load(p))

rq->nr_uninterruptible++;

dequeue_task(rq, p, flags);

}

static void update_rq_clock_task(struct rq *rq, s64 delta)

{

/*

* In theory, the compile should just see 0 here, and optimize out the call

* to sched_rt_avg_update. But I don't trust it...

*/

#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)

s64 steal = 0, irq_delta = 0;

#endif

#ifdef CONFIG_IRQ_TIME_ACCOUNTING

irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;

/*

* Since irq_time is only updated on {soft,}irq_exit, we might run into

* this case when a previous update_rq_clock() happened inside a

* {soft,}irq region.

*

* When this happens, we stop ->clock_task and only update the

* prev_irq_time stamp to account for the part that fit, so that a next

* update will consume the rest. This ensures ->clock_task is

* monotonic.

*

* It does however cause some slight miss-attribution of {soft,}irq

* time, a more accurate solution would be to update the irq_time using

* the current rq->clock timestamp, except that would require using

* atomic ops.

*/

if (irq_delta > delta)

irq_delta = delta;

rq->prev_irq_time += irq_delta;

delta -= irq_delta;

#endif

#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING

if (static_key_false((&paravirt_steal_rq_enabled))) {

steal = paravirt_steal_clock(cpu_of(rq));

steal -= rq->prev_steal_time_rq;

if (unlikely(steal > delta))

steal = delta;

rq->prev_steal_time_rq += steal;

delta -= steal;

}

#endif

rq->clock_task += delta;

#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)

if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))

sched_rt_avg_update(rq, irq_delta + steal);

#endif

}

void sched_set_stop_task(int cpu, struct task_struct *stop)

{

struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };

struct task_struct *old_stop = cpu_rq(cpu)->stop;

if (stop) {

/*

* Make it appear like a SCHED_FIFO task, its something

* userspace knows about and won't get confused about.

*

* Also, it will make PI more or less work without too

* much confusion -- but then, stop work should not

* rely on PI working anyway.

*/

sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);

stop->sched_class = &stop_sched_class;

}

cpu_rq(cpu)->stop = stop;

if (old_stop) {

/*

* Reset it back to a normal scheduling class so that

* it can die in pieces.

*/

old_stop->sched_class = &rt_sched_class;

}

}

/*

* __normal_prio - return the priority that is based on the static prio

*/

static inline int __normal_prio(struct task_struct *p)

{

return p->static_prio;

}

/*

* Calculate the expected normal priority: i.e. priority

* without taking RT-inheritance into account. Might be

* boosted by interactivity modifiers. Changes upon fork,

* setprio syscalls, and whenever the interactivity

* estimator recalculates.

*/

static inline int normal_prio(struct task_struct *p)

{

int prio;

if (task_has_dl_policy(p))

prio = MAX_DL_PRIO-1;

else if (task_has_rt_policy(p))

prio = MAX_RT_PRIO-1 - p->rt_priority;

else

prio = __normal_prio(p);

return prio;

}

/*

* Calculate the current priority, i.e. the priority

* taken into account by the scheduler. This value might

* be boosted by RT tasks, or might be boosted by

* interactivity modifiers. Will be RT if the task got

* RT-boosted. If not then it returns p->normal_prio.

*/

static int effective_prio(struct task_struct *p)

{

p->normal_prio = normal_prio(p);

/*

* If we are RT tasks or we were boosted to RT priority,

* keep the priority unchanged. Otherwise, update priority

* to the normal priority:

*/

if (!rt_prio(p->prio))

return p->normal_prio;

return p->prio;

}

/**

* task_curr - is this task currently executing on a CPU?

* @p: the task in question.

*

* Return: 1 if the task is currently executing. 0 otherwise.

*/

inline int task_curr(const struct task_struct *p)

{

return cpu_curr(task_cpu(p)) == p;

}

/*

* switched_from, switched_to and prio_changed must _NOT_ drop rq->lock,

* use the balance_callback list if you want balancing.

*

* this means any call to check_class_changed() must be followed by a call to

* balance_callback().

*/

static inline void check_class_changed(struct rq *rq, struct task_struct *p,

const struct sched_class *prev_class,

int oldprio)

{

if (prev_class != p->sched_class) {

if (prev_class->switched_from)

prev_class->switched_from(rq, p);

p->sched_class->switched_to(rq, p);

} else if (oldprio != p->prio || dl_task(p))

p->sched_class->prio_changed(rq, p, oldprio);

}

void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)

{

const struct sched_class *class;

if (p->sched_class == rq->curr->sched_class) {

rq->curr->sched_class->check_preempt_curr(rq, p, flags);

} else {

for_each_class(class) {

if (class == rq->curr->sched_class)

break;

if (class == p->sched_class) {

resched_curr(rq);

break;

}

}

}

/*

* A queue event has occurred, and we're going to schedule. In

* this case, we can save a useless back to back clock update.

*/

if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))

rq_clock_skip_update(rq, true);

}

#ifdef CONFIG_SMP

/*

* This is how migration works:

*

* 1) we invoke migration_cpu_stop() on the target CPU using

* stop_one_cpu().

* 2) stopper starts to run (implicitly forcing the migrated thread

* off the CPU)

* 3) it checks whether the migrated task is still in the wrong runqueue.

* 4) if it's in the wrong runqueue then the migration thread removes

* it and puts it into the right queue.

* 5) stopper completes and stop_one_cpu() returns and the migration

* is done.

*/

/*

* move_queued_task - move a queued task to new rq.

*

* Returns (locked) new rq. Old rq's lock is released.

*/

static struct rq *move_queued_task(struct rq *rq, struct task_struct *p, int new_cpu)

{

lockdep_assert_held(&rq->lock);

p->on_rq = TASK_ON_RQ_MIGRATING;

dequeue_task(rq, p, 0);

set_task_cpu(p, new_cpu);

raw_spin_unlock(&rq->lock);

rq = cpu_rq(new_cpu);

raw_spin_lock(&rq->lock);

BUG_ON(task_cpu(p) != new_cpu);