core.c (afe06efdf07c12fd9370d5cce5383398cedf6c90) core.c (c1de45ca831acee9b72c9320dde447edafadb43f)
1/*
2 * kernel/sched/core.c
3 *
4 * Kernel scheduler and related syscalls
5 *
6 * Copyright (C) 1991-2002 Linus Torvalds
7 *
8 * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and

--- 1981 unchanged lines hidden (view full) ---

1990 */
1991
1992/**
1993 * try_to_wake_up - wake up a thread
1994 * @p: the thread to be awakened
1995 * @state: the mask of task states that can be woken
1996 * @wake_flags: wake modifier flags (WF_*)
1997 *
1/*
2 * kernel/sched/core.c
3 *
4 * Kernel scheduler and related syscalls
5 *
6 * Copyright (C) 1991-2002 Linus Torvalds
7 *
8 * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and

--- 1981 unchanged lines hidden (view full) ---

1990 */
1991
1992/**
1993 * try_to_wake_up - wake up a thread
1994 * @p: the thread to be awakened
1995 * @state: the mask of task states that can be woken
1996 * @wake_flags: wake modifier flags (WF_*)
1997 *
1998 * If (@state & @p->state) @p->state = TASK_RUNNING.
1998 * Put it on the run-queue if it's not already there. The "current"
1999 * thread is always on the run-queue (except when the actual
2000 * re-schedule is in progress), and as such you're allowed to do
2001 * the simpler "current->state = TASK_RUNNING" to mark yourself
2002 * runnable without the overhead of this.
1999 *
2003 *
2000 * If the task was not queued/runnable, also place it back on a runqueue.
2001 *
2002 * Atomic against schedule() which would dequeue a task, also see
2003 * set_current_state().
2004 *
2005 * Return: %true if @p->state changes (an actual wakeup was done),
2006 * %false otherwise.
2004 * Return: %true if @p was woken up, %false if it was already running.
2005 * or @state didn't match @p's state.
2007 */
2008static int
2009try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
2010{
2011 unsigned long flags;
2012 int cpu, success = 0;
2013
2014 /*

--- 3173 unchanged lines hidden (view full) ---

5188static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
5189
5190void sched_show_task(struct task_struct *p)
5191{
5192 unsigned long free = 0;
5193 int ppid;
5194 unsigned long state = p->state;
5195
2006 */
2007static int
2008try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
2009{
2010 unsigned long flags;
2011 int cpu, success = 0;
2012
2013 /*

--- 3173 unchanged lines hidden (view full) ---

5187static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
5188
5189void sched_show_task(struct task_struct *p)
5190{
5191 unsigned long free = 0;
5192 int ppid;
5193 unsigned long state = p->state;
5194
5196 if (!try_get_task_stack(p))
5197 return;
5198 if (state)
5199 state = __ffs(state) + 1;
5200 printk(KERN_INFO "%-15.15s %c", p->comm,
5201 state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
5195 if (state)
5196 state = __ffs(state) + 1;
5197 printk(KERN_INFO "%-15.15s %c", p->comm,
5198 state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
5199#if BITS_PER_LONG == 32
5202 if (state == TASK_RUNNING)
5200 if (state == TASK_RUNNING)
5201 printk(KERN_CONT " running ");
5202 else
5203 printk(KERN_CONT " %08lx ", thread_saved_pc(p));
5204#else
5205 if (state == TASK_RUNNING)
5203 printk(KERN_CONT " running task ");
5206 printk(KERN_CONT " running task ");
5207 else
5208 printk(KERN_CONT " %016lx ", thread_saved_pc(p));
5209#endif
5204#ifdef CONFIG_DEBUG_STACK_USAGE
5205 free = stack_not_used(p);
5206#endif
5207 ppid = 0;
5208 rcu_read_lock();
5209 if (pid_alive(p))
5210 ppid = task_pid_nr(rcu_dereference(p->real_parent));
5211 rcu_read_unlock();
5212 printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
5213 task_pid_nr(p), ppid,
5214 (unsigned long)task_thread_info(p)->flags);
5215
5216 print_worker_info(KERN_INFO, p);
5217 show_stack(p, NULL);
5210#ifdef CONFIG_DEBUG_STACK_USAGE
5211 free = stack_not_used(p);
5212#endif
5213 ppid = 0;
5214 rcu_read_lock();
5215 if (pid_alive(p))
5216 ppid = task_pid_nr(rcu_dereference(p->real_parent));
5217 rcu_read_unlock();
5218 printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
5219 task_pid_nr(p), ppid,
5220 (unsigned long)task_thread_info(p)->flags);
5221
5222 print_worker_info(KERN_INFO, p);
5223 show_stack(p, NULL);
5218 put_task_stack(p);
5219}
5220
5221void show_state_filter(unsigned long state_filter)
5222{
5223 struct task_struct *g, *p;
5224
5225#if BITS_PER_LONG == 32
5226 printk(KERN_INFO

--- 48 unchanged lines hidden (view full) ---

5275 unsigned long flags;
5276
5277 raw_spin_lock_irqsave(&idle->pi_lock, flags);
5278 raw_spin_lock(&rq->lock);
5279
5280 __sched_fork(0, idle);
5281 idle->state = TASK_RUNNING;
5282 idle->se.exec_start = sched_clock();
5224}
5225
5226void show_state_filter(unsigned long state_filter)
5227{
5228 struct task_struct *g, *p;
5229
5230#if BITS_PER_LONG == 32
5231 printk(KERN_INFO

--- 48 unchanged lines hidden (view full) ---

5280 unsigned long flags;
5281
5282 raw_spin_lock_irqsave(&idle->pi_lock, flags);
5283 raw_spin_lock(&rq->lock);
5284
5285 __sched_fork(0, idle);
5286 idle->state = TASK_RUNNING;
5287 idle->se.exec_start = sched_clock();
5288 idle->flags |= PF_IDLE;
5283
5284 kasan_unpoison_task_stack(idle);
5285
5286#ifdef CONFIG_SMP
5287 /*
5288 * Its possible that init_idle() gets called multiple times on a task,
5289 * in that case do_set_cpus_allowed() will not do the right thing.
5290 *

--- 412 unchanged lines hidden (view full) ---

5703 break;
5704 }
5705
5706 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
5707
5708 printk(KERN_CONT " %*pbl",
5709 cpumask_pr_args(sched_group_cpus(group)));
5710 if (group->sgc->capacity != SCHED_CAPACITY_SCALE) {
5289
5290 kasan_unpoison_task_stack(idle);
5291
5292#ifdef CONFIG_SMP
5293 /*
5294 * Its possible that init_idle() gets called multiple times on a task,
5295 * in that case do_set_cpus_allowed() will not do the right thing.
5296 *

--- 412 unchanged lines hidden (view full) ---

5709 break;
5710 }
5711
5712 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
5713
5714 printk(KERN_CONT " %*pbl",
5715 cpumask_pr_args(sched_group_cpus(group)));
5716 if (group->sgc->capacity != SCHED_CAPACITY_SCALE) {
5711 printk(KERN_CONT " (cpu_capacity = %lu)",
5717 printk(KERN_CONT " (cpu_capacity = %d)",
5712 group->sgc->capacity);
5713 }
5714
5715 group = group->next;
5716 } while (group != sd->groups);
5717 printk(KERN_CONT "\n");
5718
5719 if (!cpumask_equal(sched_domain_span(sd), groupmask))

--- 460 unchanged lines hidden (view full) ---

6180 build_group_mask(sd, sg);
6181
6182 /*
6183 * Initialize sgc->capacity such that even if we mess up the
6184 * domains and no possible iteration will get us here, we won't
6185 * die on a /0 trap.
6186 */
6187 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
5718 group->sgc->capacity);
5719 }
5720
5721 group = group->next;
5722 } while (group != sd->groups);
5723 printk(KERN_CONT "\n");
5724
5725 if (!cpumask_equal(sched_domain_span(sd), groupmask))

--- 460 unchanged lines hidden (view full) ---

6186 build_group_mask(sd, sg);
6187
6188 /*
6189 * Initialize sgc->capacity such that even if we mess up the
6190 * domains and no possible iteration will get us here, we won't
6191 * die on a /0 trap.
6192 */
6193 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
6188 sg->sgc->min_capacity = SCHED_CAPACITY_SCALE;
6189
6190 /*
6191 * Make sure the first group of this domain contains the
6192 * canonical balance cpu. Otherwise the sched_domain iteration
6193 * breaks. See update_sg_lb_stats().
6194 */
6195 if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
6196 group_balance_cpu(sg) == cpu)

--- 101 unchanged lines hidden (view full) ---

6298 */
6299static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
6300{
6301 struct sched_group *sg = sd->groups;
6302
6303 WARN_ON(!sg);
6304
6305 do {
6194
6195 /*
6196 * Make sure the first group of this domain contains the
6197 * canonical balance cpu. Otherwise the sched_domain iteration
6198 * breaks. See update_sg_lb_stats().
6199 */
6200 if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
6201 group_balance_cpu(sg) == cpu)

--- 101 unchanged lines hidden (view full) ---

6303 */
6304static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
6305{
6306 struct sched_group *sg = sd->groups;
6307
6308 WARN_ON(!sg);
6309
6310 do {
6306 int cpu, max_cpu = -1;
6307
6308 sg->group_weight = cpumask_weight(sched_group_cpus(sg));
6311 sg->group_weight = cpumask_weight(sched_group_cpus(sg));
6309
6310 if (!(sd->flags & SD_ASYM_PACKING))
6311 goto next;
6312
6313 for_each_cpu(cpu, sched_group_cpus(sg)) {
6314 if (max_cpu < 0)
6315 max_cpu = cpu;
6316 else if (sched_asym_prefer(cpu, max_cpu))
6317 max_cpu = cpu;
6318 }
6319 sg->asym_prefer_cpu = max_cpu;
6320
6321next:
6322 sg = sg->next;
6323 } while (sg != sd->groups);
6324
6325 if (cpu != group_balance_cpu(sg))
6326 return;
6327
6328 update_group_capacity(sd, cpu);
6329}

--- 1191 unchanged lines hidden (view full) ---

7521
7522/* Cacheline aligned slab cache for task_group */
7523static struct kmem_cache *task_group_cache __read_mostly;
7524#endif
7525
7526DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
7527DECLARE_PER_CPU(cpumask_var_t, select_idle_mask);
7528
6312 sg = sg->next;
6313 } while (sg != sd->groups);
6314
6315 if (cpu != group_balance_cpu(sg))
6316 return;
6317
6318 update_group_capacity(sd, cpu);
6319}

--- 1191 unchanged lines hidden (view full) ---

7511
7512/* Cacheline aligned slab cache for task_group */
7513static struct kmem_cache *task_group_cache __read_mostly;
7514#endif
7515
7516DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
7517DECLARE_PER_CPU(cpumask_var_t, select_idle_mask);
7518
7529#define WAIT_TABLE_BITS 8
7530#define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
7531static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
7532
7533wait_queue_head_t *bit_waitqueue(void *word, int bit)
7534{
7535 const int shift = BITS_PER_LONG == 32 ? 5 : 6;
7536 unsigned long val = (unsigned long)word << shift | bit;
7537
7538 return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
7539}
7540EXPORT_SYMBOL(bit_waitqueue);
7541
7542void __init sched_init(void)
7543{
7544 int i, j;
7545 unsigned long alloc_size = 0, ptr;
7546
7519void __init sched_init(void)
7520{
7521 int i, j;
7522 unsigned long alloc_size = 0, ptr;
7523
7547 for (i = 0; i < WAIT_TABLE_SIZE; i++)
7548 init_waitqueue_head(bit_wait_table + i);
7549
7550#ifdef CONFIG_FAIR_GROUP_SCHED
7551 alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7552#endif
7553#ifdef CONFIG_RT_GROUP_SCHED
7554 alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7555#endif
7556 if (alloc_size) {
7557 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);

--- 56 unchanged lines hidden (view full) ---

7614 rq->calc_load_active = 0;
7615 rq->calc_load_update = jiffies + LOAD_FREQ;
7616 init_cfs_rq(&rq->cfs);
7617 init_rt_rq(&rq->rt);
7618 init_dl_rq(&rq->dl);
7619#ifdef CONFIG_FAIR_GROUP_SCHED
7620 root_task_group.shares = ROOT_TASK_GROUP_LOAD;
7621 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
7524#ifdef CONFIG_FAIR_GROUP_SCHED
7525 alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7526#endif
7527#ifdef CONFIG_RT_GROUP_SCHED
7528 alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7529#endif
7530 if (alloc_size) {
7531 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);

--- 56 unchanged lines hidden (view full) ---

7588 rq->calc_load_active = 0;
7589 rq->calc_load_update = jiffies + LOAD_FREQ;
7590 init_cfs_rq(&rq->cfs);
7591 init_rt_rq(&rq->rt);
7592 init_dl_rq(&rq->dl);
7593#ifdef CONFIG_FAIR_GROUP_SCHED
7594 root_task_group.shares = ROOT_TASK_GROUP_LOAD;
7595 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
7622 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
7623 /*
7624 * How much cpu bandwidth does root_task_group get?
7625 *
7626 * In case of task-groups formed thr' the cgroup filesystem, it
7627 * gets 100% of the cpu resources in the system. This overall
7628 * system cpu resource is divided among the tasks of
7629 * root_task_group and its child task-groups in a fair manner,
7630 * based on each entity's (task or task-group's) weight

--- 1233 unchanged lines hidden --- 		7596 /*
7597 * How much cpu bandwidth does root_task_group get?
7598 *
7599 * In case of task-groups formed thr' the cgroup filesystem, it
7600 * gets 100% of the cpu resources in the system. This overall
7601 * system cpu resource is divided among the tasks of
7602 * root_task_group and its child task-groups in a fair manner,
7603 * based on each entity's (task or task-group's) weight

--- 1233 unchanged lines hidden ---