core.c revision 4a850cbefa9592ddde3670a41c10c9576a657c43
1/* 2 * kernel/sched/core.c 3 * 4 * Kernel scheduler and related syscalls 5 * 6 * Copyright (C) 1991-2002 Linus Torvalds 7 * 8 * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and 9 * make semaphores SMP safe 10 * 1998-11-19 Implemented schedule_timeout() and related stuff 11 * by Andrea Arcangeli 12 * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar: 13 * hybrid priority-list and round-robin design with 14 * an array-switch method of distributing timeslices 15 * and per-CPU runqueues. Cleanups and useful suggestions 16 * by Davide Libenzi, preemptible kernel bits by Robert Love. 17 * 2003-09-03 Interactivity tuning by Con Kolivas. 18 * 2004-04-02 Scheduler domains code by Nick Piggin 19 * 2007-04-15 Work begun on replacing all interactivity tuning with a 20 * fair scheduling design by Con Kolivas. 21 * 2007-05-05 Load balancing (smp-nice) and other improvements 22 * by Peter Williams 23 * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith 24 * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri 25 * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins, 26 * Thomas Gleixner, Mike Kravetz 27 */ 28 29#include <linux/mm.h> 30#include <linux/module.h> 31#include <linux/nmi.h> 32#include <linux/init.h> 33#include <linux/uaccess.h> 34#include <linux/highmem.h> 35#include <asm/mmu_context.h> 36#include <linux/interrupt.h> 37#include <linux/capability.h> 38#include <linux/completion.h> 39#include <linux/kernel_stat.h> 40#include <linux/debug_locks.h> 41#include <linux/perf_event.h> 42#include <linux/security.h> 43#include <linux/notifier.h> 44#include <linux/profile.h> 45#include <linux/freezer.h> 46#include <linux/vmalloc.h> 47#include <linux/blkdev.h> 48#include <linux/delay.h> 49#include <linux/pid_namespace.h> 50#include <linux/smp.h> 51#include <linux/threads.h> 52#include <linux/timer.h> 53#include <linux/rcupdate.h> 54#include <linux/cpu.h> 55#include <linux/cpuset.h> 56#include <linux/percpu.h> 57#include <linux/proc_fs.h> 58#include <linux/seq_file.h> 59#include <linux/sysctl.h> 60#include <linux/syscalls.h> 61#include <linux/times.h> 62#include <linux/tsacct_kern.h> 63#include <linux/kprobes.h> 64#include <linux/delayacct.h> 65#include <linux/unistd.h> 66#include <linux/pagemap.h> 67#include <linux/hrtimer.h> 68#include <linux/tick.h> 69#include <linux/debugfs.h> 70#include <linux/ctype.h> 71#include <linux/ftrace.h> 72#include <linux/slab.h> 73#include <linux/init_task.h> 74#include <linux/binfmts.h> 75#include <linux/context_tracking.h> 76 77#include <asm/switch_to.h> 78#include <asm/tlb.h> 79#include <asm/irq_regs.h> 80#include <asm/mutex.h> 81#ifdef CONFIG_PARAVIRT 82#include <asm/paravirt.h> 83#endif 84 85#include "sched.h" 86#include "../workqueue_internal.h" 87#include "../smpboot.h" 88 89#define CREATE_TRACE_POINTS 90#include <trace/events/sched.h> 91 92void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period) 93{ 94 unsigned long delta; 95 ktime_t soft, hard, now; 96 97 for (;;) { 98 if (hrtimer_active(period_timer)) 99 break; 100 101 now = hrtimer_cb_get_time(period_timer); 102 hrtimer_forward(period_timer, now, period); 103 104 soft = hrtimer_get_softexpires(period_timer); 105 hard = hrtimer_get_expires(period_timer); 106 delta = ktime_to_ns(ktime_sub(hard, soft)); 107 __hrtimer_start_range_ns(period_timer, soft, delta, 108 HRTIMER_MODE_ABS_PINNED, 0); 109 } 110} 111 112DEFINE_MUTEX(sched_domains_mutex); 113DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 114 115static void update_rq_clock_task(struct rq *rq, s64 delta); 116 117void update_rq_clock(struct rq *rq) 118{ 119 s64 delta; 120 121 if (rq->skip_clock_update > 0) 122 return; 123 124 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; 125 rq->clock += delta; 126 update_rq_clock_task(rq, delta); 127} 128 129/* 130 * Debugging: various feature bits 131 */ 132 133#define SCHED_FEAT(name, enabled) \ 134 (1UL << __SCHED_FEAT_##name) * enabled | 135 136const_debug unsigned int sysctl_sched_features = 137#include "features.h" 138 0; 139 140#undef SCHED_FEAT 141 142#ifdef CONFIG_SCHED_DEBUG 143#define SCHED_FEAT(name, enabled) \ 144 #name , 145 146static const char * const sched_feat_names[] = { 147#include "features.h" 148}; 149 150#undef SCHED_FEAT 151 152static int sched_feat_show(struct seq_file *m, void *v) 153{ 154 int i; 155 156 for (i = 0; i < __SCHED_FEAT_NR; i++) { 157 if (!(sysctl_sched_features & (1UL << i))) 158 seq_puts(m, "NO_"); 159 seq_printf(m, "%s ", sched_feat_names[i]); 160 } 161 seq_puts(m, "\n"); 162 163 return 0; 164} 165 166#ifdef HAVE_JUMP_LABEL 167 168#define jump_label_key__true STATIC_KEY_INIT_TRUE 169#define jump_label_key__false STATIC_KEY_INIT_FALSE 170 171#define SCHED_FEAT(name, enabled) \ 172 jump_label_key__##enabled , 173 174struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { 175#include "features.h" 176}; 177 178#undef SCHED_FEAT 179 180static void sched_feat_disable(int i) 181{ 182 if (static_key_enabled(&sched_feat_keys[i])) 183 static_key_slow_dec(&sched_feat_keys[i]); 184} 185 186static void sched_feat_enable(int i) 187{ 188 if (!static_key_enabled(&sched_feat_keys[i])) 189 static_key_slow_inc(&sched_feat_keys[i]); 190} 191#else 192static void sched_feat_disable(int i) { }; 193static void sched_feat_enable(int i) { }; 194#endif /* HAVE_JUMP_LABEL */ 195 196static int sched_feat_set(char *cmp) 197{ 198 int i; 199 int neg = 0; 200 201 if (strncmp(cmp, "NO_", 3) == 0) { 202 neg = 1; 203 cmp += 3; 204 } 205 206 for (i = 0; i < __SCHED_FEAT_NR; i++) { 207 if (strcmp(cmp, sched_feat_names[i]) == 0) { 208 if (neg) { 209 sysctl_sched_features &= ~(1UL << i); 210 sched_feat_disable(i); 211 } else { 212 sysctl_sched_features |= (1UL << i); 213 sched_feat_enable(i); 214 } 215 break; 216 } 217 } 218 219 return i; 220} 221 222static ssize_t 223sched_feat_write(struct file *filp, const char __user *ubuf, 224 size_t cnt, loff_t *ppos) 225{ 226 char buf[64]; 227 char *cmp; 228 int i; 229 230 if (cnt > 63) 231 cnt = 63; 232 233 if (copy_from_user(&buf, ubuf, cnt)) 234 return -EFAULT; 235 236 buf[cnt] = 0; 237 cmp = strstrip(buf); 238 239 i = sched_feat_set(cmp); 240 if (i == __SCHED_FEAT_NR) 241 return -EINVAL; 242 243 *ppos += cnt; 244 245 return cnt; 246} 247 248static int sched_feat_open(struct inode *inode, struct file *filp) 249{ 250 return single_open(filp, sched_feat_show, NULL); 251} 252 253static const struct file_operations sched_feat_fops = { 254 .open = sched_feat_open, 255 .write = sched_feat_write, 256 .read = seq_read, 257 .llseek = seq_lseek, 258 .release = single_release, 259}; 260 261static __init int sched_init_debug(void) 262{ 263 debugfs_create_file("sched_features", 0644, NULL, NULL, 264 &sched_feat_fops); 265 266 return 0; 267} 268late_initcall(sched_init_debug); 269#endif /* CONFIG_SCHED_DEBUG */ 270 271/* 272 * Number of tasks to iterate in a single balance run. 273 * Limited because this is done with IRQs disabled. 274 */ 275const_debug unsigned int sysctl_sched_nr_migrate = 32; 276 277/* 278 * period over which we average the RT time consumption, measured 279 * in ms. 280 * 281 * default: 1s 282 */ 283const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC; 284 285/* 286 * period over which we measure -rt task cpu usage in us. 287 * default: 1s 288 */ 289unsigned int sysctl_sched_rt_period = 1000000; 290 291__read_mostly int scheduler_running; 292 293/* 294 * part of the period that we allow rt tasks to run in us. 295 * default: 0.95s 296 */ 297int sysctl_sched_rt_runtime = 950000; 298 299 300 301/* 302 * __task_rq_lock - lock the rq @p resides on. 303 */ 304static inline struct rq *__task_rq_lock(struct task_struct *p) 305 __acquires(rq->lock) 306{ 307 struct rq *rq; 308 309 lockdep_assert_held(&p->pi_lock); 310 311 for (;;) { 312 rq = task_rq(p); 313 raw_spin_lock(&rq->lock); 314 if (likely(rq == task_rq(p))) 315 return rq; 316 raw_spin_unlock(&rq->lock); 317 } 318} 319 320/* 321 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. 322 */ 323static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags) 324 __acquires(p->pi_lock) 325 __acquires(rq->lock) 326{ 327 struct rq *rq; 328 329 for (;;) { 330 raw_spin_lock_irqsave(&p->pi_lock, *flags); 331 rq = task_rq(p); 332 raw_spin_lock(&rq->lock); 333 if (likely(rq == task_rq(p))) 334 return rq; 335 raw_spin_unlock(&rq->lock); 336 raw_spin_unlock_irqrestore(&p->pi_lock, *flags); 337 } 338} 339 340static void __task_rq_unlock(struct rq *rq) 341 __releases(rq->lock) 342{ 343 raw_spin_unlock(&rq->lock); 344} 345 346static inline void 347task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags) 348 __releases(rq->lock) 349 __releases(p->pi_lock) 350{ 351 raw_spin_unlock(&rq->lock); 352 raw_spin_unlock_irqrestore(&p->pi_lock, *flags); 353} 354 355/* 356 * this_rq_lock - lock this runqueue and disable interrupts. 357 */ 358static struct rq *this_rq_lock(void) 359 __acquires(rq->lock) 360{ 361 struct rq *rq; 362 363 local_irq_disable(); 364 rq = this_rq(); 365 raw_spin_lock(&rq->lock); 366 367 return rq; 368} 369 370#ifdef CONFIG_SCHED_HRTICK 371/* 372 * Use HR-timers to deliver accurate preemption points. 373 * 374 * Its all a bit involved since we cannot program an hrt while holding the 375 * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a 376 * reschedule event. 377 * 378 * When we get rescheduled we reprogram the hrtick_timer outside of the 379 * rq->lock. 380 */ 381 382static void hrtick_clear(struct rq *rq) 383{ 384 if (hrtimer_active(&rq->hrtick_timer)) 385 hrtimer_cancel(&rq->hrtick_timer); 386} 387 388/* 389 * High-resolution timer tick. 390 * Runs from hardirq context with interrupts disabled. 391 */ 392static enum hrtimer_restart hrtick(struct hrtimer *timer) 393{ 394 struct rq *rq = container_of(timer, struct rq, hrtick_timer); 395 396 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 397 398 raw_spin_lock(&rq->lock); 399 update_rq_clock(rq); 400 rq->curr->sched_class->task_tick(rq, rq->curr, 1); 401 raw_spin_unlock(&rq->lock); 402 403 return HRTIMER_NORESTART; 404} 405 406#ifdef CONFIG_SMP 407/* 408 * called from hardirq (IPI) context 409 */ 410static void __hrtick_start(void *arg) 411{ 412 struct rq *rq = arg; 413 414 raw_spin_lock(&rq->lock); 415 hrtimer_restart(&rq->hrtick_timer); 416 rq->hrtick_csd_pending = 0; 417 raw_spin_unlock(&rq->lock); 418} 419 420/* 421 * Called to set the hrtick timer state. 422 * 423 * called with rq->lock held and irqs disabled 424 */ 425void hrtick_start(struct rq *rq, u64 delay) 426{ 427 struct hrtimer *timer = &rq->hrtick_timer; 428 ktime_t time = ktime_add_ns(timer->base->get_time(), delay); 429 430 hrtimer_set_expires(timer, time); 431 432 if (rq == this_rq()) { 433 hrtimer_restart(timer); 434 } else if (!rq->hrtick_csd_pending) { 435 __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0); 436 rq->hrtick_csd_pending = 1; 437 } 438} 439 440static int 441hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu) 442{ 443 int cpu = (int)(long)hcpu; 444 445 switch (action) { 446 case CPU_UP_CANCELED: 447 case CPU_UP_CANCELED_FROZEN: 448 case CPU_DOWN_PREPARE: 449 case CPU_DOWN_PREPARE_FROZEN: 450 case CPU_DEAD: 451 case CPU_DEAD_FROZEN: 452 hrtick_clear(cpu_rq(cpu)); 453 return NOTIFY_OK; 454 } 455 456 return NOTIFY_DONE; 457} 458 459static __init void init_hrtick(void) 460{ 461 hotcpu_notifier(hotplug_hrtick, 0); 462} 463#else 464/* 465 * Called to set the hrtick timer state. 466 * 467 * called with rq->lock held and irqs disabled 468 */ 469void hrtick_start(struct rq *rq, u64 delay) 470{ 471 __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0, 472 HRTIMER_MODE_REL_PINNED, 0); 473} 474 475static inline void init_hrtick(void) 476{ 477} 478#endif /* CONFIG_SMP */ 479 480static void init_rq_hrtick(struct rq *rq) 481{ 482#ifdef CONFIG_SMP 483 rq->hrtick_csd_pending = 0; 484 485 rq->hrtick_csd.flags = 0; 486 rq->hrtick_csd.func = __hrtick_start; 487 rq->hrtick_csd.info = rq; 488#endif 489 490 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 491 rq->hrtick_timer.function = hrtick; 492} 493#else /* CONFIG_SCHED_HRTICK */ 494static inline void hrtick_clear(struct rq *rq) 495{ 496} 497 498static inline void init_rq_hrtick(struct rq *rq) 499{ 500} 501 502static inline void init_hrtick(void) 503{ 504} 505#endif /* CONFIG_SCHED_HRTICK */ 506 507/* 508 * resched_task - mark a task 'to be rescheduled now'. 509 * 510 * On UP this means the setting of the need_resched flag, on SMP it 511 * might also involve a cross-CPU call to trigger the scheduler on 512 * the target CPU. 513 */ 514#ifdef CONFIG_SMP 515void resched_task(struct task_struct *p) 516{ 517 int cpu; 518 519 assert_raw_spin_locked(&task_rq(p)->lock); 520 521 if (test_tsk_need_resched(p)) 522 return; 523 524 set_tsk_need_resched(p); 525 526 cpu = task_cpu(p); 527 if (cpu == smp_processor_id()) 528 return; 529 530 /* NEED_RESCHED must be visible before we test polling */ 531 smp_mb(); 532 if (!tsk_is_polling(p)) 533 smp_send_reschedule(cpu); 534} 535 536void resched_cpu(int cpu) 537{ 538 struct rq *rq = cpu_rq(cpu); 539 unsigned long flags; 540 541 if (!raw_spin_trylock_irqsave(&rq->lock, flags)) 542 return; 543 resched_task(cpu_curr(cpu)); 544 raw_spin_unlock_irqrestore(&rq->lock, flags); 545} 546 547#ifdef CONFIG_NO_HZ_COMMON 548/* 549 * In the semi idle case, use the nearest busy cpu for migrating timers 550 * from an idle cpu. This is good for power-savings. 551 * 552 * We don't do similar optimization for completely idle system, as 553 * selecting an idle cpu will add more delays to the timers than intended 554 * (as that cpu's timer base may not be uptodate wrt jiffies etc). 555 */ 556int get_nohz_timer_target(void) 557{ 558 int cpu = smp_processor_id(); 559 int i; 560 struct sched_domain *sd; 561 562 rcu_read_lock(); 563 for_each_domain(cpu, sd) { 564 for_each_cpu(i, sched_domain_span(sd)) { 565 if (!idle_cpu(i)) { 566 cpu = i; 567 goto unlock; 568 } 569 } 570 } 571unlock: 572 rcu_read_unlock(); 573 return cpu; 574} 575/* 576 * When add_timer_on() enqueues a timer into the timer wheel of an 577 * idle CPU then this timer might expire before the next timer event 578 * which is scheduled to wake up that CPU. In case of a completely 579 * idle system the next event might even be infinite time into the 580 * future. wake_up_idle_cpu() ensures that the CPU is woken up and 581 * leaves the inner idle loop so the newly added timer is taken into 582 * account when the CPU goes back to idle and evaluates the timer 583 * wheel for the next timer event. 584 */ 585static void wake_up_idle_cpu(int cpu) 586{ 587 struct rq *rq = cpu_rq(cpu); 588 589 if (cpu == smp_processor_id()) 590 return; 591 592 /* 593 * This is safe, as this function is called with the timer 594 * wheel base lock of (cpu) held. When the CPU is on the way 595 * to idle and has not yet set rq->curr to idle then it will 596 * be serialized on the timer wheel base lock and take the new 597 * timer into account automatically. 598 */ 599 if (rq->curr != rq->idle) 600 return; 601 602 /* 603 * We can set TIF_RESCHED on the idle task of the other CPU 604 * lockless. The worst case is that the other CPU runs the 605 * idle task through an additional NOOP schedule() 606 */ 607 set_tsk_need_resched(rq->idle); 608 609 /* NEED_RESCHED must be visible before we test polling */ 610 smp_mb(); 611 if (!tsk_is_polling(rq->idle)) 612 smp_send_reschedule(cpu); 613} 614 615static bool wake_up_full_nohz_cpu(int cpu) 616{ 617 if (tick_nohz_full_cpu(cpu)) { 618 if (cpu != smp_processor_id() || 619 tick_nohz_tick_stopped()) 620 smp_send_reschedule(cpu); 621 return true; 622 } 623 624 return false; 625} 626 627void wake_up_nohz_cpu(int cpu) 628{ 629 if (!wake_up_full_nohz_cpu(cpu)) 630 wake_up_idle_cpu(cpu); 631} 632 633static inline bool got_nohz_idle_kick(void) 634{ 635 int cpu = smp_processor_id(); 636 637 if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu))) 638 return false; 639 640 if (idle_cpu(cpu) && !need_resched()) 641 return true; 642 643 /* 644 * We can't run Idle Load Balance on this CPU for this time so we 645 * cancel it and clear NOHZ_BALANCE_KICK 646 */ 647 clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)); 648 return false; 649} 650 651#else /* CONFIG_NO_HZ_COMMON */ 652 653static inline bool got_nohz_idle_kick(void) 654{ 655 return false; 656} 657 658#endif /* CONFIG_NO_HZ_COMMON */ 659 660#ifdef CONFIG_NO_HZ_FULL 661bool sched_can_stop_tick(void) 662{ 663 struct rq *rq; 664 665 rq = this_rq(); 666 667 /* Make sure rq->nr_running update is visible after the IPI */ 668 smp_rmb(); 669 670 /* More than one running task need preemption */ 671 if (rq->nr_running > 1) 672 return false; 673 674 return true; 675} 676#endif /* CONFIG_NO_HZ_FULL */ 677 678void sched_avg_update(struct rq *rq) 679{ 680 s64 period = sched_avg_period(); 681 682 while ((s64)(rq_clock(rq) - rq->age_stamp) > period) { 683 /* 684 * Inline assembly required to prevent the compiler 685 * optimising this loop into a divmod call. 686 * See __iter_div_u64_rem() for another example of this. 687 */ 688 asm("" : "+rm" (rq->age_stamp)); 689 rq->age_stamp += period; 690 rq->rt_avg /= 2; 691 } 692} 693 694#else /* !CONFIG_SMP */ 695void resched_task(struct task_struct *p) 696{ 697 assert_raw_spin_locked(&task_rq(p)->lock); 698 set_tsk_need_resched(p); 699} 700#endif /* CONFIG_SMP */ 701 702#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ 703 (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) 704/* 705 * Iterate task_group tree rooted at *from, calling @down when first entering a 706 * node and @up when leaving it for the final time. 707 * 708 * Caller must hold rcu_lock or sufficient equivalent. 709 */ 710int walk_tg_tree_from(struct task_group *from, 711 tg_visitor down, tg_visitor up, void *data) 712{ 713 struct task_group *parent, *child; 714 int ret; 715 716 parent = from; 717 718down: 719 ret = (*down)(parent, data); 720 if (ret) 721 goto out; 722 list_for_each_entry_rcu(child, &parent->children, siblings) { 723 parent = child; 724 goto down; 725 726up: 727 continue; 728 } 729 ret = (*up)(parent, data); 730 if (ret || parent == from) 731 goto out; 732 733 child = parent; 734 parent = parent->parent; 735 if (parent) 736 goto up; 737out: 738 return ret; 739} 740 741int tg_nop(struct task_group *tg, void *data) 742{ 743 return 0; 744} 745#endif 746 747static void set_load_weight(struct task_struct *p) 748{ 749 int prio = p->static_prio - MAX_RT_PRIO; 750 struct load_weight *load = &p->se.load; 751 752 /* 753 * SCHED_IDLE tasks get minimal weight: 754 */ 755 if (p->policy == SCHED_IDLE) { 756 load->weight = scale_load(WEIGHT_IDLEPRIO); 757 load->inv_weight = WMULT_IDLEPRIO; 758 return; 759 } 760 761 load->weight = scale_load(prio_to_weight[prio]); 762 load->inv_weight = prio_to_wmult[prio]; 763} 764 765static void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 766{ 767 update_rq_clock(rq); 768 sched_info_queued(p); 769 p->sched_class->enqueue_task(rq, p, flags); 770} 771 772static void dequeue_task(struct rq *rq, struct task_struct *p, int flags) 773{ 774 update_rq_clock(rq); 775 sched_info_dequeued(p); 776 p->sched_class->dequeue_task(rq, p, flags); 777} 778 779void activate_task(struct rq *rq, struct task_struct *p, int flags) 780{ 781 if (task_contributes_to_load(p)) 782 rq->nr_uninterruptible--; 783 784 enqueue_task(rq, p, flags); 785} 786 787void deactivate_task(struct rq *rq, struct task_struct *p, int flags) 788{ 789 if (task_contributes_to_load(p)) 790 rq->nr_uninterruptible++; 791 792 dequeue_task(rq, p, flags); 793} 794 795static void update_rq_clock_task(struct rq *rq, s64 delta) 796{ 797/* 798 * In theory, the compile should just see 0 here, and optimize out the call 799 * to sched_rt_avg_update. But I don't trust it... 800 */ 801#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) 802 s64 steal = 0, irq_delta = 0; 803#endif 804#ifdef CONFIG_IRQ_TIME_ACCOUNTING 805 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 806 807 /* 808 * Since irq_time is only updated on {soft,}irq_exit, we might run into 809 * this case when a previous update_rq_clock() happened inside a 810 * {soft,}irq region. 811 * 812 * When this happens, we stop ->clock_task and only update the 813 * prev_irq_time stamp to account for the part that fit, so that a next 814 * update will consume the rest. This ensures ->clock_task is 815 * monotonic. 816 * 817 * It does however cause some slight miss-attribution of {soft,}irq 818 * time, a more accurate solution would be to update the irq_time using 819 * the current rq->clock timestamp, except that would require using 820 * atomic ops. 821 */ 822 if (irq_delta > delta) 823 irq_delta = delta; 824 825 rq->prev_irq_time += irq_delta; 826 delta -= irq_delta; 827#endif 828#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 829 if (static_key_false((¶virt_steal_rq_enabled))) { 830 u64 st; 831 832 steal = paravirt_steal_clock(cpu_of(rq)); 833 steal -= rq->prev_steal_time_rq; 834 835 if (unlikely(steal > delta)) 836 steal = delta; 837 838 st = steal_ticks(steal); 839 steal = st * TICK_NSEC; 840 841 rq->prev_steal_time_rq += steal; 842 843 delta -= steal; 844 } 845#endif 846 847 rq->clock_task += delta; 848 849#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) 850 if ((irq_delta + steal) && sched_feat(NONTASK_POWER)) 851 sched_rt_avg_update(rq, irq_delta + steal); 852#endif 853} 854 855void sched_set_stop_task(int cpu, struct task_struct *stop) 856{ 857 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; 858 struct task_struct *old_stop = cpu_rq(cpu)->stop; 859 860 if (stop) { 861 /* 862 * Make it appear like a SCHED_FIFO task, its something 863 * userspace knows about and won't get confused about. 864 * 865 * Also, it will make PI more or less work without too 866 * much confusion -- but then, stop work should not 867 * rely on PI working anyway. 868 */ 869 sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); 870 871 stop->sched_class = &stop_sched_class; 872 } 873 874 cpu_rq(cpu)->stop = stop; 875 876 if (old_stop) { 877 /* 878 * Reset it back to a normal scheduling class so that 879 * it can die in pieces. 880 */ 881 old_stop->sched_class = &rt_sched_class; 882 } 883} 884 885/* 886 * __normal_prio - return the priority that is based on the static prio 887 */ 888static inline int __normal_prio(struct task_struct *p) 889{ 890 return p->static_prio; 891} 892 893/* 894 * Calculate the expected normal priority: i.e. priority 895 * without taking RT-inheritance into account. Might be 896 * boosted by interactivity modifiers. Changes upon fork, 897 * setprio syscalls, and whenever the interactivity 898 * estimator recalculates. 899 */ 900static inline int normal_prio(struct task_struct *p) 901{ 902 int prio; 903 904 if (task_has_rt_policy(p)) 905 prio = MAX_RT_PRIO-1 - p->rt_priority; 906 else 907 prio = __normal_prio(p); 908 return prio; 909} 910 911/* 912 * Calculate the current priority, i.e. the priority 913 * taken into account by the scheduler. This value might 914 * be boosted by RT tasks, or might be boosted by 915 * interactivity modifiers. Will be RT if the task got 916 * RT-boosted. If not then it returns p->normal_prio. 917 */ 918static int effective_prio(struct task_struct *p) 919{ 920 p->normal_prio = normal_prio(p); 921 /* 922 * If we are RT tasks or we were boosted to RT priority, 923 * keep the priority unchanged. Otherwise, update priority 924 * to the normal priority: 925 */ 926 if (!rt_prio(p->prio)) 927 return p->normal_prio; 928 return p->prio; 929} 930 931/** 932 * task_curr - is this task currently executing on a CPU? 933 * @p: the task in question. 934 */ 935inline int task_curr(const struct task_struct *p) 936{ 937 return cpu_curr(task_cpu(p)) == p; 938} 939 940static inline void check_class_changed(struct rq *rq, struct task_struct *p, 941 const struct sched_class *prev_class, 942 int oldprio) 943{ 944 if (prev_class != p->sched_class) { 945 if (prev_class->switched_from) 946 prev_class->switched_from(rq, p); 947 p->sched_class->switched_to(rq, p); 948 } else if (oldprio != p->prio) 949 p->sched_class->prio_changed(rq, p, oldprio); 950} 951 952void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) 953{ 954 const struct sched_class *class; 955 956 if (p->sched_class == rq->curr->sched_class) { 957 rq->curr->sched_class->check_preempt_curr(rq, p, flags); 958 } else { 959 for_each_class(class) { 960 if (class == rq->curr->sched_class) 961 break; 962 if (class == p->sched_class) { 963 resched_task(rq->curr); 964 break; 965 } 966 } 967 } 968 969 /* 970 * A queue event has occurred, and we're going to schedule. In 971 * this case, we can save a useless back to back clock update. 972 */ 973 if (rq->curr->on_rq && test_tsk_need_resched(rq->curr)) 974 rq->skip_clock_update = 1; 975} 976 977static ATOMIC_NOTIFIER_HEAD(task_migration_notifier); 978 979void register_task_migration_notifier(struct notifier_block *n) 980{ 981 atomic_notifier_chain_register(&task_migration_notifier, n); 982} 983 984#ifdef CONFIG_SMP 985void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 986{ 987#ifdef CONFIG_SCHED_DEBUG 988 /* 989 * We should never call set_task_cpu() on a blocked task, 990 * ttwu() will sort out the placement. 991 */ 992 WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && 993 !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE)); 994 995#ifdef CONFIG_LOCKDEP 996 /* 997 * The caller should hold either p->pi_lock or rq->lock, when changing 998 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. 999 * 1000 * sched_move_task() holds both and thus holding either pins the cgroup, 1001 * see task_group(). 1002 * 1003 * Furthermore, all task_rq users should acquire both locks, see 1004 * task_rq_lock(). 1005 */ 1006 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || 1007 lockdep_is_held(&task_rq(p)->lock))); 1008#endif 1009#endif 1010 1011 trace_sched_migrate_task(p, new_cpu); 1012 1013 if (task_cpu(p) != new_cpu) { 1014 struct task_migration_notifier tmn; 1015 1016 if (p->sched_class->migrate_task_rq) 1017 p->sched_class->migrate_task_rq(p, new_cpu); 1018 p->se.nr_migrations++; 1019 perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0); 1020 1021 tmn.task = p; 1022 tmn.from_cpu = task_cpu(p); 1023 tmn.to_cpu = new_cpu; 1024 1025 atomic_notifier_call_chain(&task_migration_notifier, 0, &tmn); 1026 } 1027 1028 __set_task_cpu(p, new_cpu); 1029} 1030 1031struct migration_arg { 1032 struct task_struct *task; 1033 int dest_cpu; 1034}; 1035 1036static int migration_cpu_stop(void *data); 1037 1038/* 1039 * wait_task_inactive - wait for a thread to unschedule. 1040 * 1041 * If @match_state is nonzero, it's the @p->state value just checked and 1042 * not expected to change. If it changes, i.e. @p might have woken up, 1043 * then return zero. When we succeed in waiting for @p to be off its CPU, 1044 * we return a positive number (its total switch count). If a second call 1045 * a short while later returns the same number, the caller can be sure that 1046 * @p has remained unscheduled the whole time. 1047 * 1048 * The caller must ensure that the task *will* unschedule sometime soon, 1049 * else this function might spin for a *long* time. This function can't 1050 * be called with interrupts off, or it may introduce deadlock with 1051 * smp_call_function() if an IPI is sent by the same process we are 1052 * waiting to become inactive. 1053 */ 1054unsigned long wait_task_inactive(struct task_struct *p, long match_state) 1055{ 1056 unsigned long flags; 1057 int running, on_rq; 1058 unsigned long ncsw; 1059 struct rq *rq; 1060 1061 for (;;) { 1062 /* 1063 * We do the initial early heuristics without holding 1064 * any task-queue locks at all. We'll only try to get 1065 * the runqueue lock when things look like they will 1066 * work out! 1067 */ 1068 rq = task_rq(p); 1069 1070 /* 1071 * If the task is actively running on another CPU 1072 * still, just relax and busy-wait without holding 1073 * any locks. 1074 * 1075 * NOTE! Since we don't hold any locks, it's not 1076 * even sure that "rq" stays as the right runqueue! 1077 * But we don't care, since "task_running()" will 1078 * return false if the runqueue has changed and p 1079 * is actually now running somewhere else! 1080 */ 1081 while (task_running(rq, p)) { 1082 if (match_state && unlikely(p->state != match_state)) 1083 return 0; 1084 cpu_relax(); 1085 } 1086 1087 /* 1088 * Ok, time to look more closely! We need the rq 1089 * lock now, to be *sure*. If we're wrong, we'll 1090 * just go back and repeat. 1091 */ 1092 rq = task_rq_lock(p, &flags); 1093 trace_sched_wait_task(p); 1094 running = task_running(rq, p); 1095 on_rq = p->on_rq; 1096 ncsw = 0; 1097 if (!match_state || p->state == match_state) 1098 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ 1099 task_rq_unlock(rq, p, &flags); 1100 1101 /* 1102 * If it changed from the expected state, bail out now. 1103 */ 1104 if (unlikely(!ncsw)) 1105 break; 1106 1107 /* 1108 * Was it really running after all now that we 1109 * checked with the proper locks actually held? 1110 * 1111 * Oops. Go back and try again.. 1112 */ 1113 if (unlikely(running)) { 1114 cpu_relax(); 1115 continue; 1116 } 1117 1118 /* 1119 * It's not enough that it's not actively running, 1120 * it must be off the runqueue _entirely_, and not 1121 * preempted! 1122 * 1123 * So if it was still runnable (but just not actively 1124 * running right now), it's preempted, and we should 1125 * yield - it could be a while. 1126 */ 1127 if (unlikely(on_rq)) { 1128 ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ); 1129 1130 set_current_state(TASK_UNINTERRUPTIBLE); 1131 schedule_hrtimeout(&to, HRTIMER_MODE_REL); 1132 continue; 1133 } 1134 1135 /* 1136 * Ahh, all good. It wasn't running, and it wasn't 1137 * runnable, which means that it will never become 1138 * running in the future either. We're all done! 1139 */ 1140 break; 1141 } 1142 1143 return ncsw; 1144} 1145 1146/*** 1147 * kick_process - kick a running thread to enter/exit the kernel 1148 * @p: the to-be-kicked thread 1149 * 1150 * Cause a process which is running on another CPU to enter 1151 * kernel-mode, without any delay. (to get signals handled.) 1152 * 1153 * NOTE: this function doesn't have to take the runqueue lock, 1154 * because all it wants to ensure is that the remote task enters 1155 * the kernel. If the IPI races and the task has been migrated 1156 * to another CPU then no harm is done and the purpose has been 1157 * achieved as well. 1158 */ 1159void kick_process(struct task_struct *p) 1160{ 1161 int cpu; 1162 1163 preempt_disable(); 1164 cpu = task_cpu(p); 1165 if ((cpu != smp_processor_id()) && task_curr(p)) 1166 smp_send_reschedule(cpu); 1167 preempt_enable(); 1168} 1169EXPORT_SYMBOL_GPL(kick_process); 1170#endif /* CONFIG_SMP */ 1171 1172#ifdef CONFIG_SMP 1173/* 1174 * ->cpus_allowed is protected by both rq->lock and p->pi_lock 1175 */ 1176static int select_fallback_rq(int cpu, struct task_struct *p) 1177{ 1178 int nid = cpu_to_node(cpu); 1179 const struct cpumask *nodemask = NULL; 1180 enum { cpuset, possible, fail } state = cpuset; 1181 int dest_cpu; 1182 1183 /* 1184 * If the node that the cpu is on has been offlined, cpu_to_node() 1185 * will return -1. There is no cpu on the node, and we should 1186 * select the cpu on the other node. 1187 */ 1188 if (nid != -1) { 1189 nodemask = cpumask_of_node(nid); 1190 1191 /* Look for allowed, online CPU in same node. */ 1192 for_each_cpu(dest_cpu, nodemask) { 1193 if (!cpu_online(dest_cpu)) 1194 continue; 1195 if (!cpu_active(dest_cpu)) 1196 continue; 1197 if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) 1198 return dest_cpu; 1199 } 1200 } 1201 1202 for (;;) { 1203 /* Any allowed, online CPU? */ 1204 for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) { 1205 if (!cpu_online(dest_cpu)) 1206 continue; 1207 if (!cpu_active(dest_cpu)) 1208 continue; 1209 goto out; 1210 } 1211 1212 switch (state) { 1213 case cpuset: 1214 /* No more Mr. Nice Guy. */ 1215 cpuset_cpus_allowed_fallback(p); 1216 state = possible; 1217 break; 1218 1219 case possible: 1220 do_set_cpus_allowed(p, cpu_possible_mask); 1221 state = fail; 1222 break; 1223 1224 case fail: 1225 BUG(); 1226 break; 1227 } 1228 } 1229 1230out: 1231 if (state != cpuset) { 1232 /* 1233 * Don't tell them about moving exiting tasks or 1234 * kernel threads (both mm NULL), since they never 1235 * leave kernel. 1236 */ 1237 if (p->mm && printk_ratelimit()) { 1238 printk_sched("process %d (%s) no longer affine to cpu%d\n", 1239 task_pid_nr(p), p->comm, cpu); 1240 } 1241 } 1242 1243 return dest_cpu; 1244} 1245 1246/* 1247 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable. 1248 */ 1249static inline 1250int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags) 1251{ 1252 int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags); 1253 1254 /* 1255 * In order not to call set_task_cpu() on a blocking task we need 1256 * to rely on ttwu() to place the task on a valid ->cpus_allowed 1257 * cpu. 1258 * 1259 * Since this is common to all placement strategies, this lives here. 1260 * 1261 * [ this allows ->select_task() to simply return task_cpu(p) and 1262 * not worry about this generic constraint ] 1263 */ 1264 if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) || 1265 !cpu_online(cpu))) 1266 cpu = select_fallback_rq(task_cpu(p), p); 1267 1268 return cpu; 1269} 1270 1271static void update_avg(u64 *avg, u64 sample) 1272{ 1273 s64 diff = sample - *avg; 1274 *avg += diff >> 3; 1275} 1276#endif 1277 1278static void 1279ttwu_stat(struct task_struct *p, int cpu, int wake_flags) 1280{ 1281#ifdef CONFIG_SCHEDSTATS 1282 struct rq *rq = this_rq(); 1283 1284#ifdef CONFIG_SMP 1285 int this_cpu = smp_processor_id(); 1286 1287 if (cpu == this_cpu) { 1288 schedstat_inc(rq, ttwu_local); 1289 schedstat_inc(p, se.statistics.nr_wakeups_local); 1290 } else { 1291 struct sched_domain *sd; 1292 1293 schedstat_inc(p, se.statistics.nr_wakeups_remote); 1294 rcu_read_lock(); 1295 for_each_domain(this_cpu, sd) { 1296 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { 1297 schedstat_inc(sd, ttwu_wake_remote); 1298 break; 1299 } 1300 } 1301 rcu_read_unlock(); 1302 } 1303 1304 if (wake_flags & WF_MIGRATED) 1305 schedstat_inc(p, se.statistics.nr_wakeups_migrate); 1306 1307#endif /* CONFIG_SMP */ 1308 1309 schedstat_inc(rq, ttwu_count); 1310 schedstat_inc(p, se.statistics.nr_wakeups); 1311 1312 if (wake_flags & WF_SYNC) 1313 schedstat_inc(p, se.statistics.nr_wakeups_sync); 1314 1315#endif /* CONFIG_SCHEDSTATS */ 1316} 1317 1318static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags) 1319{ 1320 activate_task(rq, p, en_flags); 1321 p->on_rq = 1; 1322 1323 /* if a worker is waking up, notify workqueue */ 1324 if (p->flags & PF_WQ_WORKER) 1325 wq_worker_waking_up(p, cpu_of(rq)); 1326} 1327 1328/* 1329 * Mark the task runnable and perform wakeup-preemption. 1330 */ 1331static void 1332ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) 1333{ 1334 check_preempt_curr(rq, p, wake_flags); 1335 trace_sched_wakeup(p, true); 1336 1337 p->state = TASK_RUNNING; 1338#ifdef CONFIG_SMP 1339 if (p->sched_class->task_woken) 1340 p->sched_class->task_woken(rq, p); 1341 1342 if (rq->idle_stamp) { 1343 u64 delta = rq_clock(rq) - rq->idle_stamp; 1344 u64 max = 2*sysctl_sched_migration_cost; 1345 1346 if (delta > max) 1347 rq->avg_idle = max; 1348 else 1349 update_avg(&rq->avg_idle, delta); 1350 rq->idle_stamp = 0; 1351 } 1352#endif 1353} 1354 1355static void 1356ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags) 1357{ 1358#ifdef CONFIG_SMP 1359 if (p->sched_contributes_to_load) 1360 rq->nr_uninterruptible--; 1361#endif 1362 1363 ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING); 1364 ttwu_do_wakeup(rq, p, wake_flags); 1365} 1366 1367/* 1368 * Called in case the task @p isn't fully descheduled from its runqueue, 1369 * in this case we must do a remote wakeup. Its a 'light' wakeup though, 1370 * since all we need to do is flip p->state to TASK_RUNNING, since 1371 * the task is still ->on_rq. 1372 */ 1373static int ttwu_remote(struct task_struct *p, int wake_flags) 1374{ 1375 struct rq *rq; 1376 int ret = 0; 1377 1378 rq = __task_rq_lock(p); 1379 if (p->on_rq) { 1380 /* check_preempt_curr() may use rq clock */ 1381 update_rq_clock(rq); 1382 ttwu_do_wakeup(rq, p, wake_flags); 1383 ret = 1; 1384 } 1385 __task_rq_unlock(rq); 1386 1387 return ret; 1388} 1389 1390#ifdef CONFIG_SMP 1391static void sched_ttwu_pending(void) 1392{ 1393 struct rq *rq = this_rq(); 1394 struct llist_node *llist = llist_del_all(&rq->wake_list); 1395 struct task_struct *p; 1396 1397 raw_spin_lock(&rq->lock); 1398 1399 while (llist) { 1400 p = llist_entry(llist, struct task_struct, wake_entry); 1401 llist = llist_next(llist); 1402 ttwu_do_activate(rq, p, 0); 1403 } 1404 1405 raw_spin_unlock(&rq->lock); 1406} 1407 1408void scheduler_ipi(void) 1409{ 1410 if (llist_empty(&this_rq()->wake_list) 1411 && !tick_nohz_full_cpu(smp_processor_id()) 1412 && !got_nohz_idle_kick()) 1413 return; 1414 1415 /* 1416 * Not all reschedule IPI handlers call irq_enter/irq_exit, since 1417 * traditionally all their work was done from the interrupt return 1418 * path. Now that we actually do some work, we need to make sure 1419 * we do call them. 1420 * 1421 * Some archs already do call them, luckily irq_enter/exit nest 1422 * properly. 1423 * 1424 * Arguably we should visit all archs and update all handlers, 1425 * however a fair share of IPIs are still resched only so this would 1426 * somewhat pessimize the simple resched case. 1427 */ 1428 irq_enter(); 1429 tick_nohz_full_check(); 1430 sched_ttwu_pending(); 1431 1432 /* 1433 * Check if someone kicked us for doing the nohz idle load balance. 1434 */ 1435 if (unlikely(got_nohz_idle_kick())) { 1436 this_rq()->idle_balance = 1; 1437 raise_softirq_irqoff(SCHED_SOFTIRQ); 1438 } 1439 irq_exit(); 1440} 1441 1442static void ttwu_queue_remote(struct task_struct *p, int cpu) 1443{ 1444 if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) 1445 smp_send_reschedule(cpu); 1446} 1447 1448bool cpus_share_cache(int this_cpu, int that_cpu) 1449{ 1450 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 1451} 1452#endif /* CONFIG_SMP */ 1453 1454static void ttwu_queue(struct task_struct *p, int cpu) 1455{ 1456 struct rq *rq = cpu_rq(cpu); 1457 1458#if defined(CONFIG_SMP) 1459 if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { 1460 sched_clock_cpu(cpu); /* sync clocks x-cpu */ 1461 ttwu_queue_remote(p, cpu); 1462 return; 1463 } 1464#endif 1465 1466 raw_spin_lock(&rq->lock); 1467 ttwu_do_activate(rq, p, 0); 1468 raw_spin_unlock(&rq->lock); 1469} 1470 1471/** 1472 * try_to_wake_up - wake up a thread 1473 * @p: the thread to be awakened 1474 * @state: the mask of task states that can be woken 1475 * @wake_flags: wake modifier flags (WF_*) 1476 * 1477 * Put it on the run-queue if it's not already there. The "current" 1478 * thread is always on the run-queue (except when the actual 1479 * re-schedule is in progress), and as such you're allowed to do 1480 * the simpler "current->state = TASK_RUNNING" to mark yourself 1481 * runnable without the overhead of this. 1482 * 1483 * Returns %true if @p was woken up, %false if it was already running 1484 * or @state didn't match @p's state. 1485 */ 1486static int 1487try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 1488{ 1489 unsigned long flags; 1490 int cpu, success = 0; 1491 1492 smp_wmb(); 1493 raw_spin_lock_irqsave(&p->pi_lock, flags); 1494 if (!(p->state & state)) 1495 goto out; 1496 1497 success = 1; /* we're going to change ->state */ 1498 cpu = task_cpu(p); 1499 1500 if (p->on_rq && ttwu_remote(p, wake_flags)) 1501 goto stat; 1502 1503#ifdef CONFIG_SMP 1504 /* 1505 * If the owning (remote) cpu is still in the middle of schedule() with 1506 * this task as prev, wait until its done referencing the task. 1507 */ 1508 while (p->on_cpu) 1509 cpu_relax(); 1510 /* 1511 * Pairs with the smp_wmb() in finish_lock_switch(). 1512 */ 1513 smp_rmb(); 1514 1515 p->sched_contributes_to_load = !!task_contributes_to_load(p); 1516 p->state = TASK_WAKING; 1517 1518 if (p->sched_class->task_waking) 1519 p->sched_class->task_waking(p); 1520 1521 cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags); 1522 if (task_cpu(p) != cpu) { 1523 wake_flags |= WF_MIGRATED; 1524 set_task_cpu(p, cpu); 1525 } 1526#endif /* CONFIG_SMP */ 1527 1528 ttwu_queue(p, cpu); 1529stat: 1530 ttwu_stat(p, cpu, wake_flags); 1531out: 1532 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 1533 1534 return success; 1535} 1536 1537/** 1538 * try_to_wake_up_local - try to wake up a local task with rq lock held 1539 * @p: the thread to be awakened 1540 * 1541 * Put @p on the run-queue if it's not already there. The caller must 1542 * ensure that this_rq() is locked, @p is bound to this_rq() and not 1543 * the current task. 1544 */ 1545static void try_to_wake_up_local(struct task_struct *p) 1546{ 1547 struct rq *rq = task_rq(p); 1548 1549 if (WARN_ON_ONCE(rq != this_rq()) || 1550 WARN_ON_ONCE(p == current)) 1551 return; 1552 1553 lockdep_assert_held(&rq->lock); 1554 1555 if (!raw_spin_trylock(&p->pi_lock)) { 1556 raw_spin_unlock(&rq->lock); 1557 raw_spin_lock(&p->pi_lock); 1558 raw_spin_lock(&rq->lock); 1559 } 1560 1561 if (!(p->state & TASK_NORMAL)) 1562 goto out; 1563 1564 if (!p->on_rq) 1565 ttwu_activate(rq, p, ENQUEUE_WAKEUP); 1566 1567 ttwu_do_wakeup(rq, p, 0); 1568 ttwu_stat(p, smp_processor_id(), 0); 1569out: 1570 raw_spin_unlock(&p->pi_lock); 1571} 1572 1573/** 1574 * wake_up_process - Wake up a specific process 1575 * @p: The process to be woken up. 1576 * 1577 * Attempt to wake up the nominated process and move it to the set of runnable 1578 * processes. Returns 1 if the process was woken up, 0 if it was already 1579 * running. 1580 * 1581 * It may be assumed that this function implies a write memory barrier before 1582 * changing the task state if and only if any tasks are woken up. 1583 */ 1584int wake_up_process(struct task_struct *p) 1585{ 1586 WARN_ON(task_is_stopped_or_traced(p)); 1587 return try_to_wake_up(p, TASK_NORMAL, 0); 1588} 1589EXPORT_SYMBOL(wake_up_process); 1590 1591int wake_up_state(struct task_struct *p, unsigned int state) 1592{ 1593 return try_to_wake_up(p, state, 0); 1594} 1595 1596/* 1597 * Perform scheduler related setup for a newly forked process p. 1598 * p is forked by current. 1599 * 1600 * __sched_fork() is basic setup used by init_idle() too: 1601 */ 1602static void __sched_fork(struct task_struct *p) 1603{ 1604 p->on_rq = 0; 1605 1606 p->se.on_rq = 0; 1607 p->se.exec_start = 0; 1608 p->se.sum_exec_runtime = 0; 1609 p->se.prev_sum_exec_runtime = 0; 1610 p->se.nr_migrations = 0; 1611 p->se.vruntime = 0; 1612 INIT_LIST_HEAD(&p->se.group_node); 1613 1614/* 1615 * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be 1616 * removed when useful for applications beyond shares distribution (e.g. 1617 * load-balance). 1618 */ 1619#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) 1620 p->se.avg.runnable_avg_period = 0; 1621 p->se.avg.runnable_avg_sum = 0; 1622#endif 1623#ifdef CONFIG_SCHEDSTATS 1624 memset(&p->se.statistics, 0, sizeof(p->se.statistics)); 1625#endif 1626 1627 INIT_LIST_HEAD(&p->rt.run_list); 1628 1629#ifdef CONFIG_PREEMPT_NOTIFIERS 1630 INIT_HLIST_HEAD(&p->preempt_notifiers); 1631#endif 1632 1633#ifdef CONFIG_NUMA_BALANCING 1634 if (p->mm && atomic_read(&p->mm->mm_users) == 1) { 1635 p->mm->numa_next_scan = jiffies; 1636 p->mm->numa_next_reset = jiffies; 1637 p->mm->numa_scan_seq = 0; 1638 } 1639 1640 p->node_stamp = 0ULL; 1641 p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0; 1642 p->numa_migrate_seq = p->mm ? p->mm->numa_scan_seq - 1 : 0; 1643 p->numa_scan_period = sysctl_numa_balancing_scan_delay; 1644 p->numa_work.next = &p->numa_work; 1645#endif /* CONFIG_NUMA_BALANCING */ 1646} 1647 1648#ifdef CONFIG_NUMA_BALANCING 1649#ifdef CONFIG_SCHED_DEBUG 1650void set_numabalancing_state(bool enabled) 1651{ 1652 if (enabled) 1653 sched_feat_set("NUMA"); 1654 else 1655 sched_feat_set("NO_NUMA"); 1656} 1657#else 1658__read_mostly bool numabalancing_enabled; 1659 1660void set_numabalancing_state(bool enabled) 1661{ 1662 numabalancing_enabled = enabled; 1663} 1664#endif /* CONFIG_SCHED_DEBUG */ 1665#endif /* CONFIG_NUMA_BALANCING */ 1666 1667/* 1668 * fork()/clone()-time setup: 1669 */ 1670void sched_fork(struct task_struct *p) 1671{ 1672 unsigned long flags; 1673 int cpu = get_cpu(); 1674 1675 __sched_fork(p); 1676 /* 1677 * We mark the process as running here. This guarantees that 1678 * nobody will actually run it, and a signal or other external 1679 * event cannot wake it up and insert it on the runqueue either. 1680 */ 1681 p->state = TASK_RUNNING; 1682 1683 /* 1684 * Make sure we do not leak PI boosting priority to the child. 1685 */ 1686 p->prio = current->normal_prio; 1687 1688 /* 1689 * Revert to default priority/policy on fork if requested. 1690 */ 1691 if (unlikely(p->sched_reset_on_fork)) { 1692 if (task_has_rt_policy(p)) { 1693 p->policy = SCHED_NORMAL; 1694 p->static_prio = NICE_TO_PRIO(0); 1695 p->rt_priority = 0; 1696 } else if (PRIO_TO_NICE(p->static_prio) < 0) 1697 p->static_prio = NICE_TO_PRIO(0); 1698 1699 p->prio = p->normal_prio = __normal_prio(p); 1700 set_load_weight(p); 1701 1702 /* 1703 * We don't need the reset flag anymore after the fork. It has 1704 * fulfilled its duty: 1705 */ 1706 p->sched_reset_on_fork = 0; 1707 } 1708 1709 if (!rt_prio(p->prio)) 1710 p->sched_class = &fair_sched_class; 1711 1712 if (p->sched_class->task_fork) 1713 p->sched_class->task_fork(p); 1714 1715 /* 1716 * The child is not yet in the pid-hash so no cgroup attach races, 1717 * and the cgroup is pinned to this child due to cgroup_fork() 1718 * is ran before sched_fork(). 1719 * 1720 * Silence PROVE_RCU. 1721 */ 1722 raw_spin_lock_irqsave(&p->pi_lock, flags); 1723 set_task_cpu(p, cpu); 1724 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 1725 1726#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) 1727 if (likely(sched_info_on())) 1728 memset(&p->sched_info, 0, sizeof(p->sched_info)); 1729#endif 1730#if defined(CONFIG_SMP) 1731 p->on_cpu = 0; 1732#endif 1733#ifdef CONFIG_PREEMPT_COUNT 1734 /* Want to start with kernel preemption disabled. */ 1735 task_thread_info(p)->preempt_count = 1; 1736#endif 1737#ifdef CONFIG_SMP 1738 plist_node_init(&p->pushable_tasks, MAX_PRIO); 1739#endif 1740 1741 put_cpu(); 1742} 1743 1744/* 1745 * wake_up_new_task - wake up a newly created task for the first time. 1746 * 1747 * This function will do some initial scheduler statistics housekeeping 1748 * that must be done for every newly created context, then puts the task 1749 * on the runqueue and wakes it. 1750 */ 1751void wake_up_new_task(struct task_struct *p) 1752{ 1753 unsigned long flags; 1754 struct rq *rq; 1755 1756 raw_spin_lock_irqsave(&p->pi_lock, flags); 1757#ifdef CONFIG_SMP 1758 /* 1759 * Fork balancing, do it here and not earlier because: 1760 * - cpus_allowed can change in the fork path 1761 * - any previously selected cpu might disappear through hotplug 1762 */ 1763 set_task_cpu(p, select_task_rq(p, SD_BALANCE_FORK, 0)); 1764#endif 1765 1766 rq = __task_rq_lock(p); 1767 activate_task(rq, p, 0); 1768 p->on_rq = 1; 1769 trace_sched_wakeup_new(p, true); 1770 check_preempt_curr(rq, p, WF_FORK); 1771#ifdef CONFIG_SMP 1772 if (p->sched_class->task_woken) 1773 p->sched_class->task_woken(rq, p); 1774#endif 1775 task_rq_unlock(rq, p, &flags); 1776} 1777 1778#ifdef CONFIG_PREEMPT_NOTIFIERS 1779 1780/** 1781 * preempt_notifier_register - tell me when current is being preempted & rescheduled 1782 * @notifier: notifier struct to register 1783 */ 1784void preempt_notifier_register(struct preempt_notifier *notifier) 1785{ 1786 hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); 1787} 1788EXPORT_SYMBOL_GPL(preempt_notifier_register); 1789 1790/** 1791 * preempt_notifier_unregister - no longer interested in preemption notifications 1792 * @notifier: notifier struct to unregister 1793 * 1794 * This is safe to call from within a preemption notifier. 1795 */ 1796void preempt_notifier_unregister(struct preempt_notifier *notifier) 1797{ 1798 hlist_del(¬ifier->link); 1799} 1800EXPORT_SYMBOL_GPL(preempt_notifier_unregister); 1801 1802static void fire_sched_in_preempt_notifiers(struct task_struct *curr) 1803{ 1804 struct preempt_notifier *notifier; 1805 1806 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 1807 notifier->ops->sched_in(notifier, raw_smp_processor_id()); 1808} 1809 1810static void 1811fire_sched_out_preempt_notifiers(struct task_struct *curr, 1812 struct task_struct *next) 1813{ 1814 struct preempt_notifier *notifier; 1815 1816 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 1817 notifier->ops->sched_out(notifier, next); 1818} 1819 1820#else /* !CONFIG_PREEMPT_NOTIFIERS */ 1821 1822static void fire_sched_in_preempt_notifiers(struct task_struct *curr) 1823{ 1824} 1825 1826static void 1827fire_sched_out_preempt_notifiers(struct task_struct *curr, 1828 struct task_struct *next) 1829{ 1830} 1831 1832#endif /* CONFIG_PREEMPT_NOTIFIERS */ 1833 1834/** 1835 * prepare_task_switch - prepare to switch tasks 1836 * @rq: the runqueue preparing to switch 1837 * @prev: the current task that is being switched out 1838 * @next: the task we are going to switch to. 1839 * 1840 * This is called with the rq lock held and interrupts off. It must 1841 * be paired with a subsequent finish_task_switch after the context 1842 * switch. 1843 * 1844 * prepare_task_switch sets up locking and calls architecture specific 1845 * hooks. 1846 */ 1847static inline void 1848prepare_task_switch(struct rq *rq, struct task_struct *prev, 1849 struct task_struct *next) 1850{ 1851 trace_sched_switch(prev, next); 1852 sched_info_switch(prev, next); 1853 perf_event_task_sched_out(prev, next); 1854 fire_sched_out_preempt_notifiers(prev, next); 1855 prepare_lock_switch(rq, next); 1856 prepare_arch_switch(next); 1857} 1858 1859/** 1860 * finish_task_switch - clean up after a task-switch 1861 * @rq: runqueue associated with task-switch 1862 * @prev: the thread we just switched away from. 1863 * 1864 * finish_task_switch must be called after the context switch, paired 1865 * with a prepare_task_switch call before the context switch. 1866 * finish_task_switch will reconcile locking set up by prepare_task_switch, 1867 * and do any other architecture-specific cleanup actions. 1868 * 1869 * Note that we may have delayed dropping an mm in context_switch(). If 1870 * so, we finish that here outside of the runqueue lock. (Doing it 1871 * with the lock held can cause deadlocks; see schedule() for 1872 * details.) 1873 */ 1874static void finish_task_switch(struct rq *rq, struct task_struct *prev) 1875 __releases(rq->lock) 1876{ 1877 struct mm_struct *mm = rq->prev_mm; 1878 long prev_state; 1879 1880 rq->prev_mm = NULL; 1881 1882 /* 1883 * A task struct has one reference for the use as "current". 1884 * If a task dies, then it sets TASK_DEAD in tsk->state and calls 1885 * schedule one last time. The schedule call will never return, and 1886 * the scheduled task must drop that reference. 1887 * The test for TASK_DEAD must occur while the runqueue locks are 1888 * still held, otherwise prev could be scheduled on another cpu, die 1889 * there before we look at prev->state, and then the reference would 1890 * be dropped twice. 1891 * Manfred Spraul <manfred@colorfullife.com> 1892 */ 1893 prev_state = prev->state; 1894 vtime_task_switch(prev); 1895 finish_arch_switch(prev); 1896 perf_event_task_sched_in(prev, current); 1897 finish_lock_switch(rq, prev); 1898 finish_arch_post_lock_switch(); 1899 1900 fire_sched_in_preempt_notifiers(current); 1901 if (mm) 1902 mmdrop(mm); 1903 if (unlikely(prev_state == TASK_DEAD)) { 1904 /* 1905 * Remove function-return probe instances associated with this 1906 * task and put them back on the free list. 1907 */ 1908 kprobe_flush_task(prev); 1909 put_task_struct(prev); 1910 } 1911 1912 tick_nohz_task_switch(current); 1913} 1914 1915#ifdef CONFIG_SMP 1916 1917/* assumes rq->lock is held */ 1918static inline void pre_schedule(struct rq *rq, struct task_struct *prev) 1919{ 1920 if (prev->sched_class->pre_schedule) 1921 prev->sched_class->pre_schedule(rq, prev); 1922} 1923 1924/* rq->lock is NOT held, but preemption is disabled */ 1925static inline void post_schedule(struct rq *rq) 1926{ 1927 if (rq->post_schedule) { 1928 unsigned long flags; 1929 1930 raw_spin_lock_irqsave(&rq->lock, flags); 1931 if (rq->curr->sched_class->post_schedule) 1932 rq->curr->sched_class->post_schedule(rq); 1933 raw_spin_unlock_irqrestore(&rq->lock, flags); 1934 1935 rq->post_schedule = 0; 1936 } 1937} 1938 1939#else 1940 1941static inline void pre_schedule(struct rq *rq, struct task_struct *p) 1942{ 1943} 1944 1945static inline void post_schedule(struct rq *rq) 1946{ 1947} 1948 1949#endif 1950 1951/** 1952 * schedule_tail - first thing a freshly forked thread must call. 1953 * @prev: the thread we just switched away from. 1954 */ 1955asmlinkage void schedule_tail(struct task_struct *prev) 1956 __releases(rq->lock) 1957{ 1958 struct rq *rq = this_rq(); 1959 1960 finish_task_switch(rq, prev); 1961 1962 /* 1963 * FIXME: do we need to worry about rq being invalidated by the 1964 * task_switch? 1965 */ 1966 post_schedule(rq); 1967 1968#ifdef __ARCH_WANT_UNLOCKED_CTXSW 1969 /* In this case, finish_task_switch does not reenable preemption */ 1970 preempt_enable(); 1971#endif 1972 if (current->set_child_tid) 1973 put_user(task_pid_vnr(current), current->set_child_tid); 1974} 1975 1976/* 1977 * context_switch - switch to the new MM and the new 1978 * thread's register state. 1979 */ 1980static inline void 1981context_switch(struct rq *rq, struct task_struct *prev, 1982 struct task_struct *next) 1983{ 1984 struct mm_struct *mm, *oldmm; 1985 1986 prepare_task_switch(rq, prev, next); 1987 1988 mm = next->mm; 1989 oldmm = prev->active_mm; 1990 /* 1991 * For paravirt, this is coupled with an exit in switch_to to 1992 * combine the page table reload and the switch backend into 1993 * one hypercall. 1994 */ 1995 arch_start_context_switch(prev); 1996 1997 if (!mm) { 1998 next->active_mm = oldmm; 1999 atomic_inc(&oldmm->mm_count); 2000 enter_lazy_tlb(oldmm, next); 2001 } else 2002 switch_mm(oldmm, mm, next); 2003 2004 if (!prev->mm) { 2005 prev->active_mm = NULL; 2006 rq->prev_mm = oldmm; 2007 } 2008 /* 2009 * Since the runqueue lock will be released by the next 2010 * task (which is an invalid locking op but in the case 2011 * of the scheduler it's an obvious special-case), so we 2012 * do an early lockdep release here: 2013 */ 2014#ifndef __ARCH_WANT_UNLOCKED_CTXSW 2015 spin_release(&rq->lock.dep_map, 1, _THIS_IP_); 2016#endif 2017 2018 context_tracking_task_switch(prev, next); 2019 /* Here we just switch the register state and the stack. */ 2020 switch_to(prev, next, prev); 2021 2022 barrier(); 2023 /* 2024 * this_rq must be evaluated again because prev may have moved 2025 * CPUs since it called schedule(), thus the 'rq' on its stack 2026 * frame will be invalid. 2027 */ 2028 finish_task_switch(this_rq(), prev); 2029} 2030 2031/* 2032 * nr_running and nr_context_switches: 2033 * 2034 * externally visible scheduler statistics: current number of runnable 2035 * threads, total number of context switches performed since bootup. 2036 */ 2037unsigned long nr_running(void) 2038{ 2039 unsigned long i, sum = 0; 2040 2041 for_each_online_cpu(i) 2042 sum += cpu_rq(i)->nr_running; 2043 2044 return sum; 2045} 2046 2047unsigned long long nr_context_switches(void) 2048{ 2049 int i; 2050 unsigned long long sum = 0; 2051 2052 for_each_possible_cpu(i) 2053 sum += cpu_rq(i)->nr_switches; 2054 2055 return sum; 2056} 2057 2058unsigned long nr_iowait(void) 2059{ 2060 unsigned long i, sum = 0; 2061 2062 for_each_possible_cpu(i) 2063 sum += atomic_read(&cpu_rq(i)->nr_iowait); 2064 2065 return sum; 2066} 2067 2068unsigned long nr_iowait_cpu(int cpu) 2069{ 2070 struct rq *this = cpu_rq(cpu); 2071 return atomic_read(&this->nr_iowait); 2072} 2073 2074#ifdef CONFIG_SMP 2075 2076/* 2077 * sched_exec - execve() is a valuable balancing opportunity, because at 2078 * this point the task has the smallest effective memory and cache footprint. 2079 */ 2080void sched_exec(void) 2081{ 2082 struct task_struct *p = current; 2083 unsigned long flags; 2084 int dest_cpu; 2085 2086 raw_spin_lock_irqsave(&p->pi_lock, flags); 2087 dest_cpu = p->sched_class->select_task_rq(p, SD_BALANCE_EXEC, 0); 2088 if (dest_cpu == smp_processor_id()) 2089 goto unlock; 2090 2091 if (likely(cpu_active(dest_cpu))) { 2092 struct migration_arg arg = { p, dest_cpu }; 2093 2094 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 2095 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); 2096 return; 2097 } 2098unlock: 2099 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 2100} 2101 2102#endif 2103 2104DEFINE_PER_CPU(struct kernel_stat, kstat); 2105DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); 2106 2107EXPORT_PER_CPU_SYMBOL(kstat); 2108EXPORT_PER_CPU_SYMBOL(kernel_cpustat); 2109 2110/* 2111 * Return any ns on the sched_clock that have not yet been accounted in 2112 * @p in case that task is currently running. 2113 * 2114 * Called with task_rq_lock() held on @rq. 2115 */ 2116static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq) 2117{ 2118 u64 ns = 0; 2119 2120 if (task_current(rq, p)) { 2121 update_rq_clock(rq); 2122 ns = rq_clock_task(rq) - p->se.exec_start; 2123 if ((s64)ns < 0) 2124 ns = 0; 2125 } 2126 2127 return ns; 2128} 2129 2130unsigned long long task_delta_exec(struct task_struct *p) 2131{ 2132 unsigned long flags; 2133 struct rq *rq; 2134 u64 ns = 0; 2135 2136 rq = task_rq_lock(p, &flags); 2137 ns = do_task_delta_exec(p, rq); 2138 task_rq_unlock(rq, p, &flags); 2139 2140 return ns; 2141} 2142 2143/* 2144 * Return accounted runtime for the task. 2145 * In case the task is currently running, return the runtime plus current's 2146 * pending runtime that have not been accounted yet. 2147 */ 2148unsigned long long task_sched_runtime(struct task_struct *p) 2149{ 2150 unsigned long flags; 2151 struct rq *rq; 2152 u64 ns = 0; 2153 2154 rq = task_rq_lock(p, &flags); 2155 ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq); 2156 task_rq_unlock(rq, p, &flags); 2157 2158 return ns; 2159} 2160 2161/* 2162 * This function gets called by the timer code, with HZ frequency. 2163 * We call it with interrupts disabled. 2164 */ 2165void scheduler_tick(void) 2166{ 2167 int cpu = smp_processor_id(); 2168 struct rq *rq = cpu_rq(cpu); 2169 struct task_struct *curr = rq->curr; 2170 2171 sched_clock_tick(); 2172 2173 raw_spin_lock(&rq->lock); 2174 update_rq_clock(rq); 2175 update_cpu_load_active(rq); 2176 curr->sched_class->task_tick(rq, curr, 0); 2177 raw_spin_unlock(&rq->lock); 2178 2179 perf_event_task_tick(); 2180 2181#ifdef CONFIG_SMP 2182 rq->idle_balance = idle_cpu(cpu); 2183 trigger_load_balance(rq, cpu); 2184#endif 2185 rq_last_tick_reset(rq); 2186} 2187 2188#ifdef CONFIG_NO_HZ_FULL 2189/** 2190 * scheduler_tick_max_deferment 2191 * 2192 * Keep at least one tick per second when a single 2193 * active task is running because the scheduler doesn't 2194 * yet completely support full dynticks environment. 2195 * 2196 * This makes sure that uptime, CFS vruntime, load 2197 * balancing, etc... continue to move forward, even 2198 * with a very low granularity. 2199 */ 2200u64 scheduler_tick_max_deferment(void) 2201{ 2202 struct rq *rq = this_rq(); 2203 unsigned long next, now = ACCESS_ONCE(jiffies); 2204 2205 next = rq->last_sched_tick + HZ; 2206 2207 if (time_before_eq(next, now)) 2208 return 0; 2209 2210 return jiffies_to_usecs(next - now) * NSEC_PER_USEC; 2211} 2212#endif 2213 2214notrace unsigned long get_parent_ip(unsigned long addr) 2215{ 2216 if (in_lock_functions(addr)) { 2217 addr = CALLER_ADDR2; 2218 if (in_lock_functions(addr)) 2219 addr = CALLER_ADDR3; 2220 } 2221 return addr; 2222} 2223 2224#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ 2225 defined(CONFIG_PREEMPT_TRACER)) 2226 2227void __kprobes add_preempt_count(int val) 2228{ 2229#ifdef CONFIG_DEBUG_PREEMPT 2230 /* 2231 * Underflow? 2232 */ 2233 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) 2234 return; 2235#endif 2236 preempt_count() += val; 2237#ifdef CONFIG_DEBUG_PREEMPT 2238 /* 2239 * Spinlock count overflowing soon? 2240 */ 2241 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= 2242 PREEMPT_MASK - 10); 2243#endif 2244 if (preempt_count() == val) 2245 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); 2246} 2247EXPORT_SYMBOL(add_preempt_count); 2248 2249void __kprobes sub_preempt_count(int val) 2250{ 2251#ifdef CONFIG_DEBUG_PREEMPT 2252 /* 2253 * Underflow? 2254 */ 2255 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 2256 return; 2257 /* 2258 * Is the spinlock portion underflowing? 2259 */ 2260 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && 2261 !(preempt_count() & PREEMPT_MASK))) 2262 return; 2263#endif 2264 2265 if (preempt_count() == val) 2266 trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); 2267 preempt_count() -= val; 2268} 2269EXPORT_SYMBOL(sub_preempt_count); 2270 2271#endif 2272 2273/* 2274 * Print scheduling while atomic bug: 2275 */ 2276static noinline void __schedule_bug(struct task_struct *prev) 2277{ 2278 if (oops_in_progress) 2279 return; 2280 2281 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", 2282 prev->comm, prev->pid, preempt_count()); 2283 2284 debug_show_held_locks(prev); 2285 print_modules(); 2286 if (irqs_disabled()) 2287 print_irqtrace_events(prev); 2288 dump_stack(); 2289 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 2290} 2291 2292/* 2293 * Various schedule()-time debugging checks and statistics: 2294 */ 2295static inline void schedule_debug(struct task_struct *prev) 2296{ 2297 /* 2298 * Test if we are atomic. Since do_exit() needs to call into 2299 * schedule() atomically, we ignore that path for now. 2300 * Otherwise, whine if we are scheduling when we should not be. 2301 */ 2302 if (unlikely(in_atomic_preempt_off() && !prev->exit_state)) 2303 __schedule_bug(prev); 2304 rcu_sleep_check(); 2305 2306 profile_hit(SCHED_PROFILING, __builtin_return_address(0)); 2307 2308 schedstat_inc(this_rq(), sched_count); 2309} 2310 2311static void put_prev_task(struct rq *rq, struct task_struct *prev) 2312{ 2313 if (prev->on_rq || rq->skip_clock_update < 0) 2314 update_rq_clock(rq); 2315 prev->sched_class->put_prev_task(rq, prev); 2316} 2317 2318/* 2319 * Pick up the highest-prio task: 2320 */ 2321static inline struct task_struct * 2322pick_next_task(struct rq *rq) 2323{ 2324 const struct sched_class *class; 2325 struct task_struct *p; 2326 2327 /* 2328 * Optimization: we know that if all tasks are in 2329 * the fair class we can call that function directly: 2330 */ 2331 if (likely(rq->nr_running == rq->cfs.h_nr_running)) { 2332 p = fair_sched_class.pick_next_task(rq); 2333 if (likely(p)) 2334 return p; 2335 } 2336 2337 for_each_class(class) { 2338 p = class->pick_next_task(rq); 2339 if (p) 2340 return p; 2341 } 2342 2343 BUG(); /* the idle class will always have a runnable task */ 2344} 2345 2346/* 2347 * __schedule() is the main scheduler function. 2348 * 2349 * The main means of driving the scheduler and thus entering this function are: 2350 * 2351 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. 2352 * 2353 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return 2354 * paths. For example, see arch/x86/entry_64.S. 2355 * 2356 * To drive preemption between tasks, the scheduler sets the flag in timer 2357 * interrupt handler scheduler_tick(). 2358 * 2359 * 3. Wakeups don't really cause entry into schedule(). They add a 2360 * task to the run-queue and that's it. 2361 * 2362 * Now, if the new task added to the run-queue preempts the current 2363 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets 2364 * called on the nearest possible occasion: 2365 * 2366 * - If the kernel is preemptible (CONFIG_PREEMPT=y): 2367 * 2368 * - in syscall or exception context, at the next outmost 2369 * preempt_enable(). (this might be as soon as the wake_up()'s 2370 * spin_unlock()!) 2371 * 2372 * - in IRQ context, return from interrupt-handler to 2373 * preemptible context 2374 * 2375 * - If the kernel is not preemptible (CONFIG_PREEMPT is not set) 2376 * then at the next: 2377 * 2378 * - cond_resched() call 2379 * - explicit schedule() call 2380 * - return from syscall or exception to user-space 2381 * - return from interrupt-handler to user-space 2382 */ 2383static void __sched __schedule(void) 2384{ 2385 struct task_struct *prev, *next; 2386 unsigned long *switch_count; 2387 struct rq *rq; 2388 int cpu; 2389 2390need_resched: 2391 preempt_disable(); 2392 cpu = smp_processor_id(); 2393 rq = cpu_rq(cpu); 2394 rcu_note_context_switch(cpu); 2395 prev = rq->curr; 2396 2397 schedule_debug(prev); 2398 2399 if (sched_feat(HRTICK)) 2400 hrtick_clear(rq); 2401 2402 raw_spin_lock_irq(&rq->lock); 2403 2404 switch_count = &prev->nivcsw; 2405 if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { 2406 if (unlikely(signal_pending_state(prev->state, prev))) { 2407 prev->state = TASK_RUNNING; 2408 } else { 2409 deactivate_task(rq, prev, DEQUEUE_SLEEP); 2410 prev->on_rq = 0; 2411 2412 /* 2413 * If a worker went to sleep, notify and ask workqueue 2414 * whether it wants to wake up a task to maintain 2415 * concurrency. 2416 */ 2417 if (prev->flags & PF_WQ_WORKER) { 2418 struct task_struct *to_wakeup; 2419 2420 to_wakeup = wq_worker_sleeping(prev, cpu); 2421 if (to_wakeup) 2422 try_to_wake_up_local(to_wakeup); 2423 } 2424 } 2425 switch_count = &prev->nvcsw; 2426 } 2427 2428 pre_schedule(rq, prev); 2429 2430 if (unlikely(!rq->nr_running)) 2431 idle_balance(cpu, rq); 2432 2433 put_prev_task(rq, prev); 2434 next = pick_next_task(rq); 2435 clear_tsk_need_resched(prev); 2436 rq->skip_clock_update = 0; 2437 2438 if (likely(prev != next)) { 2439 rq->nr_switches++; 2440 rq->curr = next; 2441 ++*switch_count; 2442 2443 context_switch(rq, prev, next); /* unlocks the rq */ 2444 /* 2445 * The context switch have flipped the stack from under us 2446 * and restored the local variables which were saved when 2447 * this task called schedule() in the past. prev == current 2448 * is still correct, but it can be moved to another cpu/rq. 2449 */ 2450 cpu = smp_processor_id(); 2451 rq = cpu_rq(cpu); 2452 } else 2453 raw_spin_unlock_irq(&rq->lock); 2454 2455 post_schedule(rq); 2456 2457 sched_preempt_enable_no_resched(); 2458 if (need_resched()) 2459 goto need_resched; 2460} 2461 2462static inline void sched_submit_work(struct task_struct *tsk) 2463{ 2464 if (!tsk->state || tsk_is_pi_blocked(tsk)) 2465 return; 2466 /* 2467 * If we are going to sleep and we have plugged IO queued, 2468 * make sure to submit it to avoid deadlocks. 2469 */ 2470 if (blk_needs_flush_plug(tsk)) 2471 blk_schedule_flush_plug(tsk); 2472} 2473 2474asmlinkage void __sched schedule(void) 2475{ 2476 struct task_struct *tsk = current; 2477 2478 sched_submit_work(tsk); 2479 __schedule(); 2480} 2481EXPORT_SYMBOL(schedule); 2482 2483#ifdef CONFIG_CONTEXT_TRACKING 2484asmlinkage void __sched schedule_user(void) 2485{ 2486 /* 2487 * If we come here after a random call to set_need_resched(), 2488 * or we have been woken up remotely but the IPI has not yet arrived, 2489 * we haven't yet exited the RCU idle mode. Do it here manually until 2490 * we find a better solution. 2491 */ 2492 user_exit(); 2493 schedule(); 2494 user_enter(); 2495} 2496#endif 2497 2498/** 2499 * schedule_preempt_disabled - called with preemption disabled 2500 * 2501 * Returns with preemption disabled. Note: preempt_count must be 1 2502 */ 2503void __sched schedule_preempt_disabled(void) 2504{ 2505 sched_preempt_enable_no_resched(); 2506 schedule(); 2507 preempt_disable(); 2508} 2509 2510#ifdef CONFIG_PREEMPT 2511/* 2512 * this is the entry point to schedule() from in-kernel preemption 2513 * off of preempt_enable. Kernel preemptions off return from interrupt 2514 * occur there and call schedule directly. 2515 */ 2516asmlinkage void __sched notrace preempt_schedule(void) 2517{ 2518 struct thread_info *ti = current_thread_info(); 2519 2520 /* 2521 * If there is a non-zero preempt_count or interrupts are disabled, 2522 * we do not want to preempt the current task. Just return.. 2523 */ 2524 if (likely(ti->preempt_count || irqs_disabled())) 2525 return; 2526 2527 do { 2528 add_preempt_count_notrace(PREEMPT_ACTIVE); 2529 __schedule(); 2530 sub_preempt_count_notrace(PREEMPT_ACTIVE); 2531 2532 /* 2533 * Check again in case we missed a preemption opportunity 2534 * between schedule and now. 2535 */ 2536 barrier(); 2537 } while (need_resched()); 2538} 2539EXPORT_SYMBOL(preempt_schedule); 2540 2541/* 2542 * this is the entry point to schedule() from kernel preemption 2543 * off of irq context. 2544 * Note, that this is called and return with irqs disabled. This will 2545 * protect us against recursive calling from irq. 2546 */ 2547asmlinkage void __sched preempt_schedule_irq(void) 2548{ 2549 struct thread_info *ti = current_thread_info(); 2550 enum ctx_state prev_state; 2551 2552 /* Catch callers which need to be fixed */ 2553 BUG_ON(ti->preempt_count || !irqs_disabled()); 2554 2555 prev_state = exception_enter(); 2556 2557 do { 2558 add_preempt_count(PREEMPT_ACTIVE); 2559 local_irq_enable(); 2560 __schedule(); 2561 local_irq_disable(); 2562 sub_preempt_count(PREEMPT_ACTIVE); 2563 2564 /* 2565 * Check again in case we missed a preemption opportunity 2566 * between schedule and now. 2567 */ 2568 barrier(); 2569 } while (need_resched()); 2570 2571 exception_exit(prev_state); 2572} 2573 2574#endif /* CONFIG_PREEMPT */ 2575 2576int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags, 2577 void *key) 2578{ 2579 return try_to_wake_up(curr->private, mode, wake_flags); 2580} 2581EXPORT_SYMBOL(default_wake_function); 2582 2583/* 2584 * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just 2585 * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve 2586 * number) then we wake all the non-exclusive tasks and one exclusive task. 2587 * 2588 * There are circumstances in which we can try to wake a task which has already 2589 * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns 2590 * zero in this (rare) case, and we handle it by continuing to scan the queue. 2591 */ 2592static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, 2593 int nr_exclusive, int wake_flags, void *key) 2594{ 2595 wait_queue_t *curr, *next; 2596 2597 list_for_each_entry_safe(curr, next, &q->task_list, task_list) { 2598 unsigned flags = curr->flags; 2599 2600 if (curr->func(curr, mode, wake_flags, key) && 2601 (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) 2602 break; 2603 } 2604} 2605 2606/** 2607 * __wake_up - wake up threads blocked on a waitqueue. 2608 * @q: the waitqueue 2609 * @mode: which threads 2610 * @nr_exclusive: how many wake-one or wake-many threads to wake up 2611 * @key: is directly passed to the wakeup function 2612 * 2613 * It may be assumed that this function implies a write memory barrier before 2614 * changing the task state if and only if any tasks are woken up. 2615 */ 2616void __wake_up(wait_queue_head_t *q, unsigned int mode, 2617 int nr_exclusive, void *key) 2618{ 2619 unsigned long flags; 2620 2621 spin_lock_irqsave(&q->lock, flags); 2622 __wake_up_common(q, mode, nr_exclusive, 0, key); 2623 spin_unlock_irqrestore(&q->lock, flags); 2624} 2625EXPORT_SYMBOL(__wake_up); 2626 2627/* 2628 * Same as __wake_up but called with the spinlock in wait_queue_head_t held. 2629 */ 2630void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr) 2631{ 2632 __wake_up_common(q, mode, nr, 0, NULL); 2633} 2634EXPORT_SYMBOL_GPL(__wake_up_locked); 2635 2636void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key) 2637{ 2638 __wake_up_common(q, mode, 1, 0, key); 2639} 2640EXPORT_SYMBOL_GPL(__wake_up_locked_key); 2641 2642/** 2643 * __wake_up_sync_key - wake up threads blocked on a waitqueue. 2644 * @q: the waitqueue 2645 * @mode: which threads 2646 * @nr_exclusive: how many wake-one or wake-many threads to wake up 2647 * @key: opaque value to be passed to wakeup targets 2648 * 2649 * The sync wakeup differs that the waker knows that it will schedule 2650 * away soon, so while the target thread will be woken up, it will not 2651 * be migrated to another CPU - ie. the two threads are 'synchronized' 2652 * with each other. This can prevent needless bouncing between CPUs. 2653 * 2654 * On UP it can prevent extra preemption. 2655 * 2656 * It may be assumed that this function implies a write memory barrier before 2657 * changing the task state if and only if any tasks are woken up. 2658 */ 2659void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, 2660 int nr_exclusive, void *key) 2661{ 2662 unsigned long flags; 2663 int wake_flags = WF_SYNC; 2664 2665 if (unlikely(!q)) 2666 return; 2667 2668 if (unlikely(!nr_exclusive)) 2669 wake_flags = 0; 2670 2671 spin_lock_irqsave(&q->lock, flags); 2672 __wake_up_common(q, mode, nr_exclusive, wake_flags, key); 2673 spin_unlock_irqrestore(&q->lock, flags); 2674} 2675EXPORT_SYMBOL_GPL(__wake_up_sync_key); 2676 2677/* 2678 * __wake_up_sync - see __wake_up_sync_key() 2679 */ 2680void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) 2681{ 2682 __wake_up_sync_key(q, mode, nr_exclusive, NULL); 2683} 2684EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ 2685 2686/** 2687 * complete: - signals a single thread waiting on this completion 2688 * @x: holds the state of this particular completion 2689 * 2690 * This will wake up a single thread waiting on this completion. Threads will be 2691 * awakened in the same order in which they were queued. 2692 * 2693 * See also complete_all(), wait_for_completion() and related routines. 2694 * 2695 * It may be assumed that this function implies a write memory barrier before 2696 * changing the task state if and only if any tasks are woken up. 2697 */ 2698void complete(struct completion *x) 2699{ 2700 unsigned long flags; 2701 2702 spin_lock_irqsave(&x->wait.lock, flags); 2703 x->done++; 2704 __wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL); 2705 spin_unlock_irqrestore(&x->wait.lock, flags); 2706} 2707EXPORT_SYMBOL(complete); 2708 2709/** 2710 * complete_all: - signals all threads waiting on this completion 2711 * @x: holds the state of this particular completion 2712 * 2713 * This will wake up all threads waiting on this particular completion event. 2714 * 2715 * It may be assumed that this function implies a write memory barrier before 2716 * changing the task state if and only if any tasks are woken up. 2717 */ 2718void complete_all(struct completion *x) 2719{ 2720 unsigned long flags; 2721 2722 spin_lock_irqsave(&x->wait.lock, flags); 2723 x->done += UINT_MAX/2; 2724 __wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL); 2725 spin_unlock_irqrestore(&x->wait.lock, flags); 2726} 2727EXPORT_SYMBOL(complete_all); 2728 2729static inline long __sched 2730do_wait_for_common(struct completion *x, 2731 long (*action)(long), long timeout, int state) 2732{ 2733 if (!x->done) { 2734 DECLARE_WAITQUEUE(wait, current); 2735 2736 __add_wait_queue_tail_exclusive(&x->wait, &wait); 2737 do { 2738 if (signal_pending_state(state, current)) { 2739 timeout = -ERESTARTSYS; 2740 break; 2741 } 2742 __set_current_state(state); 2743 spin_unlock_irq(&x->wait.lock); 2744 timeout = action(timeout); 2745 spin_lock_irq(&x->wait.lock); 2746 } while (!x->done && timeout); 2747 __remove_wait_queue(&x->wait, &wait); 2748 if (!x->done) 2749 return timeout; 2750 } 2751 x->done--; 2752 return timeout ?: 1; 2753} 2754 2755static inline long __sched 2756__wait_for_common(struct completion *x, 2757 long (*action)(long), long timeout, int state) 2758{ 2759 might_sleep(); 2760 2761 spin_lock_irq(&x->wait.lock); 2762 timeout = do_wait_for_common(x, action, timeout, state); 2763 spin_unlock_irq(&x->wait.lock); 2764 return timeout; 2765} 2766 2767static long __sched 2768wait_for_common(struct completion *x, long timeout, int state) 2769{ 2770 return __wait_for_common(x, schedule_timeout, timeout, state); 2771} 2772 2773static long __sched 2774wait_for_common_io(struct completion *x, long timeout, int state) 2775{ 2776 return __wait_for_common(x, io_schedule_timeout, timeout, state); 2777} 2778 2779/** 2780 * wait_for_completion: - waits for completion of a task 2781 * @x: holds the state of this particular completion 2782 * 2783 * This waits to be signaled for completion of a specific task. It is NOT 2784 * interruptible and there is no timeout. 2785 * 2786 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout 2787 * and interrupt capability. Also see complete(). 2788 */ 2789void __sched wait_for_completion(struct completion *x) 2790{ 2791 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); 2792} 2793EXPORT_SYMBOL(wait_for_completion); 2794 2795/** 2796 * wait_for_completion_timeout: - waits for completion of a task (w/timeout) 2797 * @x: holds the state of this particular completion 2798 * @timeout: timeout value in jiffies 2799 * 2800 * This waits for either a completion of a specific task to be signaled or for a 2801 * specified timeout to expire. The timeout is in jiffies. It is not 2802 * interruptible. 2803 * 2804 * The return value is 0 if timed out, and positive (at least 1, or number of 2805 * jiffies left till timeout) if completed. 2806 */ 2807unsigned long __sched 2808wait_for_completion_timeout(struct completion *x, unsigned long timeout) 2809{ 2810 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); 2811} 2812EXPORT_SYMBOL(wait_for_completion_timeout); 2813 2814/** 2815 * wait_for_completion_io: - waits for completion of a task 2816 * @x: holds the state of this particular completion 2817 * 2818 * This waits to be signaled for completion of a specific task. It is NOT 2819 * interruptible and there is no timeout. The caller is accounted as waiting 2820 * for IO. 2821 */ 2822void __sched wait_for_completion_io(struct completion *x) 2823{ 2824 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); 2825} 2826EXPORT_SYMBOL(wait_for_completion_io); 2827 2828/** 2829 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout) 2830 * @x: holds the state of this particular completion 2831 * @timeout: timeout value in jiffies 2832 * 2833 * This waits for either a completion of a specific task to be signaled or for a 2834 * specified timeout to expire. The timeout is in jiffies. It is not 2835 * interruptible. The caller is accounted as waiting for IO. 2836 * 2837 * The return value is 0 if timed out, and positive (at least 1, or number of 2838 * jiffies left till timeout) if completed. 2839 */ 2840unsigned long __sched 2841wait_for_completion_io_timeout(struct completion *x, unsigned long timeout) 2842{ 2843 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE); 2844} 2845EXPORT_SYMBOL(wait_for_completion_io_timeout); 2846 2847/** 2848 * wait_for_completion_interruptible: - waits for completion of a task (w/intr) 2849 * @x: holds the state of this particular completion 2850 * 2851 * This waits for completion of a specific task to be signaled. It is 2852 * interruptible. 2853 * 2854 * The return value is -ERESTARTSYS if interrupted, 0 if completed. 2855 */ 2856int __sched wait_for_completion_interruptible(struct completion *x) 2857{ 2858 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); 2859 if (t == -ERESTARTSYS) 2860 return t; 2861 return 0; 2862} 2863EXPORT_SYMBOL(wait_for_completion_interruptible); 2864 2865/** 2866 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr)) 2867 * @x: holds the state of this particular completion 2868 * @timeout: timeout value in jiffies 2869 * 2870 * This waits for either a completion of a specific task to be signaled or for a 2871 * specified timeout to expire. It is interruptible. The timeout is in jiffies. 2872 * 2873 * The return value is -ERESTARTSYS if interrupted, 0 if timed out, 2874 * positive (at least 1, or number of jiffies left till timeout) if completed. 2875 */ 2876long __sched 2877wait_for_completion_interruptible_timeout(struct completion *x, 2878 unsigned long timeout) 2879{ 2880 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE); 2881} 2882EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); 2883 2884/** 2885 * wait_for_completion_killable: - waits for completion of a task (killable) 2886 * @x: holds the state of this particular completion 2887 * 2888 * This waits to be signaled for completion of a specific task. It can be 2889 * interrupted by a kill signal. 2890 * 2891 * The return value is -ERESTARTSYS if interrupted, 0 if completed. 2892 */ 2893int __sched wait_for_completion_killable(struct completion *x) 2894{ 2895 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); 2896 if (t == -ERESTARTSYS) 2897 return t; 2898 return 0; 2899} 2900EXPORT_SYMBOL(wait_for_completion_killable); 2901 2902/** 2903 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable)) 2904 * @x: holds the state of this particular completion 2905 * @timeout: timeout value in jiffies 2906 * 2907 * This waits for either a completion of a specific task to be 2908 * signaled or for a specified timeout to expire. It can be 2909 * interrupted by a kill signal. The timeout is in jiffies. 2910 * 2911 * The return value is -ERESTARTSYS if interrupted, 0 if timed out, 2912 * positive (at least 1, or number of jiffies left till timeout) if completed. 2913 */ 2914long __sched 2915wait_for_completion_killable_timeout(struct completion *x, 2916 unsigned long timeout) 2917{ 2918 return wait_for_common(x, timeout, TASK_KILLABLE); 2919} 2920EXPORT_SYMBOL(wait_for_completion_killable_timeout); 2921 2922/** 2923 * try_wait_for_completion - try to decrement a completion without blocking 2924 * @x: completion structure 2925 * 2926 * Returns: 0 if a decrement cannot be done without blocking 2927 * 1 if a decrement succeeded. 2928 * 2929 * If a completion is being used as a counting completion, 2930 * attempt to decrement the counter without blocking. This 2931 * enables us to avoid waiting if the resource the completion 2932 * is protecting is not available. 2933 */ 2934bool try_wait_for_completion(struct completion *x) 2935{ 2936 unsigned long flags; 2937 int ret = 1; 2938 2939 spin_lock_irqsave(&x->wait.lock, flags); 2940 if (!x->done) 2941 ret = 0; 2942 else 2943 x->done--; 2944 spin_unlock_irqrestore(&x->wait.lock, flags); 2945 return ret; 2946} 2947EXPORT_SYMBOL(try_wait_for_completion); 2948 2949/** 2950 * completion_done - Test to see if a completion has any waiters 2951 * @x: completion structure 2952 * 2953 * Returns: 0 if there are waiters (wait_for_completion() in progress) 2954 * 1 if there are no waiters. 2955 * 2956 */ 2957bool completion_done(struct completion *x) 2958{ 2959 unsigned long flags; 2960 int ret = 1; 2961 2962 spin_lock_irqsave(&x->wait.lock, flags); 2963 if (!x->done) 2964 ret = 0; 2965 spin_unlock_irqrestore(&x->wait.lock, flags); 2966 return ret; 2967} 2968EXPORT_SYMBOL(completion_done); 2969 2970static long __sched 2971sleep_on_common(wait_queue_head_t *q, int state, long timeout) 2972{ 2973 unsigned long flags; 2974 wait_queue_t wait; 2975 2976 init_waitqueue_entry(&wait, current); 2977 2978 __set_current_state(state); 2979 2980 spin_lock_irqsave(&q->lock, flags); 2981 __add_wait_queue(q, &wait); 2982 spin_unlock(&q->lock); 2983 timeout = schedule_timeout(timeout); 2984 spin_lock_irq(&q->lock); 2985 __remove_wait_queue(q, &wait); 2986 spin_unlock_irqrestore(&q->lock, flags); 2987 2988 return timeout; 2989} 2990 2991void __sched interruptible_sleep_on(wait_queue_head_t *q) 2992{ 2993 sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); 2994} 2995EXPORT_SYMBOL(interruptible_sleep_on); 2996 2997long __sched 2998interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout) 2999{ 3000 return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout); 3001} 3002EXPORT_SYMBOL(interruptible_sleep_on_timeout); 3003 3004void __sched sleep_on(wait_queue_head_t *q) 3005{ 3006 sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); 3007} 3008EXPORT_SYMBOL(sleep_on); 3009 3010long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout) 3011{ 3012 return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout); 3013} 3014EXPORT_SYMBOL(sleep_on_timeout); 3015 3016#ifdef CONFIG_RT_MUTEXES 3017 3018/* 3019 * rt_mutex_setprio - set the current priority of a task 3020 * @p: task 3021 * @prio: prio value (kernel-internal form) 3022 * 3023 * This function changes the 'effective' priority of a task. It does 3024 * not touch ->normal_prio like __setscheduler(). 3025 * 3026 * Used by the rt_mutex code to implement priority inheritance logic. 3027 */ 3028void rt_mutex_setprio(struct task_struct *p, int prio) 3029{ 3030 int oldprio, on_rq, running; 3031 struct rq *rq; 3032 const struct sched_class *prev_class; 3033 3034 BUG_ON(prio < 0 || prio > MAX_PRIO); 3035 3036 rq = __task_rq_lock(p); 3037 3038 /* 3039 * Idle task boosting is a nono in general. There is one 3040 * exception, when PREEMPT_RT and NOHZ is active: 3041 * 3042 * The idle task calls get_next_timer_interrupt() and holds 3043 * the timer wheel base->lock on the CPU and another CPU wants 3044 * to access the timer (probably to cancel it). We can safely 3045 * ignore the boosting request, as the idle CPU runs this code 3046 * with interrupts disabled and will complete the lock 3047 * protected section without being interrupted. So there is no 3048 * real need to boost. 3049 */ 3050 if (unlikely(p == rq->idle)) { 3051 WARN_ON(p != rq->curr); 3052 WARN_ON(p->pi_blocked_on); 3053 goto out_unlock; 3054 } 3055 3056 trace_sched_pi_setprio(p, prio); 3057 oldprio = p->prio; 3058 prev_class = p->sched_class; 3059 on_rq = p->on_rq; 3060 running = task_current(rq, p); 3061 if (on_rq) 3062 dequeue_task(rq, p, 0); 3063 if (running) 3064 p->sched_class->put_prev_task(rq, p); 3065 3066 if (rt_prio(prio)) 3067 p->sched_class = &rt_sched_class; 3068 else 3069 p->sched_class = &fair_sched_class; 3070 3071 p->prio = prio; 3072 3073 if (running) 3074 p->sched_class->set_curr_task(rq); 3075 if (on_rq) 3076 enqueue_task(rq, p, oldprio < prio ? ENQUEUE_HEAD : 0); 3077 3078 check_class_changed(rq, p, prev_class, oldprio); 3079out_unlock: 3080 __task_rq_unlock(rq); 3081} 3082#endif 3083void set_user_nice(struct task_struct *p, long nice) 3084{ 3085 int old_prio, delta, on_rq; 3086 unsigned long flags; 3087 struct rq *rq; 3088 3089 if (TASK_NICE(p) == nice || nice < -20 || nice > 19) 3090 return; 3091 /* 3092 * We have to be careful, if called from sys_setpriority(), 3093 * the task might be in the middle of scheduling on another CPU. 3094 */ 3095 rq = task_rq_lock(p, &flags); 3096 /* 3097 * The RT priorities are set via sched_setscheduler(), but we still 3098 * allow the 'normal' nice value to be set - but as expected 3099 * it wont have any effect on scheduling until the task is 3100 * SCHED_FIFO/SCHED_RR: 3101 */ 3102 if (task_has_rt_policy(p)) { 3103 p->static_prio = NICE_TO_PRIO(nice); 3104 goto out_unlock; 3105 } 3106 on_rq = p->on_rq; 3107 if (on_rq) 3108 dequeue_task(rq, p, 0); 3109 3110 p->static_prio = NICE_TO_PRIO(nice); 3111 set_load_weight(p); 3112 old_prio = p->prio; 3113 p->prio = effective_prio(p); 3114 delta = p->prio - old_prio; 3115 3116 if (on_rq) { 3117 enqueue_task(rq, p, 0); 3118 /* 3119 * If the task increased its priority or is running and 3120 * lowered its priority, then reschedule its CPU: 3121 */ 3122 if (delta < 0 || (delta > 0 && task_running(rq, p))) 3123 resched_task(rq->curr); 3124 } 3125out_unlock: 3126 task_rq_unlock(rq, p, &flags); 3127} 3128EXPORT_SYMBOL(set_user_nice); 3129 3130/* 3131 * can_nice - check if a task can reduce its nice value 3132 * @p: task 3133 * @nice: nice value 3134 */ 3135int can_nice(const struct task_struct *p, const int nice) 3136{ 3137 /* convert nice value [19,-20] to rlimit style value [1,40] */ 3138 int nice_rlim = 20 - nice; 3139 3140 return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || 3141 capable(CAP_SYS_NICE)); 3142} 3143 3144#ifdef __ARCH_WANT_SYS_NICE 3145 3146/* 3147 * sys_nice - change the priority of the current process. 3148 * @increment: priority increment 3149 * 3150 * sys_setpriority is a more generic, but much slower function that 3151 * does similar things. 3152 */ 3153SYSCALL_DEFINE1(nice, int, increment) 3154{ 3155 long nice, retval; 3156 3157 /* 3158 * Setpriority might change our priority at the same moment. 3159 * We don't have to worry. Conceptually one call occurs first 3160 * and we have a single winner. 3161 */ 3162 if (increment < -40) 3163 increment = -40; 3164 if (increment > 40) 3165 increment = 40; 3166 3167 nice = TASK_NICE(current) + increment; 3168 if (nice < -20) 3169 nice = -20; 3170 if (nice > 19) 3171 nice = 19; 3172 3173 if (increment < 0 && !can_nice(current, nice)) 3174 return -EPERM; 3175 3176 retval = security_task_setnice(current, nice); 3177 if (retval) 3178 return retval; 3179 3180 set_user_nice(current, nice); 3181 return 0; 3182} 3183 3184#endif 3185 3186/** 3187 * task_prio - return the priority value of a given task. 3188 * @p: the task in question. 3189 * 3190 * This is the priority value as seen by users in /proc. 3191 * RT tasks are offset by -200. Normal tasks are centered 3192 * around 0, value goes from -16 to +15. 3193 */ 3194int task_prio(const struct task_struct *p) 3195{ 3196 return p->prio - MAX_RT_PRIO; 3197} 3198 3199/** 3200 * task_nice - return the nice value of a given task. 3201 * @p: the task in question. 3202 */ 3203int task_nice(const struct task_struct *p) 3204{ 3205 return TASK_NICE(p); 3206} 3207EXPORT_SYMBOL(task_nice); 3208 3209/** 3210 * idle_cpu - is a given cpu idle currently? 3211 * @cpu: the processor in question. 3212 */ 3213int idle_cpu(int cpu) 3214{ 3215 struct rq *rq = cpu_rq(cpu); 3216 3217 if (rq->curr != rq->idle) 3218 return 0; 3219 3220 if (rq->nr_running) 3221 return 0; 3222 3223#ifdef CONFIG_SMP 3224 if (!llist_empty(&rq->wake_list)) 3225 return 0; 3226#endif 3227 3228 return 1; 3229} 3230 3231/** 3232 * idle_task - return the idle task for a given cpu. 3233 * @cpu: the processor in question. 3234 */ 3235struct task_struct *idle_task(int cpu) 3236{ 3237 return cpu_rq(cpu)->idle; 3238} 3239 3240/** 3241 * find_process_by_pid - find a process with a matching PID value. 3242 * @pid: the pid in question. 3243 */ 3244static struct task_struct *find_process_by_pid(pid_t pid) 3245{ 3246 return pid ? find_task_by_vpid(pid) : current; 3247} 3248 3249/* Actually do priority change: must hold rq lock. */ 3250static void 3251__setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio) 3252{ 3253 p->policy = policy; 3254 p->rt_priority = prio; 3255 p->normal_prio = normal_prio(p); 3256 /* we are holding p->pi_lock already */ 3257 p->prio = rt_mutex_getprio(p); 3258 if (rt_prio(p->prio)) 3259 p->sched_class = &rt_sched_class; 3260 else 3261 p->sched_class = &fair_sched_class; 3262 set_load_weight(p); 3263} 3264 3265/* 3266 * check the target process has a UID that matches the current process's 3267 */ 3268static bool check_same_owner(struct task_struct *p) 3269{ 3270 const struct cred *cred = current_cred(), *pcred; 3271 bool match; 3272 3273 rcu_read_lock(); 3274 pcred = __task_cred(p); 3275 match = (uid_eq(cred->euid, pcred->euid) || 3276 uid_eq(cred->euid, pcred->uid)); 3277 rcu_read_unlock(); 3278 return match; 3279} 3280 3281static int __sched_setscheduler(struct task_struct *p, int policy, 3282 const struct sched_param *param, bool user) 3283{ 3284 int retval, oldprio, oldpolicy = -1, on_rq, running; 3285 unsigned long flags; 3286 const struct sched_class *prev_class; 3287 struct rq *rq; 3288 int reset_on_fork; 3289 3290 /* may grab non-irq protected spin_locks */ 3291 BUG_ON(in_interrupt()); 3292recheck: 3293 /* double check policy once rq lock held */ 3294 if (policy < 0) { 3295 reset_on_fork = p->sched_reset_on_fork; 3296 policy = oldpolicy = p->policy; 3297 } else { 3298 reset_on_fork = !!(policy & SCHED_RESET_ON_FORK); 3299 policy &= ~SCHED_RESET_ON_FORK; 3300 3301 if (policy != SCHED_FIFO && policy != SCHED_RR && 3302 policy != SCHED_NORMAL && policy != SCHED_BATCH && 3303 policy != SCHED_IDLE) 3304 return -EINVAL; 3305 } 3306 3307 /* 3308 * Valid priorities for SCHED_FIFO and SCHED_RR are 3309 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, 3310 * SCHED_BATCH and SCHED_IDLE is 0. 3311 */ 3312 if (param->sched_priority < 0 || 3313 (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) || 3314 (!p->mm && param->sched_priority > MAX_RT_PRIO-1)) 3315 return -EINVAL; 3316 if (rt_policy(policy) != (param->sched_priority != 0)) 3317 return -EINVAL; 3318 3319 /* 3320 * Allow unprivileged RT tasks to decrease priority: 3321 */ 3322 if (user && !capable(CAP_SYS_NICE)) { 3323 if (rt_policy(policy)) { 3324 unsigned long rlim_rtprio = 3325 task_rlimit(p, RLIMIT_RTPRIO); 3326 3327 /* can't set/change the rt policy */ 3328 if (policy != p->policy && !rlim_rtprio) 3329 return -EPERM; 3330 3331 /* can't increase priority */ 3332 if (param->sched_priority > p->rt_priority && 3333 param->sched_priority > rlim_rtprio) 3334 return -EPERM; 3335 } 3336 3337 /* 3338 * Treat SCHED_IDLE as nice 20. Only allow a switch to 3339 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. 3340 */ 3341 if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) { 3342 if (!can_nice(p, TASK_NICE(p))) 3343 return -EPERM; 3344 } 3345 3346 /* can't change other user's priorities */ 3347 if (!check_same_owner(p)) 3348 return -EPERM; 3349 3350 /* Normal users shall not reset the sched_reset_on_fork flag */ 3351 if (p->sched_reset_on_fork && !reset_on_fork) 3352 return -EPERM; 3353 } 3354 3355 if (user) { 3356 retval = security_task_setscheduler(p); 3357 if (retval) 3358 return retval; 3359 } 3360 3361 /* 3362 * make sure no PI-waiters arrive (or leave) while we are 3363 * changing the priority of the task: 3364 * 3365 * To be able to change p->policy safely, the appropriate 3366 * runqueue lock must be held. 3367 */ 3368 rq = task_rq_lock(p, &flags); 3369 3370 /* 3371 * Changing the policy of the stop threads its a very bad idea 3372 */ 3373 if (p == rq->stop) { 3374 task_rq_unlock(rq, p, &flags); 3375 return -EINVAL; 3376 } 3377 3378 /* 3379 * If not changing anything there's no need to proceed further: 3380 */ 3381 if (unlikely(policy == p->policy && (!rt_policy(policy) || 3382 param->sched_priority == p->rt_priority))) { 3383 task_rq_unlock(rq, p, &flags); 3384 return 0; 3385 } 3386 3387#ifdef CONFIG_RT_GROUP_SCHED 3388 if (user) { 3389 /* 3390 * Do not allow realtime tasks into groups that have no runtime 3391 * assigned. 3392 */ 3393 if (rt_bandwidth_enabled() && rt_policy(policy) && 3394 task_group(p)->rt_bandwidth.rt_runtime == 0 && 3395 !task_group_is_autogroup(task_group(p))) { 3396 task_rq_unlock(rq, p, &flags); 3397 return -EPERM; 3398 } 3399 } 3400#endif 3401 3402 /* recheck policy now with rq lock held */ 3403 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { 3404 policy = oldpolicy = -1; 3405 task_rq_unlock(rq, p, &flags); 3406 goto recheck; 3407 } 3408 on_rq = p->on_rq; 3409 running = task_current(rq, p); 3410 if (on_rq) 3411 dequeue_task(rq, p, 0); 3412 if (running) 3413 p->sched_class->put_prev_task(rq, p); 3414 3415 p->sched_reset_on_fork = reset_on_fork; 3416 3417 oldprio = p->prio; 3418 prev_class = p->sched_class; 3419 __setscheduler(rq, p, policy, param->sched_priority); 3420 3421 if (running) 3422 p->sched_class->set_curr_task(rq); 3423 if (on_rq) 3424 enqueue_task(rq, p, 0); 3425 3426 check_class_changed(rq, p, prev_class, oldprio); 3427 task_rq_unlock(rq, p, &flags); 3428 3429 rt_mutex_adjust_pi(p); 3430 3431 return 0; 3432} 3433 3434/** 3435 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. 3436 * @p: the task in question. 3437 * @policy: new policy. 3438 * @param: structure containing the new RT priority. 3439 * 3440 * NOTE that the task may be already dead. 3441 */ 3442int sched_setscheduler(struct task_struct *p, int policy, 3443 const struct sched_param *param) 3444{ 3445 return __sched_setscheduler(p, policy, param, true); 3446} 3447EXPORT_SYMBOL_GPL(sched_setscheduler); 3448 3449/** 3450 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. 3451 * @p: the task in question. 3452 * @policy: new policy. 3453 * @param: structure containing the new RT priority. 3454 * 3455 * Just like sched_setscheduler, only don't bother checking if the 3456 * current context has permission. For example, this is needed in 3457 * stop_machine(): we create temporary high priority worker threads, 3458 * but our caller might not have that capability. 3459 */ 3460int sched_setscheduler_nocheck(struct task_struct *p, int policy, 3461 const struct sched_param *param) 3462{ 3463 return __sched_setscheduler(p, policy, param, false); 3464} 3465 3466static int 3467do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) 3468{ 3469 struct sched_param lparam; 3470 struct task_struct *p; 3471 int retval; 3472 3473 if (!param || pid < 0) 3474 return -EINVAL; 3475 if (copy_from_user(&lparam, param, sizeof(struct sched_param))) 3476 return -EFAULT; 3477 3478 rcu_read_lock(); 3479 retval = -ESRCH; 3480 p = find_process_by_pid(pid); 3481 if (p != NULL) 3482 retval = sched_setscheduler(p, policy, &lparam); 3483 rcu_read_unlock(); 3484 3485 return retval; 3486} 3487 3488/** 3489 * sys_sched_setscheduler - set/change the scheduler policy and RT priority 3490 * @pid: the pid in question. 3491 * @policy: new policy. 3492 * @param: structure containing the new RT priority. 3493 */ 3494SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, 3495 struct sched_param __user *, param) 3496{ 3497 /* negative values for policy are not valid */ 3498 if (policy < 0) 3499 return -EINVAL; 3500 3501 return do_sched_setscheduler(pid, policy, param); 3502} 3503 3504/** 3505 * sys_sched_setparam - set/change the RT priority of a thread 3506 * @pid: the pid in question. 3507 * @param: structure containing the new RT priority. 3508 */ 3509SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) 3510{ 3511 return do_sched_setscheduler(pid, -1, param); 3512} 3513 3514/** 3515 * sys_sched_getscheduler - get the policy (scheduling class) of a thread 3516 * @pid: the pid in question. 3517 */ 3518SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) 3519{ 3520 struct task_struct *p; 3521 int retval; 3522 3523 if (pid < 0) 3524 return -EINVAL; 3525 3526 retval = -ESRCH; 3527 rcu_read_lock(); 3528 p = find_process_by_pid(pid); 3529 if (p) { 3530 retval = security_task_getscheduler(p); 3531 if (!retval) 3532 retval = p->policy 3533 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); 3534 } 3535 rcu_read_unlock(); 3536 return retval; 3537} 3538 3539/** 3540 * sys_sched_getparam - get the RT priority of a thread 3541 * @pid: the pid in question. 3542 * @param: structure containing the RT priority. 3543 */ 3544SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) 3545{ 3546 struct sched_param lp; 3547 struct task_struct *p; 3548 int retval; 3549 3550 if (!param || pid < 0) 3551 return -EINVAL; 3552 3553 rcu_read_lock(); 3554 p = find_process_by_pid(pid); 3555 retval = -ESRCH; 3556 if (!p) 3557 goto out_unlock; 3558 3559 retval = security_task_getscheduler(p); 3560 if (retval) 3561 goto out_unlock; 3562 3563 lp.sched_priority = p->rt_priority; 3564 rcu_read_unlock(); 3565 3566 /* 3567 * This one might sleep, we cannot do it with a spinlock held ... 3568 */ 3569 retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; 3570 3571 return retval; 3572 3573out_unlock: 3574 rcu_read_unlock(); 3575 return retval; 3576} 3577 3578long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) 3579{ 3580 cpumask_var_t cpus_allowed, new_mask; 3581 struct task_struct *p; 3582 int retval; 3583 3584 get_online_cpus(); 3585 rcu_read_lock(); 3586 3587 p = find_process_by_pid(pid); 3588 if (!p) { 3589 rcu_read_unlock(); 3590 put_online_cpus(); 3591 return -ESRCH; 3592 } 3593 3594 /* Prevent p going away */ 3595 get_task_struct(p); 3596 rcu_read_unlock(); 3597 3598 if (p->flags & PF_NO_SETAFFINITY) { 3599 retval = -EINVAL; 3600 goto out_put_task; 3601 } 3602 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { 3603 retval = -ENOMEM; 3604 goto out_put_task; 3605 } 3606 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { 3607 retval = -ENOMEM; 3608 goto out_free_cpus_allowed; 3609 } 3610 retval = -EPERM; 3611 if (!check_same_owner(p)) { 3612 rcu_read_lock(); 3613 if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { 3614 rcu_read_unlock(); 3615 goto out_unlock; 3616 } 3617 rcu_read_unlock(); 3618 } 3619 3620 retval = security_task_setscheduler(p); 3621 if (retval) 3622 goto out_unlock; 3623 3624 cpuset_cpus_allowed(p, cpus_allowed); 3625 cpumask_and(new_mask, in_mask, cpus_allowed); 3626again: 3627 retval = set_cpus_allowed_ptr(p, new_mask); 3628 3629 if (!retval) { 3630 cpuset_cpus_allowed(p, cpus_allowed); 3631 if (!cpumask_subset(new_mask, cpus_allowed)) { 3632 /* 3633 * We must have raced with a concurrent cpuset 3634 * update. Just reset the cpus_allowed to the 3635 * cpuset's cpus_allowed 3636 */ 3637 cpumask_copy(new_mask, cpus_allowed); 3638 goto again; 3639 } 3640 } 3641out_unlock: 3642 free_cpumask_var(new_mask); 3643out_free_cpus_allowed: 3644 free_cpumask_var(cpus_allowed); 3645out_put_task: 3646 put_task_struct(p); 3647 put_online_cpus(); 3648 return retval; 3649} 3650 3651static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 3652 struct cpumask *new_mask) 3653{ 3654 if (len < cpumask_size()) 3655 cpumask_clear(new_mask); 3656 else if (len > cpumask_size()) 3657 len = cpumask_size(); 3658 3659 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 3660} 3661 3662/** 3663 * sys_sched_setaffinity - set the cpu affinity of a process 3664 * @pid: pid of the process 3665 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 3666 * @user_mask_ptr: user-space pointer to the new cpu mask 3667 */ 3668SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, 3669 unsigned long __user *, user_mask_ptr) 3670{ 3671 cpumask_var_t new_mask; 3672 int retval; 3673 3674 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) 3675 return -ENOMEM; 3676 3677 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); 3678 if (retval == 0) 3679 retval = sched_setaffinity(pid, new_mask); 3680 free_cpumask_var(new_mask); 3681 return retval; 3682} 3683 3684long sched_getaffinity(pid_t pid, struct cpumask *mask) 3685{ 3686 struct task_struct *p; 3687 unsigned long flags; 3688 int retval; 3689 3690 get_online_cpus(); 3691 rcu_read_lock(); 3692 3693 retval = -ESRCH; 3694 p = find_process_by_pid(pid); 3695 if (!p) 3696 goto out_unlock; 3697 3698 retval = security_task_getscheduler(p); 3699 if (retval) 3700 goto out_unlock; 3701 3702 raw_spin_lock_irqsave(&p->pi_lock, flags); 3703 cpumask_and(mask, &p->cpus_allowed, cpu_online_mask); 3704 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 3705 3706out_unlock: 3707 rcu_read_unlock(); 3708 put_online_cpus(); 3709 3710 return retval; 3711} 3712 3713/** 3714 * sys_sched_getaffinity - get the cpu affinity of a process 3715 * @pid: pid of the process 3716 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 3717 * @user_mask_ptr: user-space pointer to hold the current cpu mask 3718 */ 3719SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, 3720 unsigned long __user *, user_mask_ptr) 3721{ 3722 int ret; 3723 cpumask_var_t mask; 3724 3725 if ((len * BITS_PER_BYTE) < nr_cpu_ids) 3726 return -EINVAL; 3727 if (len & (sizeof(unsigned long)-1)) 3728 return -EINVAL; 3729 3730 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 3731 return -ENOMEM; 3732 3733 ret = sched_getaffinity(pid, mask); 3734 if (ret == 0) { 3735 size_t retlen = min_t(size_t, len, cpumask_size()); 3736 3737 if (copy_to_user(user_mask_ptr, mask, retlen)) 3738 ret = -EFAULT; 3739 else 3740 ret = retlen; 3741 } 3742 free_cpumask_var(mask); 3743 3744 return ret; 3745} 3746 3747/** 3748 * sys_sched_yield - yield the current processor to other threads. 3749 * 3750 * This function yields the current CPU to other tasks. If there are no 3751 * other threads running on this CPU then this function will return. 3752 */ 3753SYSCALL_DEFINE0(sched_yield) 3754{ 3755 struct rq *rq = this_rq_lock(); 3756 3757 schedstat_inc(rq, yld_count); 3758 current->sched_class->yield_task(rq); 3759 3760 /* 3761 * Since we are going to call schedule() anyway, there's 3762 * no need to preempt or enable interrupts: 3763 */ 3764 __release(rq->lock); 3765 spin_release(&rq->lock.dep_map, 1, _THIS_IP_); 3766 do_raw_spin_unlock(&rq->lock); 3767 sched_preempt_enable_no_resched(); 3768 3769 schedule(); 3770 3771 return 0; 3772} 3773 3774static inline int should_resched(void) 3775{ 3776 return need_resched() && !(preempt_count() & PREEMPT_ACTIVE); 3777} 3778 3779static void __cond_resched(void) 3780{ 3781 add_preempt_count(PREEMPT_ACTIVE); 3782 __schedule(); 3783 sub_preempt_count(PREEMPT_ACTIVE); 3784} 3785 3786int __sched _cond_resched(void) 3787{ 3788 if (should_resched()) { 3789 __cond_resched(); 3790 return 1; 3791 } 3792 return 0; 3793} 3794EXPORT_SYMBOL(_cond_resched); 3795 3796/* 3797 * __cond_resched_lock() - if a reschedule is pending, drop the given lock, 3798 * call schedule, and on return reacquire the lock. 3799 * 3800 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level 3801 * operations here to prevent schedule() from being called twice (once via 3802 * spin_unlock(), once by hand). 3803 */ 3804int __cond_resched_lock(spinlock_t *lock) 3805{ 3806 int resched = should_resched(); 3807 int ret = 0; 3808 3809 lockdep_assert_held(lock); 3810 3811 if (spin_needbreak(lock) || resched) { 3812 spin_unlock(lock); 3813 if (resched) 3814 __cond_resched(); 3815 else 3816 cpu_relax(); 3817 ret = 1; 3818 spin_lock(lock); 3819 } 3820 return ret; 3821} 3822EXPORT_SYMBOL(__cond_resched_lock); 3823 3824int __sched __cond_resched_softirq(void) 3825{ 3826 BUG_ON(!in_softirq()); 3827 3828 if (should_resched()) { 3829 local_bh_enable(); 3830 __cond_resched(); 3831 local_bh_disable(); 3832 return 1; 3833 } 3834 return 0; 3835} 3836EXPORT_SYMBOL(__cond_resched_softirq); 3837 3838/** 3839 * yield - yield the current processor to other threads. 3840 * 3841 * Do not ever use this function, there's a 99% chance you're doing it wrong. 3842 * 3843 * The scheduler is at all times free to pick the calling task as the most 3844 * eligible task to run, if removing the yield() call from your code breaks 3845 * it, its already broken. 3846 * 3847 * Typical broken usage is: 3848 * 3849 * while (!event) 3850 * yield(); 3851 * 3852 * where one assumes that yield() will let 'the other' process run that will 3853 * make event true. If the current task is a SCHED_FIFO task that will never 3854 * happen. Never use yield() as a progress guarantee!! 3855 * 3856 * If you want to use yield() to wait for something, use wait_event(). 3857 * If you want to use yield() to be 'nice' for others, use cond_resched(). 3858 * If you still want to use yield(), do not! 3859 */ 3860void __sched yield(void) 3861{ 3862 set_current_state(TASK_RUNNING); 3863 sys_sched_yield(); 3864} 3865EXPORT_SYMBOL(yield); 3866 3867/** 3868 * yield_to - yield the current processor to another thread in 3869 * your thread group, or accelerate that thread toward the 3870 * processor it's on. 3871 * @p: target task 3872 * @preempt: whether task preemption is allowed or not 3873 * 3874 * It's the caller's job to ensure that the target task struct 3875 * can't go away on us before we can do any checks. 3876 * 3877 * Returns: 3878 * true (>0) if we indeed boosted the target task. 3879 * false (0) if we failed to boost the target. 3880 * -ESRCH if there's no task to yield to. 3881 */ 3882bool __sched yield_to(struct task_struct *p, bool preempt) 3883{ 3884 struct task_struct *curr = current; 3885 struct rq *rq, *p_rq; 3886 unsigned long flags; 3887 int yielded = 0; 3888 3889 local_irq_save(flags); 3890 rq = this_rq(); 3891 3892again: 3893 p_rq = task_rq(p); 3894 /* 3895 * If we're the only runnable task on the rq and target rq also 3896 * has only one task, there's absolutely no point in yielding. 3897 */ 3898 if (rq->nr_running == 1 && p_rq->nr_running == 1) { 3899 yielded = -ESRCH; 3900 goto out_irq; 3901 } 3902 3903 double_rq_lock(rq, p_rq); 3904 while (task_rq(p) != p_rq) { 3905 double_rq_unlock(rq, p_rq); 3906 goto again; 3907 } 3908 3909 if (!curr->sched_class->yield_to_task) 3910 goto out_unlock; 3911 3912 if (curr->sched_class != p->sched_class) 3913 goto out_unlock; 3914 3915 if (task_running(p_rq, p) || p->state) 3916 goto out_unlock; 3917 3918 yielded = curr->sched_class->yield_to_task(rq, p, preempt); 3919 if (yielded) { 3920 schedstat_inc(rq, yld_count); 3921 /* 3922 * Make p's CPU reschedule; pick_next_entity takes care of 3923 * fairness. 3924 */ 3925 if (preempt && rq != p_rq) 3926 resched_task(p_rq->curr); 3927 } 3928 3929out_unlock: 3930 double_rq_unlock(rq, p_rq); 3931out_irq: 3932 local_irq_restore(flags); 3933 3934 if (yielded > 0) 3935 schedule(); 3936 3937 return yielded; 3938} 3939EXPORT_SYMBOL_GPL(yield_to); 3940 3941/* 3942 * This task is about to go to sleep on IO. Increment rq->nr_iowait so 3943 * that process accounting knows that this is a task in IO wait state. 3944 */ 3945void __sched io_schedule(void) 3946{ 3947 struct rq *rq = raw_rq(); 3948 3949 delayacct_blkio_start(); 3950 atomic_inc(&rq->nr_iowait); 3951 blk_flush_plug(current); 3952 current->in_iowait = 1; 3953 schedule(); 3954 current->in_iowait = 0; 3955 atomic_dec(&rq->nr_iowait); 3956 delayacct_blkio_end(); 3957} 3958EXPORT_SYMBOL(io_schedule); 3959 3960long __sched io_schedule_timeout(long timeout) 3961{ 3962 struct rq *rq = raw_rq(); 3963 long ret; 3964 3965 delayacct_blkio_start(); 3966 atomic_inc(&rq->nr_iowait); 3967 blk_flush_plug(current); 3968 current->in_iowait = 1; 3969 ret = schedule_timeout(timeout); 3970 current->in_iowait = 0; 3971 atomic_dec(&rq->nr_iowait); 3972 delayacct_blkio_end(); 3973 return ret; 3974} 3975 3976/** 3977 * sys_sched_get_priority_max - return maximum RT priority. 3978 * @policy: scheduling class. 3979 * 3980 * this syscall returns the maximum rt_priority that can be used 3981 * by a given scheduling class. 3982 */ 3983SYSCALL_DEFINE1(sched_get_priority_max, int, policy) 3984{ 3985 int ret = -EINVAL; 3986 3987 switch (policy) { 3988 case SCHED_FIFO: 3989 case SCHED_RR: 3990 ret = MAX_USER_RT_PRIO-1; 3991 break; 3992 case SCHED_NORMAL: 3993 case SCHED_BATCH: 3994 case SCHED_IDLE: 3995 ret = 0; 3996 break; 3997 } 3998 return ret; 3999} 4000 4001/** 4002 * sys_sched_get_priority_min - return minimum RT priority. 4003 * @policy: scheduling class. 4004 * 4005 * this syscall returns the minimum rt_priority that can be used 4006 * by a given scheduling class. 4007 */ 4008SYSCALL_DEFINE1(sched_get_priority_min, int, policy) 4009{ 4010 int ret = -EINVAL; 4011 4012 switch (policy) { 4013 case SCHED_FIFO: 4014 case SCHED_RR: 4015 ret = 1; 4016 break; 4017 case SCHED_NORMAL: 4018 case SCHED_BATCH: 4019 case SCHED_IDLE: 4020 ret = 0; 4021 } 4022 return ret; 4023} 4024 4025/** 4026 * sys_sched_rr_get_interval - return the default timeslice of a process. 4027 * @pid: pid of the process. 4028 * @interval: userspace pointer to the timeslice value. 4029 * 4030 * this syscall writes the default timeslice value of a given process 4031 * into the user-space timespec buffer. A value of '0' means infinity. 4032 */ 4033SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, 4034 struct timespec __user *, interval) 4035{ 4036 struct task_struct *p; 4037 unsigned int time_slice; 4038 unsigned long flags; 4039 struct rq *rq; 4040 int retval; 4041 struct timespec t; 4042 4043 if (pid < 0) 4044 return -EINVAL; 4045 4046 retval = -ESRCH; 4047 rcu_read_lock(); 4048 p = find_process_by_pid(pid); 4049 if (!p) 4050 goto out_unlock; 4051 4052 retval = security_task_getscheduler(p); 4053 if (retval) 4054 goto out_unlock; 4055 4056 rq = task_rq_lock(p, &flags); 4057 time_slice = p->sched_class->get_rr_interval(rq, p); 4058 task_rq_unlock(rq, p, &flags); 4059 4060 rcu_read_unlock(); 4061 jiffies_to_timespec(time_slice, &t); 4062 retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0; 4063 return retval; 4064 4065out_unlock: 4066 rcu_read_unlock(); 4067 return retval; 4068} 4069 4070static const char stat_nam[] = TASK_STATE_TO_CHAR_STR; 4071 4072void sched_show_task(struct task_struct *p) 4073{ 4074 unsigned long free = 0; 4075 int ppid; 4076 unsigned state; 4077 4078 state = p->state ? __ffs(p->state) + 1 : 0; 4079 printk(KERN_INFO "%-15.15s %c", p->comm, 4080 state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?'); 4081#if BITS_PER_LONG == 32 4082 if (state == TASK_RUNNING) 4083 printk(KERN_CONT " running "); 4084 else 4085 printk(KERN_CONT " %08lx ", thread_saved_pc(p)); 4086#else 4087 if (state == TASK_RUNNING) 4088 printk(KERN_CONT " running task "); 4089 else 4090 printk(KERN_CONT " %016lx ", thread_saved_pc(p)); 4091#endif 4092#ifdef CONFIG_DEBUG_STACK_USAGE 4093 free = stack_not_used(p); 4094#endif 4095 rcu_read_lock(); 4096 ppid = task_pid_nr(rcu_dereference(p->real_parent)); 4097 rcu_read_unlock(); 4098 printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free, 4099 task_pid_nr(p), ppid, 4100 (unsigned long)task_thread_info(p)->flags); 4101 4102 print_worker_info(KERN_INFO, p); 4103 show_stack(p, NULL); 4104} 4105 4106void show_state_filter(unsigned long state_filter) 4107{ 4108 struct task_struct *g, *p; 4109 4110#if BITS_PER_LONG == 32 4111 printk(KERN_INFO 4112 " task PC stack pid father\n"); 4113#else 4114 printk(KERN_INFO 4115 " task PC stack pid father\n"); 4116#endif 4117 rcu_read_lock(); 4118 do_each_thread(g, p) { 4119 /* 4120 * reset the NMI-timeout, listing all files on a slow 4121 * console might take a lot of time: 4122 */ 4123 touch_nmi_watchdog(); 4124 if (!state_filter || (p->state & state_filter)) 4125 sched_show_task(p); 4126 } while_each_thread(g, p); 4127 4128 touch_all_softlockup_watchdogs(); 4129 4130#ifdef CONFIG_SCHED_DEBUG 4131 sysrq_sched_debug_show(); 4132#endif 4133 rcu_read_unlock(); 4134 /* 4135 * Only show locks if all tasks are dumped: 4136 */ 4137 if (!state_filter) 4138 debug_show_all_locks(); 4139} 4140 4141void __cpuinit init_idle_bootup_task(struct task_struct *idle) 4142{ 4143 idle->sched_class = &idle_sched_class; 4144} 4145 4146/** 4147 * init_idle - set up an idle thread for a given CPU 4148 * @idle: task in question 4149 * @cpu: cpu the idle task belongs to 4150 * 4151 * NOTE: this function does not set the idle thread's NEED_RESCHED 4152 * flag, to make booting more robust. 4153 */ 4154void __cpuinit init_idle(struct task_struct *idle, int cpu) 4155{ 4156 struct rq *rq = cpu_rq(cpu); 4157 unsigned long flags; 4158 4159 raw_spin_lock_irqsave(&rq->lock, flags); 4160 4161 __sched_fork(idle); 4162 idle->state = TASK_RUNNING; 4163 idle->se.exec_start = sched_clock(); 4164 4165 do_set_cpus_allowed(idle, cpumask_of(cpu)); 4166 /* 4167 * We're having a chicken and egg problem, even though we are 4168 * holding rq->lock, the cpu isn't yet set to this cpu so the 4169 * lockdep check in task_group() will fail. 4170 * 4171 * Similar case to sched_fork(). / Alternatively we could 4172 * use task_rq_lock() here and obtain the other rq->lock. 4173 * 4174 * Silence PROVE_RCU 4175 */ 4176 rcu_read_lock(); 4177 __set_task_cpu(idle, cpu); 4178 rcu_read_unlock(); 4179 4180 rq->curr = rq->idle = idle; 4181#if defined(CONFIG_SMP) 4182 idle->on_cpu = 1; 4183#endif 4184 raw_spin_unlock_irqrestore(&rq->lock, flags); 4185 4186 /* Set the preempt count _outside_ the spinlocks! */ 4187 task_thread_info(idle)->preempt_count = 0; 4188 4189 /* 4190 * The idle tasks have their own, simple scheduling class: 4191 */ 4192 idle->sched_class = &idle_sched_class; 4193 ftrace_graph_init_idle_task(idle, cpu); 4194 vtime_init_idle(idle); 4195#if defined(CONFIG_SMP) 4196 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); 4197#endif 4198} 4199 4200#ifdef CONFIG_SMP 4201void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) 4202{ 4203 if (p->sched_class && p->sched_class->set_cpus_allowed) 4204 p->sched_class->set_cpus_allowed(p, new_mask); 4205 4206 cpumask_copy(&p->cpus_allowed, new_mask); 4207 p->nr_cpus_allowed = cpumask_weight(new_mask); 4208} 4209 4210/* 4211 * This is how migration works: 4212 * 4213 * 1) we invoke migration_cpu_stop() on the target CPU using 4214 * stop_one_cpu(). 4215 * 2) stopper starts to run (implicitly forcing the migrated thread 4216 * off the CPU) 4217 * 3) it checks whether the migrated task is still in the wrong runqueue. 4218 * 4) if it's in the wrong runqueue then the migration thread removes 4219 * it and puts it into the right queue. 4220 * 5) stopper completes and stop_one_cpu() returns and the migration 4221 * is done. 4222 */ 4223 4224/* 4225 * Change a given task's CPU affinity. Migrate the thread to a 4226 * proper CPU and schedule it away if the CPU it's executing on 4227 * is removed from the allowed bitmask. 4228 * 4229 * NOTE: the caller must have a valid reference to the task, the 4230 * task must not exit() & deallocate itself prematurely. The 4231 * call is not atomic; no spinlocks may be held. 4232 */ 4233int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) 4234{ 4235 unsigned long flags; 4236 struct rq *rq; 4237 unsigned int dest_cpu; 4238 int ret = 0; 4239 4240 rq = task_rq_lock(p, &flags); 4241 4242 if (cpumask_equal(&p->cpus_allowed, new_mask)) 4243 goto out; 4244 4245 if (!cpumask_intersects(new_mask, cpu_active_mask)) { 4246 ret = -EINVAL; 4247 goto out; 4248 } 4249 4250 do_set_cpus_allowed(p, new_mask); 4251 4252 /* Can the task run on the task's current CPU? If so, we're done */ 4253 if (cpumask_test_cpu(task_cpu(p), new_mask)) 4254 goto out; 4255 4256 dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); 4257 if (p->on_rq) { 4258 struct migration_arg arg = { p, dest_cpu }; 4259 /* Need help from migration thread: drop lock and wait. */ 4260 task_rq_unlock(rq, p, &flags); 4261 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); 4262 tlb_migrate_finish(p->mm); 4263 return 0; 4264 } 4265out: 4266 task_rq_unlock(rq, p, &flags); 4267 4268 return ret; 4269} 4270EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); 4271 4272/* 4273 * Move (not current) task off this cpu, onto dest cpu. We're doing 4274 * this because either it can't run here any more (set_cpus_allowed() 4275 * away from this CPU, or CPU going down), or because we're 4276 * attempting to rebalance this task on exec (sched_exec). 4277 * 4278 * So we race with normal scheduler movements, but that's OK, as long 4279 * as the task is no longer on this CPU. 4280 * 4281 * Returns non-zero if task was successfully migrated. 4282 */ 4283static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) 4284{ 4285 struct rq *rq_dest, *rq_src; 4286 int ret = 0; 4287 4288 if (unlikely(!cpu_active(dest_cpu))) 4289 return ret; 4290 4291 rq_src = cpu_rq(src_cpu); 4292 rq_dest = cpu_rq(dest_cpu); 4293 4294 raw_spin_lock(&p->pi_lock); 4295 double_rq_lock(rq_src, rq_dest); 4296 /* Already moved. */ 4297 if (task_cpu(p) != src_cpu) 4298 goto done; 4299 /* Affinity changed (again). */ 4300 if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) 4301 goto fail; 4302 4303 /* 4304 * If we're not on a rq, the next wake-up will ensure we're 4305 * placed properly. 4306 */ 4307 if (p->on_rq) { 4308 dequeue_task(rq_src, p, 0); 4309 set_task_cpu(p, dest_cpu); 4310 enqueue_task(rq_dest, p, 0); 4311 check_preempt_curr(rq_dest, p, 0); 4312 } 4313done: 4314 ret = 1; 4315fail: 4316 double_rq_unlock(rq_src, rq_dest); 4317 raw_spin_unlock(&p->pi_lock); 4318 return ret; 4319} 4320 4321/* 4322 * migration_cpu_stop - this will be executed by a highprio stopper thread 4323 * and performs thread migration by bumping thread off CPU then 4324 * 'pushing' onto another runqueue. 4325 */ 4326static int migration_cpu_stop(void *data) 4327{ 4328 struct migration_arg *arg = data; 4329 4330 /* 4331 * The original target cpu might have gone down and we might 4332 * be on another cpu but it doesn't matter. 4333 */ 4334 local_irq_disable(); 4335 __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu); 4336 local_irq_enable(); 4337 return 0; 4338} 4339 4340#ifdef CONFIG_HOTPLUG_CPU 4341 4342/* 4343 * Ensures that the idle task is using init_mm right before its cpu goes 4344 * offline. 4345 */ 4346void idle_task_exit(void) 4347{ 4348 struct mm_struct *mm = current->active_mm; 4349 4350 BUG_ON(cpu_online(smp_processor_id())); 4351 4352 if (mm != &init_mm) 4353 switch_mm(mm, &init_mm, current); 4354 mmdrop(mm); 4355} 4356 4357/* 4358 * Since this CPU is going 'away' for a while, fold any nr_active delta 4359 * we might have. Assumes we're called after migrate_tasks() so that the 4360 * nr_active count is stable. 4361 * 4362 * Also see the comment "Global load-average calculations". 4363 */ 4364static void calc_load_migrate(struct rq *rq) 4365{ 4366 long delta = calc_load_fold_active(rq); 4367 if (delta) 4368 atomic_long_add(delta, &calc_load_tasks); 4369} 4370 4371/* 4372 * Migrate all tasks from the rq, sleeping tasks will be migrated by 4373 * try_to_wake_up()->select_task_rq(). 4374 * 4375 * Called with rq->lock held even though we'er in stop_machine() and 4376 * there's no concurrency possible, we hold the required locks anyway 4377 * because of lock validation efforts. 4378 */ 4379static void migrate_tasks(unsigned int dead_cpu) 4380{ 4381 struct rq *rq = cpu_rq(dead_cpu); 4382 struct task_struct *next, *stop = rq->stop; 4383 int dest_cpu; 4384 4385 /* 4386 * Fudge the rq selection such that the below task selection loop 4387 * doesn't get stuck on the currently eligible stop task. 4388 * 4389 * We're currently inside stop_machine() and the rq is either stuck 4390 * in the stop_machine_cpu_stop() loop, or we're executing this code, 4391 * either way we should never end up calling schedule() until we're 4392 * done here. 4393 */ 4394 rq->stop = NULL; 4395 4396 /* 4397 * put_prev_task() and pick_next_task() sched 4398 * class method both need to have an up-to-date 4399 * value of rq->clock[_task] 4400 */ 4401 update_rq_clock(rq); 4402 4403 for ( ; ; ) { 4404 /* 4405 * There's this thread running, bail when that's the only 4406 * remaining thread. 4407 */ 4408 if (rq->nr_running == 1) 4409 break; 4410 4411 next = pick_next_task(rq); 4412 BUG_ON(!next); 4413 next->sched_class->put_prev_task(rq, next); 4414 4415 /* Find suitable destination for @next, with force if needed. */ 4416 dest_cpu = select_fallback_rq(dead_cpu, next); 4417 raw_spin_unlock(&rq->lock); 4418 4419 __migrate_task(next, dead_cpu, dest_cpu); 4420 4421 raw_spin_lock(&rq->lock); 4422 } 4423 4424 rq->stop = stop; 4425} 4426 4427#endif /* CONFIG_HOTPLUG_CPU */ 4428 4429#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL) 4430 4431static struct ctl_table sd_ctl_dir[] = { 4432 { 4433 .procname = "sched_domain", 4434 .mode = 0555, 4435 }, 4436 {} 4437}; 4438 4439static struct ctl_table sd_ctl_root[] = { 4440 { 4441 .procname = "kernel", 4442 .mode = 0555, 4443 .child = sd_ctl_dir, 4444 }, 4445 {} 4446}; 4447 4448static struct ctl_table *sd_alloc_ctl_entry(int n) 4449{ 4450 struct ctl_table *entry = 4451 kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL); 4452 4453 return entry; 4454} 4455 4456static void sd_free_ctl_entry(struct ctl_table **tablep) 4457{ 4458 struct ctl_table *entry; 4459 4460 /* 4461 * In the intermediate directories, both the child directory and 4462 * procname are dynamically allocated and could fail but the mode 4463 * will always be set. In the lowest directory the names are 4464 * static strings and all have proc handlers. 4465 */ 4466 for (entry = *tablep; entry->mode; entry++) { 4467 if (entry->child) 4468 sd_free_ctl_entry(&entry->child); 4469 if (entry->proc_handler == NULL) 4470 kfree(entry->procname); 4471 } 4472 4473 kfree(*tablep); 4474 *tablep = NULL; 4475} 4476 4477static int min_load_idx = 0; 4478static int max_load_idx = CPU_LOAD_IDX_MAX-1; 4479 4480static void 4481set_table_entry(struct ctl_table *entry, 4482 const char *procname, void *data, int maxlen, 4483 umode_t mode, proc_handler *proc_handler, 4484 bool load_idx) 4485{ 4486 entry->procname = procname; 4487 entry->data = data; 4488 entry->maxlen = maxlen; 4489 entry->mode = mode; 4490 entry->proc_handler = proc_handler; 4491 4492 if (load_idx) { 4493 entry->extra1 = &min_load_idx; 4494 entry->extra2 = &max_load_idx; 4495 } 4496} 4497 4498static struct ctl_table * 4499sd_alloc_ctl_domain_table(struct sched_domain *sd) 4500{ 4501 struct ctl_table *table = sd_alloc_ctl_entry(13); 4502 4503 if (table == NULL) 4504 return NULL; 4505 4506 set_table_entry(&table[0], "min_interval", &sd->min_interval, 4507 sizeof(long), 0644, proc_doulongvec_minmax, false); 4508 set_table_entry(&table[1], "max_interval", &sd->max_interval, 4509 sizeof(long), 0644, proc_doulongvec_minmax, false); 4510 set_table_entry(&table[2], "busy_idx", &sd->busy_idx, 4511 sizeof(int), 0644, proc_dointvec_minmax, true); 4512 set_table_entry(&table[3], "idle_idx", &sd->idle_idx, 4513 sizeof(int), 0644, proc_dointvec_minmax, true); 4514 set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx, 4515 sizeof(int), 0644, proc_dointvec_minmax, true); 4516 set_table_entry(&table[5], "wake_idx", &sd->wake_idx, 4517 sizeof(int), 0644, proc_dointvec_minmax, true); 4518 set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx, 4519 sizeof(int), 0644, proc_dointvec_minmax, true); 4520 set_table_entry(&table[7], "busy_factor", &sd->busy_factor, 4521 sizeof(int), 0644, proc_dointvec_minmax, false); 4522 set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct, 4523 sizeof(int), 0644, proc_dointvec_minmax, false); 4524 set_table_entry(&table[9], "cache_nice_tries", 4525 &sd->cache_nice_tries, 4526 sizeof(int), 0644, proc_dointvec_minmax, false); 4527 set_table_entry(&table[10], "flags", &sd->flags, 4528 sizeof(int), 0644, proc_dointvec_minmax, false); 4529 set_table_entry(&table[11], "name", sd->name, 4530 CORENAME_MAX_SIZE, 0444, proc_dostring, false); 4531 /* &table[12] is terminator */ 4532 4533 return table; 4534} 4535 4536static ctl_table *sd_alloc_ctl_cpu_table(int cpu) 4537{ 4538 struct ctl_table *entry, *table; 4539 struct sched_domain *sd; 4540 int domain_num = 0, i; 4541 char buf[32]; 4542 4543 for_each_domain(cpu, sd) 4544 domain_num++; 4545 entry = table = sd_alloc_ctl_entry(domain_num + 1); 4546 if (table == NULL) 4547 return NULL; 4548 4549 i = 0; 4550 for_each_domain(cpu, sd) { 4551 snprintf(buf, 32, "domain%d", i); 4552 entry->procname = kstrdup(buf, GFP_KERNEL); 4553 entry->mode = 0555; 4554 entry->child = sd_alloc_ctl_domain_table(sd); 4555 entry++; 4556 i++; 4557 } 4558 return table; 4559} 4560 4561static struct ctl_table_header *sd_sysctl_header; 4562static void register_sched_domain_sysctl(void) 4563{ 4564 int i, cpu_num = num_possible_cpus(); 4565 struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1); 4566 char buf[32]; 4567 4568 WARN_ON(sd_ctl_dir[0].child); 4569 sd_ctl_dir[0].child = entry; 4570 4571 if (entry == NULL) 4572 return; 4573 4574 for_each_possible_cpu(i) { 4575 snprintf(buf, 32, "cpu%d", i); 4576 entry->procname = kstrdup(buf, GFP_KERNEL); 4577 entry->mode = 0555; 4578 entry->child = sd_alloc_ctl_cpu_table(i); 4579 entry++; 4580 } 4581 4582 WARN_ON(sd_sysctl_header); 4583 sd_sysctl_header = register_sysctl_table(sd_ctl_root); 4584} 4585 4586/* may be called multiple times per register */ 4587static void unregister_sched_domain_sysctl(void) 4588{ 4589 if (sd_sysctl_header) 4590 unregister_sysctl_table(sd_sysctl_header); 4591 sd_sysctl_header = NULL; 4592 if (sd_ctl_dir[0].child) 4593 sd_free_ctl_entry(&sd_ctl_dir[0].child); 4594} 4595#else 4596static void register_sched_domain_sysctl(void) 4597{ 4598} 4599static void unregister_sched_domain_sysctl(void) 4600{ 4601} 4602#endif 4603 4604static void set_rq_online(struct rq *rq) 4605{ 4606 if (!rq->online) { 4607 const struct sched_class *class; 4608 4609 cpumask_set_cpu(rq->cpu, rq->rd->online); 4610 rq->online = 1; 4611 4612 for_each_class(class) { 4613 if (class->rq_online) 4614 class->rq_online(rq); 4615 } 4616 } 4617} 4618 4619static void set_rq_offline(struct rq *rq) 4620{ 4621 if (rq->online) { 4622 const struct sched_class *class; 4623 4624 for_each_class(class) { 4625 if (class->rq_offline) 4626 class->rq_offline(rq); 4627 } 4628 4629 cpumask_clear_cpu(rq->cpu, rq->rd->online); 4630 rq->online = 0; 4631 } 4632} 4633 4634/* 4635 * migration_call - callback that gets triggered when a CPU is added. 4636 * Here we can start up the necessary migration thread for the new CPU. 4637 */ 4638static int __cpuinit 4639migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) 4640{ 4641 int cpu = (long)hcpu; 4642 unsigned long flags; 4643 struct rq *rq = cpu_rq(cpu); 4644 4645 switch (action & ~CPU_TASKS_FROZEN) { 4646 4647 case CPU_UP_PREPARE: 4648 rq->calc_load_update = calc_load_update; 4649 break; 4650 4651 case CPU_ONLINE: 4652 /* Update our root-domain */ 4653 raw_spin_lock_irqsave(&rq->lock, flags); 4654 if (rq->rd) { 4655 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 4656 4657 set_rq_online(rq); 4658 } 4659 raw_spin_unlock_irqrestore(&rq->lock, flags); 4660 break; 4661 4662#ifdef CONFIG_HOTPLUG_CPU 4663 case CPU_DYING: 4664 sched_ttwu_pending(); 4665 /* Update our root-domain */ 4666 raw_spin_lock_irqsave(&rq->lock, flags); 4667 if (rq->rd) { 4668 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 4669 set_rq_offline(rq); 4670 } 4671 migrate_tasks(cpu); 4672 BUG_ON(rq->nr_running != 1); /* the migration thread */ 4673 raw_spin_unlock_irqrestore(&rq->lock, flags); 4674 break; 4675 4676 case CPU_DEAD: 4677 calc_load_migrate(rq); 4678 break; 4679#endif 4680 } 4681 4682 update_max_interval(); 4683 4684 return NOTIFY_OK; 4685} 4686 4687/* 4688 * Register at high priority so that task migration (migrate_all_tasks) 4689 * happens before everything else. This has to be lower priority than 4690 * the notifier in the perf_event subsystem, though. 4691 */ 4692static struct notifier_block __cpuinitdata migration_notifier = { 4693 .notifier_call = migration_call, 4694 .priority = CPU_PRI_MIGRATION, 4695}; 4696 4697static int __cpuinit sched_cpu_active(struct notifier_block *nfb, 4698 unsigned long action, void *hcpu) 4699{ 4700 switch (action & ~CPU_TASKS_FROZEN) { 4701 case CPU_STARTING: 4702 case CPU_DOWN_FAILED: 4703 set_cpu_active((long)hcpu, true); 4704 return NOTIFY_OK; 4705 default: 4706 return NOTIFY_DONE; 4707 } 4708} 4709 4710static int __cpuinit sched_cpu_inactive(struct notifier_block *nfb, 4711 unsigned long action, void *hcpu) 4712{ 4713 switch (action & ~CPU_TASKS_FROZEN) { 4714 case CPU_DOWN_PREPARE: 4715 set_cpu_active((long)hcpu, false); 4716 return NOTIFY_OK; 4717 default: 4718 return NOTIFY_DONE; 4719 } 4720} 4721 4722static int __init migration_init(void) 4723{ 4724 void *cpu = (void *)(long)smp_processor_id(); 4725 int err; 4726 4727 /* Initialize migration for the boot CPU */ 4728 err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu); 4729 BUG_ON(err == NOTIFY_BAD); 4730 migration_call(&migration_notifier, CPU_ONLINE, cpu); 4731 register_cpu_notifier(&migration_notifier); 4732 4733 /* Register cpu active notifiers */ 4734 cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE); 4735 cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE); 4736 4737 return 0; 4738} 4739early_initcall(migration_init); 4740#endif 4741 4742#ifdef CONFIG_SMP 4743 4744static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */ 4745 4746#ifdef CONFIG_SCHED_DEBUG 4747 4748static __read_mostly int sched_debug_enabled; 4749 4750static int __init sched_debug_setup(char *str) 4751{ 4752 sched_debug_enabled = 1; 4753 4754 return 0; 4755} 4756early_param("sched_debug", sched_debug_setup); 4757 4758static inline bool sched_debug(void) 4759{ 4760 return sched_debug_enabled; 4761} 4762 4763static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 4764 struct cpumask *groupmask) 4765{ 4766 struct sched_group *group = sd->groups; 4767 char str[256]; 4768 4769 cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd)); 4770 cpumask_clear(groupmask); 4771 4772 printk(KERN_DEBUG "%*s domain %d: ", level, "", level); 4773 4774 if (!(sd->flags & SD_LOAD_BALANCE)) { 4775 printk("does not load-balance\n"); 4776 if (sd->parent) 4777 printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain" 4778 " has parent"); 4779 return -1; 4780 } 4781 4782 printk(KERN_CONT "span %s level %s\n", str, sd->name); 4783 4784 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) { 4785 printk(KERN_ERR "ERROR: domain->span does not contain " 4786 "CPU%d\n", cpu); 4787 } 4788 if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) { 4789 printk(KERN_ERR "ERROR: domain->groups does not contain" 4790 " CPU%d\n", cpu); 4791 } 4792 4793 printk(KERN_DEBUG "%*s groups:", level + 1, ""); 4794 do { 4795 if (!group) { 4796 printk("\n"); 4797 printk(KERN_ERR "ERROR: group is NULL\n"); 4798 break; 4799 } 4800 4801 /* 4802 * Even though we initialize ->power to something semi-sane, 4803 * we leave power_orig unset. This allows us to detect if 4804 * domain iteration is still funny without causing /0 traps. 4805 */ 4806 if (!group->sgp->power_orig) { 4807 printk(KERN_CONT "\n"); 4808 printk(KERN_ERR "ERROR: domain->cpu_power not " 4809 "set\n"); 4810 break; 4811 } 4812 4813 if (!cpumask_weight(sched_group_cpus(group))) { 4814 printk(KERN_CONT "\n"); 4815 printk(KERN_ERR "ERROR: empty group\n"); 4816 break; 4817 } 4818 4819 if (!(sd->flags & SD_OVERLAP) && 4820 cpumask_intersects(groupmask, sched_group_cpus(group))) { 4821 printk(KERN_CONT "\n"); 4822 printk(KERN_ERR "ERROR: repeated CPUs\n"); 4823 break; 4824 } 4825 4826 cpumask_or(groupmask, groupmask, sched_group_cpus(group)); 4827 4828 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group)); 4829 4830 printk(KERN_CONT " %s", str); 4831 if (group->sgp->power != SCHED_POWER_SCALE) { 4832 printk(KERN_CONT " (cpu_power = %d)", 4833 group->sgp->power); 4834 } 4835 4836 group = group->next; 4837 } while (group != sd->groups); 4838 printk(KERN_CONT "\n"); 4839 4840 if (!cpumask_equal(sched_domain_span(sd), groupmask)) 4841 printk(KERN_ERR "ERROR: groups don't span domain->span\n"); 4842 4843 if (sd->parent && 4844 !cpumask_subset(groupmask, sched_domain_span(sd->parent))) 4845 printk(KERN_ERR "ERROR: parent span is not a superset " 4846 "of domain->span\n"); 4847 return 0; 4848} 4849 4850static void sched_domain_debug(struct sched_domain *sd, int cpu) 4851{ 4852 int level = 0; 4853 4854 if (!sched_debug_enabled) 4855 return; 4856 4857 if (!sd) { 4858 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); 4859 return; 4860 } 4861 4862 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); 4863 4864 for (;;) { 4865 if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask)) 4866 break; 4867 level++; 4868 sd = sd->parent; 4869 if (!sd) 4870 break; 4871 } 4872} 4873#else /* !CONFIG_SCHED_DEBUG */ 4874# define sched_domain_debug(sd, cpu) do { } while (0) 4875static inline bool sched_debug(void) 4876{ 4877 return false; 4878} 4879#endif /* CONFIG_SCHED_DEBUG */ 4880 4881static int sd_degenerate(struct sched_domain *sd) 4882{ 4883 if (cpumask_weight(sched_domain_span(sd)) == 1) 4884 return 1; 4885 4886 /* Following flags need at least 2 groups */ 4887 if (sd->flags & (SD_LOAD_BALANCE | 4888 SD_BALANCE_NEWIDLE | 4889 SD_BALANCE_FORK | 4890 SD_BALANCE_EXEC | 4891 SD_SHARE_CPUPOWER | 4892 SD_SHARE_PKG_RESOURCES)) { 4893 if (sd->groups != sd->groups->next) 4894 return 0; 4895 } 4896 4897 /* Following flags don't use groups */ 4898 if (sd->flags & (SD_WAKE_AFFINE)) 4899 return 0; 4900 4901 return 1; 4902} 4903 4904static int 4905sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) 4906{ 4907 unsigned long cflags = sd->flags, pflags = parent->flags; 4908 4909 if (sd_degenerate(parent)) 4910 return 1; 4911 4912 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent))) 4913 return 0; 4914 4915 /* Flags needing groups don't count if only 1 group in parent */ 4916 if (parent->groups == parent->groups->next) { 4917 pflags &= ~(SD_LOAD_BALANCE | 4918 SD_BALANCE_NEWIDLE | 4919 SD_BALANCE_FORK | 4920 SD_BALANCE_EXEC | 4921 SD_SHARE_CPUPOWER | 4922 SD_SHARE_PKG_RESOURCES); 4923 if (nr_node_ids == 1) 4924 pflags &= ~SD_SERIALIZE; 4925 } 4926 if (~cflags & pflags) 4927 return 0; 4928 4929 return 1; 4930} 4931 4932static void free_rootdomain(struct rcu_head *rcu) 4933{ 4934 struct root_domain *rd = container_of(rcu, struct root_domain, rcu); 4935 4936 cpupri_cleanup(&rd->cpupri); 4937 free_cpumask_var(rd->rto_mask); 4938 free_cpumask_var(rd->online); 4939 free_cpumask_var(rd->span); 4940 kfree(rd); 4941} 4942 4943static void rq_attach_root(struct rq *rq, struct root_domain *rd) 4944{ 4945 struct root_domain *old_rd = NULL; 4946 unsigned long flags; 4947 4948 raw_spin_lock_irqsave(&rq->lock, flags); 4949 4950 if (rq->rd) { 4951 old_rd = rq->rd; 4952 4953 if (cpumask_test_cpu(rq->cpu, old_rd->online)) 4954 set_rq_offline(rq); 4955 4956 cpumask_clear_cpu(rq->cpu, old_rd->span); 4957 4958 /* 4959 * If we dont want to free the old_rt yet then 4960 * set old_rd to NULL to skip the freeing later 4961 * in this function: 4962 */ 4963 if (!atomic_dec_and_test(&old_rd->refcount)) 4964 old_rd = NULL; 4965 } 4966 4967 atomic_inc(&rd->refcount); 4968 rq->rd = rd; 4969 4970 cpumask_set_cpu(rq->cpu, rd->span); 4971 if (cpumask_test_cpu(rq->cpu, cpu_active_mask)) 4972 set_rq_online(rq); 4973 4974 raw_spin_unlock_irqrestore(&rq->lock, flags); 4975 4976 if (old_rd) 4977 call_rcu_sched(&old_rd->rcu, free_rootdomain); 4978} 4979 4980static int init_rootdomain(struct root_domain *rd) 4981{ 4982 memset(rd, 0, sizeof(*rd)); 4983 4984 if (!alloc_cpumask_var(&rd->span, GFP_KERNEL)) 4985 goto out; 4986 if (!alloc_cpumask_var(&rd->online, GFP_KERNEL)) 4987 goto free_span; 4988 if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL)) 4989 goto free_online; 4990 4991 if (cpupri_init(&rd->cpupri) != 0) 4992 goto free_rto_mask; 4993 return 0; 4994 4995free_rto_mask: 4996 free_cpumask_var(rd->rto_mask); 4997free_online: 4998 free_cpumask_var(rd->online); 4999free_span: 5000 free_cpumask_var(rd->span); 5001out: 5002 return -ENOMEM; 5003} 5004 5005/* 5006 * By default the system creates a single root-domain with all cpus as 5007 * members (mimicking the global state we have today). 5008 */ 5009struct root_domain def_root_domain; 5010 5011static void init_defrootdomain(void) 5012{ 5013 init_rootdomain(&def_root_domain); 5014 5015 atomic_set(&def_root_domain.refcount, 1); 5016} 5017 5018static struct root_domain *alloc_rootdomain(void) 5019{ 5020 struct root_domain *rd; 5021 5022 rd = kmalloc(sizeof(*rd), GFP_KERNEL); 5023 if (!rd) 5024 return NULL; 5025 5026 if (init_rootdomain(rd) != 0) { 5027 kfree(rd); 5028 return NULL; 5029 } 5030 5031 return rd; 5032} 5033 5034static void free_sched_groups(struct sched_group *sg, int free_sgp) 5035{ 5036 struct sched_group *tmp, *first; 5037 5038 if (!sg) 5039 return; 5040 5041 first = sg; 5042 do { 5043 tmp = sg->next; 5044 5045 if (free_sgp && atomic_dec_and_test(&sg->sgp->ref)) 5046 kfree(sg->sgp); 5047 5048 kfree(sg); 5049 sg = tmp; 5050 } while (sg != first); 5051} 5052 5053static void free_sched_domain(struct rcu_head *rcu) 5054{ 5055 struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu); 5056 5057 /* 5058 * If its an overlapping domain it has private groups, iterate and 5059 * nuke them all. 5060 */ 5061 if (sd->flags & SD_OVERLAP) { 5062 free_sched_groups(sd->groups, 1); 5063 } else if (atomic_dec_and_test(&sd->groups->ref)) { 5064 kfree(sd->groups->sgp); 5065 kfree(sd->groups); 5066 } 5067 kfree(sd); 5068} 5069 5070static void destroy_sched_domain(struct sched_domain *sd, int cpu) 5071{ 5072 call_rcu(&sd->rcu, free_sched_domain); 5073} 5074 5075static void destroy_sched_domains(struct sched_domain *sd, int cpu) 5076{ 5077 for (; sd; sd = sd->parent) 5078 destroy_sched_domain(sd, cpu); 5079} 5080 5081/* 5082 * Keep a special pointer to the highest sched_domain that has 5083 * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this 5084 * allows us to avoid some pointer chasing select_idle_sibling(). 5085 * 5086 * Also keep a unique ID per domain (we use the first cpu number in 5087 * the cpumask of the domain), this allows us to quickly tell if 5088 * two cpus are in the same cache domain, see cpus_share_cache(). 5089 */ 5090DEFINE_PER_CPU(struct sched_domain *, sd_llc); 5091DEFINE_PER_CPU(int, sd_llc_id); 5092 5093static void update_top_cache_domain(int cpu) 5094{ 5095 struct sched_domain *sd; 5096 int id = cpu; 5097 5098 sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES); 5099 if (sd) 5100 id = cpumask_first(sched_domain_span(sd)); 5101 5102 rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); 5103 per_cpu(sd_llc_id, cpu) = id; 5104} 5105 5106/* 5107 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must 5108 * hold the hotplug lock. 5109 */ 5110static void 5111cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) 5112{ 5113 struct rq *rq = cpu_rq(cpu); 5114 struct sched_domain *tmp; 5115 5116 /* Remove the sched domains which do not contribute to scheduling. */ 5117 for (tmp = sd; tmp; ) { 5118 struct sched_domain *parent = tmp->parent; 5119 if (!parent) 5120 break; 5121 5122 if (sd_parent_degenerate(tmp, parent)) { 5123 tmp->parent = parent->parent; 5124 if (parent->parent) 5125 parent->parent->child = tmp; 5126 destroy_sched_domain(parent, cpu); 5127 } else 5128 tmp = tmp->parent; 5129 } 5130 5131 if (sd && sd_degenerate(sd)) { 5132 tmp = sd; 5133 sd = sd->parent; 5134 destroy_sched_domain(tmp, cpu); 5135 if (sd) 5136 sd->child = NULL; 5137 } 5138 5139 sched_domain_debug(sd, cpu); 5140 5141 rq_attach_root(rq, rd); 5142 tmp = rq->sd; 5143 rcu_assign_pointer(rq->sd, sd); 5144 destroy_sched_domains(tmp, cpu); 5145 5146 update_top_cache_domain(cpu); 5147} 5148 5149/* cpus with isolated domains */ 5150static cpumask_var_t cpu_isolated_map; 5151 5152/* Setup the mask of cpus configured for isolated domains */ 5153static int __init isolated_cpu_setup(char *str) 5154{ 5155 alloc_bootmem_cpumask_var(&cpu_isolated_map); 5156 cpulist_parse(str, cpu_isolated_map); 5157 return 1; 5158} 5159 5160__setup("isolcpus=", isolated_cpu_setup); 5161 5162static const struct cpumask *cpu_cpu_mask(int cpu) 5163{ 5164 return cpumask_of_node(cpu_to_node(cpu)); 5165} 5166 5167struct sd_data { 5168 struct sched_domain **__percpu sd; 5169 struct sched_group **__percpu sg; 5170 struct sched_group_power **__percpu sgp; 5171}; 5172 5173struct s_data { 5174 struct sched_domain ** __percpu sd; 5175 struct root_domain *rd; 5176}; 5177 5178enum s_alloc { 5179 sa_rootdomain, 5180 sa_sd, 5181 sa_sd_storage, 5182 sa_none, 5183}; 5184 5185struct sched_domain_topology_level; 5186 5187typedef struct sched_domain *(*sched_domain_init_f)(struct sched_domain_topology_level *tl, int cpu); 5188typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); 5189 5190#define SDTL_OVERLAP 0x01 5191 5192struct sched_domain_topology_level { 5193 sched_domain_init_f init; 5194 sched_domain_mask_f mask; 5195 int flags; 5196 int numa_level; 5197 struct sd_data data; 5198}; 5199 5200/* 5201 * Build an iteration mask that can exclude certain CPUs from the upwards 5202 * domain traversal. 5203 * 5204 * Asymmetric node setups can result in situations where the domain tree is of 5205 * unequal depth, make sure to skip domains that already cover the entire 5206 * range. 5207 * 5208 * In that case build_sched_domains() will have terminated the iteration early 5209 * and our sibling sd spans will be empty. Domains should always include the 5210 * cpu they're built on, so check that. 5211 * 5212 */ 5213static void build_group_mask(struct sched_domain *sd, struct sched_group *sg) 5214{ 5215 const struct cpumask *span = sched_domain_span(sd); 5216 struct sd_data *sdd = sd->private; 5217 struct sched_domain *sibling; 5218 int i; 5219 5220 for_each_cpu(i, span) { 5221 sibling = *per_cpu_ptr(sdd->sd, i); 5222 if (!cpumask_test_cpu(i, sched_domain_span(sibling))) 5223 continue; 5224 5225 cpumask_set_cpu(i, sched_group_mask(sg)); 5226 } 5227} 5228 5229/* 5230 * Return the canonical balance cpu for this group, this is the first cpu 5231 * of this group that's also in the iteration mask. 5232 */ 5233int group_balance_cpu(struct sched_group *sg) 5234{ 5235 return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg)); 5236} 5237 5238static int 5239build_overlap_sched_groups(struct sched_domain *sd, int cpu) 5240{ 5241 struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg; 5242 const struct cpumask *span = sched_domain_span(sd); 5243 struct cpumask *covered = sched_domains_tmpmask; 5244 struct sd_data *sdd = sd->private; 5245 struct sched_domain *child; 5246 int i; 5247 5248 cpumask_clear(covered); 5249 5250 for_each_cpu(i, span) { 5251 struct cpumask *sg_span; 5252 5253 if (cpumask_test_cpu(i, covered)) 5254 continue; 5255 5256 child = *per_cpu_ptr(sdd->sd, i); 5257 5258 /* See the comment near build_group_mask(). */ 5259 if (!cpumask_test_cpu(i, sched_domain_span(child))) 5260 continue; 5261 5262 sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), 5263 GFP_KERNEL, cpu_to_node(cpu)); 5264 5265 if (!sg) 5266 goto fail; 5267 5268 sg_span = sched_group_cpus(sg); 5269 if (child->child) { 5270 child = child->child; 5271 cpumask_copy(sg_span, sched_domain_span(child)); 5272 } else 5273 cpumask_set_cpu(i, sg_span); 5274 5275 cpumask_or(covered, covered, sg_span); 5276 5277 sg->sgp = *per_cpu_ptr(sdd->sgp, i); 5278 if (atomic_inc_return(&sg->sgp->ref) == 1) 5279 build_group_mask(sd, sg); 5280 5281 /* 5282 * Initialize sgp->power such that even if we mess up the 5283 * domains and no possible iteration will get us here, we won't 5284 * die on a /0 trap. 5285 */ 5286 sg->sgp->power = SCHED_POWER_SCALE * cpumask_weight(sg_span); 5287 5288 /* 5289 * Make sure the first group of this domain contains the 5290 * canonical balance cpu. Otherwise the sched_domain iteration 5291 * breaks. See update_sg_lb_stats(). 5292 */ 5293 if ((!groups && cpumask_test_cpu(cpu, sg_span)) || 5294 group_balance_cpu(sg) == cpu) 5295 groups = sg; 5296 5297 if (!first) 5298 first = sg; 5299 if (last) 5300 last->next = sg; 5301 last = sg; 5302 last->next = first; 5303 } 5304 sd->groups = groups; 5305 5306 return 0; 5307 5308fail: 5309 free_sched_groups(first, 0); 5310 5311 return -ENOMEM; 5312} 5313 5314static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg) 5315{ 5316 struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); 5317 struct sched_domain *child = sd->child; 5318 5319 if (child) 5320 cpu = cpumask_first(sched_domain_span(child)); 5321 5322 if (sg) { 5323 *sg = *per_cpu_ptr(sdd->sg, cpu); 5324 (*sg)->sgp = *per_cpu_ptr(sdd->sgp, cpu); 5325 atomic_set(&(*sg)->sgp->ref, 1); /* for claim_allocations */ 5326 } 5327 5328 return cpu; 5329} 5330 5331/* 5332 * build_sched_groups will build a circular linked list of the groups 5333 * covered by the given span, and will set each group's ->cpumask correctly, 5334 * and ->cpu_power to 0. 5335 * 5336 * Assumes the sched_domain tree is fully constructed 5337 */ 5338static int 5339build_sched_groups(struct sched_domain *sd, int cpu) 5340{ 5341 struct sched_group *first = NULL, *last = NULL; 5342 struct sd_data *sdd = sd->private; 5343 const struct cpumask *span = sched_domain_span(sd); 5344 struct cpumask *covered; 5345 int i; 5346 5347 get_group(cpu, sdd, &sd->groups); 5348 atomic_inc(&sd->groups->ref); 5349 5350 if (cpu != cpumask_first(sched_domain_span(sd))) 5351 return 0; 5352 5353 lockdep_assert_held(&sched_domains_mutex); 5354 covered = sched_domains_tmpmask; 5355 5356 cpumask_clear(covered); 5357 5358 for_each_cpu(i, span) { 5359 struct sched_group *sg; 5360 int group = get_group(i, sdd, &sg); 5361 int j; 5362 5363 if (cpumask_test_cpu(i, covered)) 5364 continue; 5365 5366 cpumask_clear(sched_group_cpus(sg)); 5367 sg->sgp->power = 0; 5368 cpumask_setall(sched_group_mask(sg)); 5369 5370 for_each_cpu(j, span) { 5371 if (get_group(j, sdd, NULL) != group) 5372 continue; 5373 5374 cpumask_set_cpu(j, covered); 5375 cpumask_set_cpu(j, sched_group_cpus(sg)); 5376 } 5377 5378 if (!first) 5379 first = sg; 5380 if (last) 5381 last->next = sg; 5382 last = sg; 5383 } 5384 last->next = first; 5385 5386 return 0; 5387} 5388 5389/* 5390 * Initialize sched groups cpu_power. 5391 * 5392 * cpu_power indicates the capacity of sched group, which is used while 5393 * distributing the load between different sched groups in a sched domain. 5394 * Typically cpu_power for all the groups in a sched domain will be same unless 5395 * there are asymmetries in the topology. If there are asymmetries, group 5396 * having more cpu_power will pickup more load compared to the group having 5397 * less cpu_power. 5398 */ 5399static void init_sched_groups_power(int cpu, struct sched_domain *sd) 5400{ 5401 struct sched_group *sg = sd->groups; 5402 5403 WARN_ON(!sd || !sg); 5404 5405 do { 5406 sg->group_weight = cpumask_weight(sched_group_cpus(sg)); 5407 sg = sg->next; 5408 } while (sg != sd->groups); 5409 5410 if (cpu != group_balance_cpu(sg)) 5411 return; 5412 5413 update_group_power(sd, cpu); 5414 atomic_set(&sg->sgp->nr_busy_cpus, sg->group_weight); 5415} 5416 5417int __weak arch_sd_sibling_asym_packing(void) 5418{ 5419 return 0*SD_ASYM_PACKING; 5420} 5421 5422/* 5423 * Initializers for schedule domains 5424 * Non-inlined to reduce accumulated stack pressure in build_sched_domains() 5425 */ 5426 5427#ifdef CONFIG_SCHED_DEBUG 5428# define SD_INIT_NAME(sd, type) sd->name = #type 5429#else 5430# define SD_INIT_NAME(sd, type) do { } while (0) 5431#endif 5432 5433#define SD_INIT_FUNC(type) \ 5434static noinline struct sched_domain * \ 5435sd_init_##type(struct sched_domain_topology_level *tl, int cpu) \ 5436{ \ 5437 struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu); \ 5438 *sd = SD_##type##_INIT; \ 5439 SD_INIT_NAME(sd, type); \ 5440 sd->private = &tl->data; \ 5441 return sd; \ 5442} 5443 5444SD_INIT_FUNC(CPU) 5445#ifdef CONFIG_SCHED_SMT 5446 SD_INIT_FUNC(SIBLING) 5447#endif 5448#ifdef CONFIG_SCHED_MC 5449 SD_INIT_FUNC(MC) 5450#endif 5451#ifdef CONFIG_SCHED_BOOK 5452 SD_INIT_FUNC(BOOK) 5453#endif 5454 5455static int default_relax_domain_level = -1; 5456int sched_domain_level_max; 5457 5458static int __init setup_relax_domain_level(char *str) 5459{ 5460 if (kstrtoint(str, 0, &default_relax_domain_level)) 5461 pr_warn("Unable to set relax_domain_level\n"); 5462 5463 return 1; 5464} 5465__setup("relax_domain_level=", setup_relax_domain_level); 5466 5467static void set_domain_attribute(struct sched_domain *sd, 5468 struct sched_domain_attr *attr) 5469{ 5470 int request; 5471 5472 if (!attr || attr->relax_domain_level < 0) { 5473 if (default_relax_domain_level < 0) 5474 return; 5475 else 5476 request = default_relax_domain_level; 5477 } else 5478 request = attr->relax_domain_level; 5479 if (request < sd->level) { 5480 /* turn off idle balance on this domain */ 5481 sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); 5482 } else { 5483 /* turn on idle balance on this domain */ 5484 sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); 5485 } 5486} 5487 5488static void __sdt_free(const struct cpumask *cpu_map); 5489static int __sdt_alloc(const struct cpumask *cpu_map); 5490 5491static void __free_domain_allocs(struct s_data *d, enum s_alloc what, 5492 const struct cpumask *cpu_map) 5493{ 5494 switch (what) { 5495 case sa_rootdomain: 5496 if (!atomic_read(&d->rd->refcount)) 5497 free_rootdomain(&d->rd->rcu); /* fall through */ 5498 case sa_sd: 5499 free_percpu(d->sd); /* fall through */ 5500 case sa_sd_storage: 5501 __sdt_free(cpu_map); /* fall through */ 5502 case sa_none: 5503 break; 5504 } 5505} 5506 5507static enum s_alloc __visit_domain_allocation_hell(struct s_data *d, 5508 const struct cpumask *cpu_map) 5509{ 5510 memset(d, 0, sizeof(*d)); 5511 5512 if (__sdt_alloc(cpu_map)) 5513 return sa_sd_storage; 5514 d->sd = alloc_percpu(struct sched_domain *); 5515 if (!d->sd) 5516 return sa_sd_storage; 5517 d->rd = alloc_rootdomain(); 5518 if (!d->rd) 5519 return sa_sd; 5520 return sa_rootdomain; 5521} 5522 5523/* 5524 * NULL the sd_data elements we've used to build the sched_domain and 5525 * sched_group structure so that the subsequent __free_domain_allocs() 5526 * will not free the data we're using. 5527 */ 5528static void claim_allocations(int cpu, struct sched_domain *sd) 5529{ 5530 struct sd_data *sdd = sd->private; 5531 5532 WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd); 5533 *per_cpu_ptr(sdd->sd, cpu) = NULL; 5534 5535 if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref)) 5536 *per_cpu_ptr(sdd->sg, cpu) = NULL; 5537 5538 if (atomic_read(&(*per_cpu_ptr(sdd->sgp, cpu))->ref)) 5539 *per_cpu_ptr(sdd->sgp, cpu) = NULL; 5540} 5541 5542#ifdef CONFIG_SCHED_SMT 5543static const struct cpumask *cpu_smt_mask(int cpu) 5544{ 5545 return topology_thread_cpumask(cpu); 5546} 5547#endif 5548 5549/* 5550 * Topology list, bottom-up. 5551 */ 5552static struct sched_domain_topology_level default_topology[] = { 5553#ifdef CONFIG_SCHED_SMT 5554 { sd_init_SIBLING, cpu_smt_mask, }, 5555#endif 5556#ifdef CONFIG_SCHED_MC 5557 { sd_init_MC, cpu_coregroup_mask, }, 5558#endif 5559#ifdef CONFIG_SCHED_BOOK 5560 { sd_init_BOOK, cpu_book_mask, }, 5561#endif 5562 { sd_init_CPU, cpu_cpu_mask, }, 5563 { NULL, }, 5564}; 5565 5566static struct sched_domain_topology_level *sched_domain_topology = default_topology; 5567 5568#ifdef CONFIG_NUMA 5569 5570static int sched_domains_numa_levels; 5571static int *sched_domains_numa_distance; 5572static struct cpumask ***sched_domains_numa_masks; 5573static int sched_domains_curr_level; 5574 5575static inline int sd_local_flags(int level) 5576{ 5577 if (sched_domains_numa_distance[level] > RECLAIM_DISTANCE) 5578 return 0; 5579 5580 return SD_BALANCE_EXEC | SD_BALANCE_FORK | SD_WAKE_AFFINE; 5581} 5582 5583static struct sched_domain * 5584sd_numa_init(struct sched_domain_topology_level *tl, int cpu) 5585{ 5586 struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu); 5587 int level = tl->numa_level; 5588 int sd_weight = cpumask_weight( 5589 sched_domains_numa_masks[level][cpu_to_node(cpu)]); 5590 5591 *sd = (struct sched_domain){ 5592 .min_interval = sd_weight, 5593 .max_interval = 2*sd_weight, 5594 .busy_factor = 32, 5595 .imbalance_pct = 125, 5596 .cache_nice_tries = 2, 5597 .busy_idx = 3, 5598 .idle_idx = 2, 5599 .newidle_idx = 0, 5600 .wake_idx = 0, 5601 .forkexec_idx = 0, 5602 5603 .flags = 1*SD_LOAD_BALANCE 5604 | 1*SD_BALANCE_NEWIDLE 5605 | 0*SD_BALANCE_EXEC 5606 | 0*SD_BALANCE_FORK 5607 | 0*SD_BALANCE_WAKE 5608 | 0*SD_WAKE_AFFINE 5609 | 0*SD_SHARE_CPUPOWER 5610 | 0*SD_SHARE_PKG_RESOURCES 5611 | 1*SD_SERIALIZE 5612 | 0*SD_PREFER_SIBLING 5613 | sd_local_flags(level) 5614 , 5615 .last_balance = jiffies, 5616 .balance_interval = sd_weight, 5617 }; 5618 SD_INIT_NAME(sd, NUMA); 5619 sd->private = &tl->data; 5620 5621 /* 5622 * Ugly hack to pass state to sd_numa_mask()... 5623 */ 5624 sched_domains_curr_level = tl->numa_level; 5625 5626 return sd; 5627} 5628 5629static const struct cpumask *sd_numa_mask(int cpu) 5630{ 5631 return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)]; 5632} 5633 5634static void sched_numa_warn(const char *str) 5635{ 5636 static int done = false; 5637 int i,j; 5638 5639 if (done) 5640 return; 5641 5642 done = true; 5643 5644 printk(KERN_WARNING "ERROR: %s\n\n", str); 5645 5646 for (i = 0; i < nr_node_ids; i++) { 5647 printk(KERN_WARNING " "); 5648 for (j = 0; j < nr_node_ids; j++) 5649 printk(KERN_CONT "%02d ", node_distance(i,j)); 5650 printk(KERN_CONT "\n"); 5651 } 5652 printk(KERN_WARNING "\n"); 5653} 5654 5655static bool find_numa_distance(int distance) 5656{ 5657 int i; 5658 5659 if (distance == node_distance(0, 0)) 5660 return true; 5661 5662 for (i = 0; i < sched_domains_numa_levels; i++) { 5663 if (sched_domains_numa_distance[i] == distance) 5664 return true; 5665 } 5666 5667 return false; 5668} 5669 5670static void sched_init_numa(void) 5671{ 5672 int next_distance, curr_distance = node_distance(0, 0); 5673 struct sched_domain_topology_level *tl; 5674 int level = 0; 5675 int i, j, k; 5676 5677 sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL); 5678 if (!sched_domains_numa_distance) 5679 return; 5680 5681 /* 5682 * O(nr_nodes^2) deduplicating selection sort -- in order to find the 5683 * unique distances in the node_distance() table. 5684 * 5685 * Assumes node_distance(0,j) includes all distances in 5686 * node_distance(i,j) in order to avoid cubic time. 5687 */ 5688 next_distance = curr_distance; 5689 for (i = 0; i < nr_node_ids; i++) { 5690 for (j = 0; j < nr_node_ids; j++) { 5691 for (k = 0; k < nr_node_ids; k++) { 5692 int distance = node_distance(i, k); 5693 5694 if (distance > curr_distance && 5695 (distance < next_distance || 5696 next_distance == curr_distance)) 5697 next_distance = distance; 5698 5699 /* 5700 * While not a strong assumption it would be nice to know 5701 * about cases where if node A is connected to B, B is not 5702 * equally connected to A. 5703 */ 5704 if (sched_debug() && node_distance(k, i) != distance) 5705 sched_numa_warn("Node-distance not symmetric"); 5706 5707 if (sched_debug() && i && !find_numa_distance(distance)) 5708 sched_numa_warn("Node-0 not representative"); 5709 } 5710 if (next_distance != curr_distance) { 5711 sched_domains_numa_distance[level++] = next_distance; 5712 sched_domains_numa_levels = level; 5713 curr_distance = next_distance; 5714 } else break; 5715 } 5716 5717 /* 5718 * In case of sched_debug() we verify the above assumption. 5719 */ 5720 if (!sched_debug()) 5721 break; 5722 } 5723 /* 5724 * 'level' contains the number of unique distances, excluding the 5725 * identity distance node_distance(i,i). 5726 * 5727 * The sched_domains_numa_distance[] array includes the actual distance 5728 * numbers. 5729 */ 5730 5731 /* 5732 * Here, we should temporarily reset sched_domains_numa_levels to 0. 5733 * If it fails to allocate memory for array sched_domains_numa_masks[][], 5734 * the array will contain less then 'level' members. This could be 5735 * dangerous when we use it to iterate array sched_domains_numa_masks[][] 5736 * in other functions. 5737 * 5738 * We reset it to 'level' at the end of this function. 5739 */ 5740 sched_domains_numa_levels = 0; 5741 5742 sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL); 5743 if (!sched_domains_numa_masks) 5744 return; 5745 5746 /* 5747 * Now for each level, construct a mask per node which contains all 5748 * cpus of nodes that are that many hops away from us. 5749 */ 5750 for (i = 0; i < level; i++) { 5751 sched_domains_numa_masks[i] = 5752 kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL); 5753 if (!sched_domains_numa_masks[i]) 5754 return; 5755 5756 for (j = 0; j < nr_node_ids; j++) { 5757 struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL); 5758 if (!mask) 5759 return; 5760 5761 sched_domains_numa_masks[i][j] = mask; 5762 5763 for (k = 0; k < nr_node_ids; k++) { 5764 if (node_distance(j, k) > sched_domains_numa_distance[i]) 5765 continue; 5766 5767 cpumask_or(mask, mask, cpumask_of_node(k)); 5768 } 5769 } 5770 } 5771 5772 tl = kzalloc((ARRAY_SIZE(default_topology) + level) * 5773 sizeof(struct sched_domain_topology_level), GFP_KERNEL); 5774 if (!tl) 5775 return; 5776 5777 /* 5778 * Copy the default topology bits.. 5779 */ 5780 for (i = 0; default_topology[i].init; i++) 5781 tl[i] = default_topology[i]; 5782 5783 /* 5784 * .. and append 'j' levels of NUMA goodness. 5785 */ 5786 for (j = 0; j < level; i++, j++) { 5787 tl[i] = (struct sched_domain_topology_level){ 5788 .init = sd_numa_init, 5789 .mask = sd_numa_mask, 5790 .flags = SDTL_OVERLAP, 5791 .numa_level = j, 5792 }; 5793 } 5794 5795 sched_domain_topology = tl; 5796 5797 sched_domains_numa_levels = level; 5798} 5799 5800static void sched_domains_numa_masks_set(int cpu) 5801{ 5802 int i, j; 5803 int node = cpu_to_node(cpu); 5804 5805 for (i = 0; i < sched_domains_numa_levels; i++) { 5806 for (j = 0; j < nr_node_ids; j++) { 5807 if (node_distance(j, node) <= sched_domains_numa_distance[i]) 5808 cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]); 5809 } 5810 } 5811} 5812 5813static void sched_domains_numa_masks_clear(int cpu) 5814{ 5815 int i, j; 5816 for (i = 0; i < sched_domains_numa_levels; i++) { 5817 for (j = 0; j < nr_node_ids; j++) 5818 cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]); 5819 } 5820} 5821 5822/* 5823 * Update sched_domains_numa_masks[level][node] array when new cpus 5824 * are onlined. 5825 */ 5826static int sched_domains_numa_masks_update(struct notifier_block *nfb, 5827 unsigned long action, 5828 void *hcpu) 5829{ 5830 int cpu = (long)hcpu; 5831 5832 switch (action & ~CPU_TASKS_FROZEN) { 5833 case CPU_ONLINE: 5834 sched_domains_numa_masks_set(cpu); 5835 break; 5836 5837 case CPU_DEAD: 5838 sched_domains_numa_masks_clear(cpu); 5839 break; 5840 5841 default: 5842 return NOTIFY_DONE; 5843 } 5844 5845 return NOTIFY_OK; 5846} 5847#else 5848static inline void sched_init_numa(void) 5849{ 5850} 5851 5852static int sched_domains_numa_masks_update(struct notifier_block *nfb, 5853 unsigned long action, 5854 void *hcpu) 5855{ 5856 return 0; 5857} 5858#endif /* CONFIG_NUMA */ 5859 5860static int __sdt_alloc(const struct cpumask *cpu_map) 5861{ 5862 struct sched_domain_topology_level *tl; 5863 int j; 5864 5865 for (tl = sched_domain_topology; tl->init; tl++) { 5866 struct sd_data *sdd = &tl->data; 5867 5868 sdd->sd = alloc_percpu(struct sched_domain *); 5869 if (!sdd->sd) 5870 return -ENOMEM; 5871 5872 sdd->sg = alloc_percpu(struct sched_group *); 5873 if (!sdd->sg) 5874 return -ENOMEM; 5875 5876 sdd->sgp = alloc_percpu(struct sched_group_power *); 5877 if (!sdd->sgp) 5878 return -ENOMEM; 5879 5880 for_each_cpu(j, cpu_map) { 5881 struct sched_domain *sd; 5882 struct sched_group *sg; 5883 struct sched_group_power *sgp; 5884 5885 sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), 5886 GFP_KERNEL, cpu_to_node(j)); 5887 if (!sd) 5888 return -ENOMEM; 5889 5890 *per_cpu_ptr(sdd->sd, j) = sd; 5891 5892 sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), 5893 GFP_KERNEL, cpu_to_node(j)); 5894 if (!sg) 5895 return -ENOMEM; 5896 5897 sg->next = sg; 5898 5899 *per_cpu_ptr(sdd->sg, j) = sg; 5900 5901 sgp = kzalloc_node(sizeof(struct sched_group_power) + cpumask_size(), 5902 GFP_KERNEL, cpu_to_node(j)); 5903 if (!sgp) 5904 return -ENOMEM; 5905 5906 *per_cpu_ptr(sdd->sgp, j) = sgp; 5907 } 5908 } 5909 5910 return 0; 5911} 5912 5913static void __sdt_free(const struct cpumask *cpu_map) 5914{ 5915 struct sched_domain_topology_level *tl; 5916 int j; 5917 5918 for (tl = sched_domain_topology; tl->init; tl++) { 5919 struct sd_data *sdd = &tl->data; 5920 5921 for_each_cpu(j, cpu_map) { 5922 struct sched_domain *sd; 5923 5924 if (sdd->sd) { 5925 sd = *per_cpu_ptr(sdd->sd, j); 5926 if (sd && (sd->flags & SD_OVERLAP)) 5927 free_sched_groups(sd->groups, 0); 5928 kfree(*per_cpu_ptr(sdd->sd, j)); 5929 } 5930 5931 if (sdd->sg) 5932 kfree(*per_cpu_ptr(sdd->sg, j)); 5933 if (sdd->sgp) 5934 kfree(*per_cpu_ptr(sdd->sgp, j)); 5935 } 5936 free_percpu(sdd->sd); 5937 sdd->sd = NULL; 5938 free_percpu(sdd->sg); 5939 sdd->sg = NULL; 5940 free_percpu(sdd->sgp); 5941 sdd->sgp = NULL; 5942 } 5943} 5944 5945struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl, 5946 const struct cpumask *cpu_map, struct sched_domain_attr *attr, 5947 struct sched_domain *child, int cpu) 5948{ 5949 struct sched_domain *sd = tl->init(tl, cpu); 5950 if (!sd) 5951 return child; 5952 5953 cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu)); 5954 if (child) { 5955 sd->level = child->level + 1; 5956 sched_domain_level_max = max(sched_domain_level_max, sd->level); 5957 child->parent = sd; 5958 } 5959 sd->child = child; 5960 set_domain_attribute(sd, attr); 5961 5962 return sd; 5963} 5964 5965/* 5966 * Build sched domains for a given set of cpus and attach the sched domains 5967 * to the individual cpus 5968 */ 5969static int build_sched_domains(const struct cpumask *cpu_map, 5970 struct sched_domain_attr *attr) 5971{ 5972 enum s_alloc alloc_state = sa_none; 5973 struct sched_domain *sd; 5974 struct s_data d; 5975 int i, ret = -ENOMEM; 5976 5977 alloc_state = __visit_domain_allocation_hell(&d, cpu_map); 5978 if (alloc_state != sa_rootdomain) 5979 goto error; 5980 5981 /* Set up domains for cpus specified by the cpu_map. */ 5982 for_each_cpu(i, cpu_map) { 5983 struct sched_domain_topology_level *tl; 5984 5985 sd = NULL; 5986 for (tl = sched_domain_topology; tl->init; tl++) { 5987 sd = build_sched_domain(tl, cpu_map, attr, sd, i); 5988 if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP)) 5989 sd->flags |= SD_OVERLAP; 5990 if (cpumask_equal(cpu_map, sched_domain_span(sd))) 5991 break; 5992 } 5993 5994 while (sd->child) 5995 sd = sd->child; 5996 5997 *per_cpu_ptr(d.sd, i) = sd; 5998 } 5999 6000 /* Build the groups for the domains */ 6001 for_each_cpu(i, cpu_map) { 6002 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { 6003 sd->span_weight = cpumask_weight(sched_domain_span(sd)); 6004 if (sd->flags & SD_OVERLAP) { 6005 if (build_overlap_sched_groups(sd, i)) 6006 goto error; 6007 } else { 6008 if (build_sched_groups(sd, i)) 6009 goto error; 6010 } 6011 } 6012 } 6013 6014 /* Calculate CPU power for physical packages and nodes */ 6015 for (i = nr_cpumask_bits-1; i >= 0; i--) { 6016 if (!cpumask_test_cpu(i, cpu_map)) 6017 continue; 6018 6019 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { 6020 claim_allocations(i, sd); 6021 init_sched_groups_power(i, sd); 6022 } 6023 } 6024 6025 /* Attach the domains */ 6026 rcu_read_lock(); 6027 for_each_cpu(i, cpu_map) { 6028 sd = *per_cpu_ptr(d.sd, i); 6029 cpu_attach_domain(sd, d.rd, i); 6030 } 6031 rcu_read_unlock(); 6032 6033 ret = 0; 6034error: 6035 __free_domain_allocs(&d, alloc_state, cpu_map); 6036 return ret; 6037} 6038 6039static cpumask_var_t *doms_cur; /* current sched domains */ 6040static int ndoms_cur; /* number of sched domains in 'doms_cur' */ 6041static struct sched_domain_attr *dattr_cur; 6042 /* attribues of custom domains in 'doms_cur' */ 6043 6044/* 6045 * Special case: If a kmalloc of a doms_cur partition (array of 6046 * cpumask) fails, then fallback to a single sched domain, 6047 * as determined by the single cpumask fallback_doms. 6048 */ 6049static cpumask_var_t fallback_doms; 6050 6051/* 6052 * arch_update_cpu_topology lets virtualized architectures update the 6053 * cpu core maps. It is supposed to return 1 if the topology changed 6054 * or 0 if it stayed the same. 6055 */ 6056int __attribute__((weak)) arch_update_cpu_topology(void) 6057{ 6058 return 0; 6059} 6060 6061cpumask_var_t *alloc_sched_domains(unsigned int ndoms) 6062{ 6063 int i; 6064 cpumask_var_t *doms; 6065 6066 doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL); 6067 if (!doms) 6068 return NULL; 6069 for (i = 0; i < ndoms; i++) { 6070 if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) { 6071 free_sched_domains(doms, i); 6072 return NULL; 6073 } 6074 } 6075 return doms; 6076} 6077 6078void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms) 6079{ 6080 unsigned int i; 6081 for (i = 0; i < ndoms; i++) 6082 free_cpumask_var(doms[i]); 6083 kfree(doms); 6084} 6085 6086/* 6087 * Set up scheduler domains and groups. Callers must hold the hotplug lock. 6088 * For now this just excludes isolated cpus, but could be used to 6089 * exclude other special cases in the future. 6090 */ 6091static int init_sched_domains(const struct cpumask *cpu_map) 6092{ 6093 int err; 6094 6095 arch_update_cpu_topology(); 6096 ndoms_cur = 1; 6097 doms_cur = alloc_sched_domains(ndoms_cur); 6098 if (!doms_cur) 6099 doms_cur = &fallback_doms; 6100 cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map); 6101 err = build_sched_domains(doms_cur[0], NULL); 6102 register_sched_domain_sysctl(); 6103 6104 return err; 6105} 6106 6107/* 6108 * Detach sched domains from a group of cpus specified in cpu_map 6109 * These cpus will now be attached to the NULL domain 6110 */ 6111static void detach_destroy_domains(const struct cpumask *cpu_map) 6112{ 6113 int i; 6114 6115 rcu_read_lock(); 6116 for_each_cpu(i, cpu_map) 6117 cpu_attach_domain(NULL, &def_root_domain, i); 6118 rcu_read_unlock(); 6119} 6120 6121/* handle null as "default" */ 6122static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur, 6123 struct sched_domain_attr *new, int idx_new) 6124{ 6125 struct sched_domain_attr tmp; 6126 6127 /* fast path */ 6128 if (!new && !cur) 6129 return 1; 6130 6131 tmp = SD_ATTR_INIT; 6132 return !memcmp(cur ? (cur + idx_cur) : &tmp, 6133 new ? (new + idx_new) : &tmp, 6134 sizeof(struct sched_domain_attr)); 6135} 6136 6137/* 6138 * Partition sched domains as specified by the 'ndoms_new' 6139 * cpumasks in the array doms_new[] of cpumasks. This compares 6140 * doms_new[] to the current sched domain partitioning, doms_cur[]. 6141 * It destroys each deleted domain and builds each new domain. 6142 * 6143 * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'. 6144 * The masks don't intersect (don't overlap.) We should setup one 6145 * sched domain for each mask. CPUs not in any of the cpumasks will 6146 * not be load balanced. If the same cpumask appears both in the 6147 * current 'doms_cur' domains and in the new 'doms_new', we can leave 6148 * it as it is. 6149 * 6150 * The passed in 'doms_new' should be allocated using 6151 * alloc_sched_domains. This routine takes ownership of it and will 6152 * free_sched_domains it when done with it. If the caller failed the 6153 * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1, 6154 * and partition_sched_domains() will fallback to the single partition 6155 * 'fallback_doms', it also forces the domains to be rebuilt. 6156 * 6157 * If doms_new == NULL it will be replaced with cpu_online_mask. 6158 * ndoms_new == 0 is a special case for destroying existing domains, 6159 * and it will not create the default domain. 6160 * 6161 * Call with hotplug lock held 6162 */ 6163void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], 6164 struct sched_domain_attr *dattr_new) 6165{ 6166 int i, j, n; 6167 int new_topology; 6168 6169 mutex_lock(&sched_domains_mutex); 6170 6171 /* always unregister in case we don't destroy any domains */ 6172 unregister_sched_domain_sysctl(); 6173 6174 /* Let architecture update cpu core mappings. */ 6175 new_topology = arch_update_cpu_topology(); 6176 6177 n = doms_new ? ndoms_new : 0; 6178 6179 /* Destroy deleted domains */ 6180 for (i = 0; i < ndoms_cur; i++) { 6181 for (j = 0; j < n && !new_topology; j++) { 6182 if (cpumask_equal(doms_cur[i], doms_new[j]) 6183 && dattrs_equal(dattr_cur, i, dattr_new, j)) 6184 goto match1; 6185 } 6186 /* no match - a current sched domain not in new doms_new[] */ 6187 detach_destroy_domains(doms_cur[i]); 6188match1: 6189 ; 6190 } 6191 6192 if (doms_new == NULL) { 6193 ndoms_cur = 0; 6194 doms_new = &fallback_doms; 6195 cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map); 6196 WARN_ON_ONCE(dattr_new); 6197 } 6198 6199 /* Build new domains */ 6200 for (i = 0; i < ndoms_new; i++) { 6201 for (j = 0; j < ndoms_cur && !new_topology; j++) { 6202 if (cpumask_equal(doms_new[i], doms_cur[j]) 6203 && dattrs_equal(dattr_new, i, dattr_cur, j)) 6204 goto match2; 6205 } 6206 /* no match - add a new doms_new */ 6207 build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL); 6208match2: 6209 ; 6210 } 6211 6212 /* Remember the new sched domains */ 6213 if (doms_cur != &fallback_doms) 6214 free_sched_domains(doms_cur, ndoms_cur); 6215 kfree(dattr_cur); /* kfree(NULL) is safe */ 6216 doms_cur = doms_new; 6217 dattr_cur = dattr_new; 6218 ndoms_cur = ndoms_new; 6219 6220 register_sched_domain_sysctl(); 6221 6222 mutex_unlock(&sched_domains_mutex); 6223} 6224 6225static int num_cpus_frozen; /* used to mark begin/end of suspend/resume */ 6226 6227/* 6228 * Update cpusets according to cpu_active mask. If cpusets are 6229 * disabled, cpuset_update_active_cpus() becomes a simple wrapper 6230 * around partition_sched_domains(). 6231 * 6232 * If we come here as part of a suspend/resume, don't touch cpusets because we 6233 * want to restore it back to its original state upon resume anyway. 6234 */ 6235static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action, 6236 void *hcpu) 6237{ 6238 switch (action) { 6239 case CPU_ONLINE_FROZEN: 6240 case CPU_DOWN_FAILED_FROZEN: 6241 6242 /* 6243 * num_cpus_frozen tracks how many CPUs are involved in suspend 6244 * resume sequence. As long as this is not the last online 6245 * operation in the resume sequence, just build a single sched 6246 * domain, ignoring cpusets. 6247 */ 6248 num_cpus_frozen--; 6249 if (likely(num_cpus_frozen)) { 6250 partition_sched_domains(1, NULL, NULL); 6251 break; 6252 } 6253 6254 /* 6255 * This is the last CPU online operation. So fall through and 6256 * restore the original sched domains by considering the 6257 * cpuset configurations. 6258 */ 6259 6260 case CPU_ONLINE: 6261 case CPU_DOWN_FAILED: 6262 cpuset_update_active_cpus(true); 6263 break; 6264 default: 6265 return NOTIFY_DONE; 6266 } 6267 return NOTIFY_OK; 6268} 6269 6270static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action, 6271 void *hcpu) 6272{ 6273 switch (action) { 6274 case CPU_DOWN_PREPARE: 6275 cpuset_update_active_cpus(false); 6276 break; 6277 case CPU_DOWN_PREPARE_FROZEN: 6278 num_cpus_frozen++; 6279 partition_sched_domains(1, NULL, NULL); 6280 break; 6281 default: 6282 return NOTIFY_DONE; 6283 } 6284 return NOTIFY_OK; 6285} 6286 6287void __init sched_init_smp(void) 6288{ 6289 cpumask_var_t non_isolated_cpus; 6290 6291 alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL); 6292 alloc_cpumask_var(&fallback_doms, GFP_KERNEL); 6293 6294 sched_init_numa(); 6295 6296 get_online_cpus(); 6297 mutex_lock(&sched_domains_mutex); 6298 init_sched_domains(cpu_active_mask); 6299 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map); 6300 if (cpumask_empty(non_isolated_cpus)) 6301 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus); 6302 mutex_unlock(&sched_domains_mutex); 6303 put_online_cpus(); 6304 6305 hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE); 6306 hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE); 6307 hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE); 6308 6309 init_hrtick(); 6310 6311 /* Move init over to a non-isolated CPU */ 6312 if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0) 6313 BUG(); 6314 sched_init_granularity(); 6315 free_cpumask_var(non_isolated_cpus); 6316 6317 init_sched_rt_class(); 6318} 6319#else 6320void __init sched_init_smp(void) 6321{ 6322 sched_init_granularity(); 6323} 6324#endif /* CONFIG_SMP */ 6325 6326const_debug unsigned int sysctl_timer_migration = 1; 6327 6328int in_sched_functions(unsigned long addr) 6329{ 6330 return in_lock_functions(addr) || 6331 (addr >= (unsigned long)__sched_text_start 6332 && addr < (unsigned long)__sched_text_end); 6333} 6334 6335#ifdef CONFIG_CGROUP_SCHED 6336/* 6337 * Default task group. 6338 * Every task in system belongs to this group at bootup. 6339 */ 6340struct task_group root_task_group; 6341LIST_HEAD(task_groups); 6342#endif 6343 6344DECLARE_PER_CPU(cpumask_var_t, load_balance_mask); 6345 6346void __init sched_init(void) 6347{ 6348 int i, j; 6349 unsigned long alloc_size = 0, ptr; 6350 6351#ifdef CONFIG_FAIR_GROUP_SCHED 6352 alloc_size += 2 * nr_cpu_ids * sizeof(void **); 6353#endif 6354#ifdef CONFIG_RT_GROUP_SCHED 6355 alloc_size += 2 * nr_cpu_ids * sizeof(void **); 6356#endif 6357#ifdef CONFIG_CPUMASK_OFFSTACK 6358 alloc_size += num_possible_cpus() * cpumask_size(); 6359#endif 6360 if (alloc_size) { 6361 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT); 6362 6363#ifdef CONFIG_FAIR_GROUP_SCHED 6364 root_task_group.se = (struct sched_entity **)ptr; 6365 ptr += nr_cpu_ids * sizeof(void **); 6366 6367 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 6368 ptr += nr_cpu_ids * sizeof(void **); 6369 6370#endif /* CONFIG_FAIR_GROUP_SCHED */ 6371#ifdef CONFIG_RT_GROUP_SCHED 6372 root_task_group.rt_se = (struct sched_rt_entity **)ptr; 6373 ptr += nr_cpu_ids * sizeof(void **); 6374 6375 root_task_group.rt_rq = (struct rt_rq **)ptr; 6376 ptr += nr_cpu_ids * sizeof(void **); 6377 6378#endif /* CONFIG_RT_GROUP_SCHED */ 6379#ifdef CONFIG_CPUMASK_OFFSTACK 6380 for_each_possible_cpu(i) { 6381 per_cpu(load_balance_mask, i) = (void *)ptr; 6382 ptr += cpumask_size(); 6383 } 6384#endif /* CONFIG_CPUMASK_OFFSTACK */ 6385 } 6386 6387#ifdef CONFIG_SMP 6388 init_defrootdomain(); 6389#endif 6390 6391 init_rt_bandwidth(&def_rt_bandwidth, 6392 global_rt_period(), global_rt_runtime()); 6393 6394#ifdef CONFIG_RT_GROUP_SCHED 6395 init_rt_bandwidth(&root_task_group.rt_bandwidth, 6396 global_rt_period(), global_rt_runtime()); 6397#endif /* CONFIG_RT_GROUP_SCHED */ 6398 6399#ifdef CONFIG_CGROUP_SCHED 6400 list_add(&root_task_group.list, &task_groups); 6401 INIT_LIST_HEAD(&root_task_group.children); 6402 INIT_LIST_HEAD(&root_task_group.siblings); 6403 autogroup_init(&init_task); 6404 6405#endif /* CONFIG_CGROUP_SCHED */ 6406 6407 for_each_possible_cpu(i) { 6408 struct rq *rq; 6409 6410 rq = cpu_rq(i); 6411 raw_spin_lock_init(&rq->lock); 6412 rq->nr_running = 0; 6413 rq->calc_load_active = 0; 6414 rq->calc_load_update = jiffies + LOAD_FREQ; 6415 init_cfs_rq(&rq->cfs); 6416 init_rt_rq(&rq->rt, rq); 6417#ifdef CONFIG_FAIR_GROUP_SCHED 6418 root_task_group.shares = ROOT_TASK_GROUP_LOAD; 6419 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); 6420 /* 6421 * How much cpu bandwidth does root_task_group get? 6422 * 6423 * In case of task-groups formed thr' the cgroup filesystem, it 6424 * gets 100% of the cpu resources in the system. This overall 6425 * system cpu resource is divided among the tasks of 6426 * root_task_group and its child task-groups in a fair manner, 6427 * based on each entity's (task or task-group's) weight 6428 * (se->load.weight). 6429 * 6430 * In other words, if root_task_group has 10 tasks of weight 6431 * 1024) and two child groups A0 and A1 (of weight 1024 each), 6432 * then A0's share of the cpu resource is: 6433 * 6434 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% 6435 * 6436 * We achieve this by letting root_task_group's tasks sit 6437 * directly in rq->cfs (i.e root_task_group->se[] = NULL). 6438 */ 6439 init_cfs_bandwidth(&root_task_group.cfs_bandwidth); 6440 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); 6441#endif /* CONFIG_FAIR_GROUP_SCHED */ 6442 6443 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; 6444#ifdef CONFIG_RT_GROUP_SCHED 6445 INIT_LIST_HEAD(&rq->leaf_rt_rq_list); 6446 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); 6447#endif 6448 6449 for (j = 0; j < CPU_LOAD_IDX_MAX; j++) 6450 rq->cpu_load[j] = 0; 6451 6452 rq->last_load_update_tick = jiffies; 6453 6454#ifdef CONFIG_SMP 6455 rq->sd = NULL; 6456 rq->rd = NULL; 6457 rq->cpu_power = SCHED_POWER_SCALE; 6458 rq->post_schedule = 0; 6459 rq->active_balance = 0; 6460 rq->next_balance = jiffies; 6461 rq->push_cpu = 0; 6462 rq->cpu = i; 6463 rq->online = 0; 6464 rq->idle_stamp = 0; 6465 rq->avg_idle = 2*sysctl_sched_migration_cost; 6466 6467 INIT_LIST_HEAD(&rq->cfs_tasks); 6468 6469 rq_attach_root(rq, &def_root_domain); 6470#ifdef CONFIG_NO_HZ_COMMON 6471 rq->nohz_flags = 0; 6472#endif 6473#ifdef CONFIG_NO_HZ_FULL 6474 rq->last_sched_tick = 0; 6475#endif 6476#endif 6477 init_rq_hrtick(rq); 6478 atomic_set(&rq->nr_iowait, 0); 6479 } 6480 6481 set_load_weight(&init_task); 6482 6483#ifdef CONFIG_PREEMPT_NOTIFIERS 6484 INIT_HLIST_HEAD(&init_task.preempt_notifiers); 6485#endif 6486 6487#ifdef CONFIG_RT_MUTEXES 6488 plist_head_init(&init_task.pi_waiters); 6489#endif 6490 6491 /* 6492 * The boot idle thread does lazy MMU switching as well: 6493 */ 6494 atomic_inc(&init_mm.mm_count); 6495 enter_lazy_tlb(&init_mm, current); 6496 6497 /* 6498 * Make us the idle thread. Technically, schedule() should not be 6499 * called from this thread, however somewhere below it might be, 6500 * but because we are the idle thread, we just pick up running again 6501 * when this runqueue becomes "idle". 6502 */ 6503 init_idle(current, smp_processor_id()); 6504 6505 calc_load_update = jiffies + LOAD_FREQ; 6506 6507 /* 6508 * During early bootup we pretend to be a normal task: 6509 */ 6510 current->sched_class = &fair_sched_class; 6511 6512#ifdef CONFIG_SMP 6513 zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT); 6514 /* May be allocated at isolcpus cmdline parse time */ 6515 if (cpu_isolated_map == NULL) 6516 zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT); 6517 idle_thread_set_boot_cpu(); 6518#endif 6519 init_sched_fair_class(); 6520 6521 scheduler_running = 1; 6522} 6523 6524#ifdef CONFIG_DEBUG_ATOMIC_SLEEP 6525static inline int preempt_count_equals(int preempt_offset) 6526{ 6527 int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth(); 6528 6529 return (nested == preempt_offset); 6530} 6531 6532void __might_sleep(const char *file, int line, int preempt_offset) 6533{ 6534 static unsigned long prev_jiffy; /* ratelimiting */ 6535 6536 rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */ 6537 if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) || 6538 system_state != SYSTEM_RUNNING || oops_in_progress) 6539 return; 6540 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 6541 return; 6542 prev_jiffy = jiffies; 6543 6544 printk(KERN_ERR 6545 "BUG: sleeping function called from invalid context at %s:%d\n", 6546 file, line); 6547 printk(KERN_ERR 6548 "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", 6549 in_atomic(), irqs_disabled(), 6550 current->pid, current->comm); 6551 6552 debug_show_held_locks(current); 6553 if (irqs_disabled()) 6554 print_irqtrace_events(current); 6555 dump_stack(); 6556} 6557EXPORT_SYMBOL(__might_sleep); 6558#endif 6559 6560#ifdef CONFIG_MAGIC_SYSRQ 6561static void normalize_task(struct rq *rq, struct task_struct *p) 6562{ 6563 const struct sched_class *prev_class = p->sched_class; 6564 int old_prio = p->prio; 6565 int on_rq; 6566 6567 on_rq = p->on_rq; 6568 if (on_rq) 6569 dequeue_task(rq, p, 0); 6570 __setscheduler(rq, p, SCHED_NORMAL, 0); 6571 if (on_rq) { 6572 enqueue_task(rq, p, 0); 6573 resched_task(rq->curr); 6574 } 6575 6576 check_class_changed(rq, p, prev_class, old_prio); 6577} 6578 6579void normalize_rt_tasks(void) 6580{ 6581 struct task_struct *g, *p; 6582 unsigned long flags; 6583 struct rq *rq; 6584 6585 read_lock_irqsave(&tasklist_lock, flags); 6586 do_each_thread(g, p) { 6587 /* 6588 * Only normalize user tasks: 6589 */ 6590 if (!p->mm) 6591 continue; 6592 6593 p->se.exec_start = 0; 6594#ifdef CONFIG_SCHEDSTATS 6595 p->se.statistics.wait_start = 0; 6596 p->se.statistics.sleep_start = 0; 6597 p->se.statistics.block_start = 0; 6598#endif 6599 6600 if (!rt_task(p)) { 6601 /* 6602 * Renice negative nice level userspace 6603 * tasks back to 0: 6604 */ 6605 if (TASK_NICE(p) < 0 && p->mm) 6606 set_user_nice(p, 0); 6607 continue; 6608 } 6609 6610 raw_spin_lock(&p->pi_lock); 6611 rq = __task_rq_lock(p); 6612 6613 normalize_task(rq, p); 6614 6615 __task_rq_unlock(rq); 6616 raw_spin_unlock(&p->pi_lock); 6617 } while_each_thread(g, p); 6618 6619 read_unlock_irqrestore(&tasklist_lock, flags); 6620} 6621 6622#endif /* CONFIG_MAGIC_SYSRQ */ 6623 6624#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) 6625/* 6626 * These functions are only useful for the IA64 MCA handling, or kdb. 6627 * 6628 * They can only be called when the whole system has been 6629 * stopped - every CPU needs to be quiescent, and no scheduling 6630 * activity can take place. Using them for anything else would 6631 * be a serious bug, and as a result, they aren't even visible 6632 * under any other configuration. 6633 */ 6634 6635/** 6636 * curr_task - return the current task for a given cpu. 6637 * @cpu: the processor in question. 6638 * 6639 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 6640 */ 6641struct task_struct *curr_task(int cpu) 6642{ 6643 return cpu_curr(cpu); 6644} 6645 6646#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ 6647 6648#ifdef CONFIG_IA64 6649/** 6650 * set_curr_task - set the current task for a given cpu. 6651 * @cpu: the processor in question. 6652 * @p: the task pointer to set. 6653 * 6654 * Description: This function must only be used when non-maskable interrupts 6655 * are serviced on a separate stack. It allows the architecture to switch the 6656 * notion of the current task on a cpu in a non-blocking manner. This function 6657 * must be called with all CPU's synchronized, and interrupts disabled, the 6658 * and caller must save the original value of the current task (see 6659 * curr_task() above) and restore that value before reenabling interrupts and 6660 * re-starting the system. 6661 * 6662 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 6663 */ 6664void set_curr_task(int cpu, struct task_struct *p) 6665{ 6666 cpu_curr(cpu) = p; 6667} 6668 6669#endif 6670 6671#ifdef CONFIG_CGROUP_SCHED 6672/* task_group_lock serializes the addition/removal of task groups */ 6673static DEFINE_SPINLOCK(task_group_lock); 6674 6675static void free_sched_group(struct task_group *tg) 6676{ 6677 free_fair_sched_group(tg); 6678 free_rt_sched_group(tg); 6679 autogroup_free(tg); 6680 kfree(tg); 6681} 6682 6683/* allocate runqueue etc for a new task group */ 6684struct task_group *sched_create_group(struct task_group *parent) 6685{ 6686 struct task_group *tg; 6687 6688 tg = kzalloc(sizeof(*tg), GFP_KERNEL); 6689 if (!tg) 6690 return ERR_PTR(-ENOMEM); 6691 6692 if (!alloc_fair_sched_group(tg, parent)) 6693 goto err; 6694 6695 if (!alloc_rt_sched_group(tg, parent)) 6696 goto err; 6697 6698 return tg; 6699 6700err: 6701 free_sched_group(tg); 6702 return ERR_PTR(-ENOMEM); 6703} 6704 6705void sched_online_group(struct task_group *tg, struct task_group *parent) 6706{ 6707 unsigned long flags; 6708 6709 spin_lock_irqsave(&task_group_lock, flags); 6710 list_add_rcu(&tg->list, &task_groups); 6711 6712 WARN_ON(!parent); /* root should already exist */ 6713 6714 tg->parent = parent; 6715 INIT_LIST_HEAD(&tg->children); 6716 list_add_rcu(&tg->siblings, &parent->children); 6717 spin_unlock_irqrestore(&task_group_lock, flags); 6718} 6719 6720/* rcu callback to free various structures associated with a task group */ 6721static void free_sched_group_rcu(struct rcu_head *rhp) 6722{ 6723 /* now it should be safe to free those cfs_rqs */ 6724 free_sched_group(container_of(rhp, struct task_group, rcu)); 6725} 6726 6727/* Destroy runqueue etc associated with a task group */ 6728void sched_destroy_group(struct task_group *tg) 6729{ 6730 /* wait for possible concurrent references to cfs_rqs complete */ 6731 call_rcu(&tg->rcu, free_sched_group_rcu); 6732} 6733 6734void sched_offline_group(struct task_group *tg) 6735{ 6736 unsigned long flags; 6737 int i; 6738 6739 /* end participation in shares distribution */ 6740 for_each_possible_cpu(i) 6741 unregister_fair_sched_group(tg, i); 6742 6743 spin_lock_irqsave(&task_group_lock, flags); 6744 list_del_rcu(&tg->list); 6745 list_del_rcu(&tg->siblings); 6746 spin_unlock_irqrestore(&task_group_lock, flags); 6747} 6748 6749/* change task's runqueue when it moves between groups. 6750 * The caller of this function should have put the task in its new group 6751 * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to 6752 * reflect its new group. 6753 */ 6754void sched_move_task(struct task_struct *tsk) 6755{ 6756 struct task_group *tg; 6757 int on_rq, running; 6758 unsigned long flags; 6759 struct rq *rq; 6760 6761 rq = task_rq_lock(tsk, &flags); 6762 6763 running = task_current(rq, tsk); 6764 on_rq = tsk->on_rq; 6765 6766 if (on_rq) 6767 dequeue_task(rq, tsk, 0); 6768 if (unlikely(running)) 6769 tsk->sched_class->put_prev_task(rq, tsk); 6770 6771 tg = container_of(task_subsys_state_check(tsk, cpu_cgroup_subsys_id, 6772 lockdep_is_held(&tsk->sighand->siglock)), 6773 struct task_group, css); 6774 tg = autogroup_task_group(tsk, tg); 6775 tsk->sched_task_group = tg; 6776 6777#ifdef CONFIG_FAIR_GROUP_SCHED 6778 if (tsk->sched_class->task_move_group) 6779 tsk->sched_class->task_move_group(tsk, on_rq); 6780 else 6781#endif 6782 set_task_rq(tsk, task_cpu(tsk)); 6783 6784 if (unlikely(running)) 6785 tsk->sched_class->set_curr_task(rq); 6786 if (on_rq) 6787 enqueue_task(rq, tsk, 0); 6788 6789 task_rq_unlock(rq, tsk, &flags); 6790} 6791#endif /* CONFIG_CGROUP_SCHED */ 6792 6793#if defined(CONFIG_RT_GROUP_SCHED) || defined(CONFIG_CFS_BANDWIDTH) 6794static unsigned long to_ratio(u64 period, u64 runtime) 6795{ 6796 if (runtime == RUNTIME_INF) 6797 return 1ULL << 20; 6798 6799 return div64_u64(runtime << 20, period); 6800} 6801#endif 6802 6803#ifdef CONFIG_RT_GROUP_SCHED 6804/* 6805 * Ensure that the real time constraints are schedulable. 6806 */ 6807static DEFINE_MUTEX(rt_constraints_mutex); 6808 6809/* Must be called with tasklist_lock held */ 6810static inline int tg_has_rt_tasks(struct task_group *tg) 6811{ 6812 struct task_struct *g, *p; 6813 6814 do_each_thread(g, p) { 6815 if (rt_task(p) && task_rq(p)->rt.tg == tg) 6816 return 1; 6817 } while_each_thread(g, p); 6818 6819 return 0; 6820} 6821 6822struct rt_schedulable_data { 6823 struct task_group *tg; 6824 u64 rt_period; 6825 u64 rt_runtime; 6826}; 6827 6828static int tg_rt_schedulable(struct task_group *tg, void *data) 6829{ 6830 struct rt_schedulable_data *d = data; 6831 struct task_group *child; 6832 unsigned long total, sum = 0; 6833 u64 period, runtime; 6834 6835 period = ktime_to_ns(tg->rt_bandwidth.rt_period); 6836 runtime = tg->rt_bandwidth.rt_runtime; 6837 6838 if (tg == d->tg) { 6839 period = d->rt_period; 6840 runtime = d->rt_runtime; 6841 } 6842 6843 /* 6844 * Cannot have more runtime than the period. 6845 */ 6846 if (runtime > period && runtime != RUNTIME_INF) 6847 return -EINVAL; 6848 6849 /* 6850 * Ensure we don't starve existing RT tasks. 6851 */ 6852 if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg)) 6853 return -EBUSY; 6854 6855 total = to_ratio(period, runtime); 6856 6857 /* 6858 * Nobody can have more than the global setting allows. 6859 */ 6860 if (total > to_ratio(global_rt_period(), global_rt_runtime())) 6861 return -EINVAL; 6862 6863 /* 6864 * The sum of our children's runtime should not exceed our own. 6865 */ 6866 list_for_each_entry_rcu(child, &tg->children, siblings) { 6867 period = ktime_to_ns(child->rt_bandwidth.rt_period); 6868 runtime = child->rt_bandwidth.rt_runtime; 6869 6870 if (child == d->tg) { 6871 period = d->rt_period; 6872 runtime = d->rt_runtime; 6873 } 6874 6875 sum += to_ratio(period, runtime); 6876 } 6877 6878 if (sum > total) 6879 return -EINVAL; 6880 6881 return 0; 6882} 6883 6884static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime) 6885{ 6886 int ret; 6887 6888 struct rt_schedulable_data data = { 6889 .tg = tg, 6890 .rt_period = period, 6891 .rt_runtime = runtime, 6892 }; 6893 6894 rcu_read_lock(); 6895 ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data); 6896 rcu_read_unlock(); 6897 6898 return ret; 6899} 6900 6901static int tg_set_rt_bandwidth(struct task_group *tg, 6902 u64 rt_period, u64 rt_runtime) 6903{ 6904 int i, err = 0; 6905 6906 mutex_lock(&rt_constraints_mutex); 6907 read_lock(&tasklist_lock); 6908 err = __rt_schedulable(tg, rt_period, rt_runtime); 6909 if (err) 6910 goto unlock; 6911 6912 raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); 6913 tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); 6914 tg->rt_bandwidth.rt_runtime = rt_runtime; 6915 6916 for_each_possible_cpu(i) { 6917 struct rt_rq *rt_rq = tg->rt_rq[i]; 6918 6919 raw_spin_lock(&rt_rq->rt_runtime_lock); 6920 rt_rq->rt_runtime = rt_runtime; 6921 raw_spin_unlock(&rt_rq->rt_runtime_lock); 6922 } 6923 raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); 6924unlock: 6925 read_unlock(&tasklist_lock); 6926 mutex_unlock(&rt_constraints_mutex); 6927 6928 return err; 6929} 6930 6931static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us) 6932{ 6933 u64 rt_runtime, rt_period; 6934 6935 rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period); 6936 rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC; 6937 if (rt_runtime_us < 0) 6938 rt_runtime = RUNTIME_INF; 6939 6940 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); 6941} 6942 6943static long sched_group_rt_runtime(struct task_group *tg) 6944{ 6945 u64 rt_runtime_us; 6946 6947 if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF) 6948 return -1; 6949 6950 rt_runtime_us = tg->rt_bandwidth.rt_runtime; 6951 do_div(rt_runtime_us, NSEC_PER_USEC); 6952 return rt_runtime_us; 6953} 6954 6955static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us) 6956{ 6957 u64 rt_runtime, rt_period; 6958 6959 rt_period = (u64)rt_period_us * NSEC_PER_USEC; 6960 rt_runtime = tg->rt_bandwidth.rt_runtime; 6961 6962 if (rt_period == 0) 6963 return -EINVAL; 6964 6965 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); 6966} 6967 6968static long sched_group_rt_period(struct task_group *tg) 6969{ 6970 u64 rt_period_us; 6971 6972 rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period); 6973 do_div(rt_period_us, NSEC_PER_USEC); 6974 return rt_period_us; 6975} 6976 6977static int sched_rt_global_constraints(void) 6978{ 6979 u64 runtime, period; 6980 int ret = 0; 6981 6982 if (sysctl_sched_rt_period <= 0) 6983 return -EINVAL; 6984 6985 runtime = global_rt_runtime(); 6986 period = global_rt_period(); 6987 6988 /* 6989 * Sanity check on the sysctl variables. 6990 */ 6991 if (runtime > period && runtime != RUNTIME_INF) 6992 return -EINVAL; 6993 6994 mutex_lock(&rt_constraints_mutex); 6995 read_lock(&tasklist_lock); 6996 ret = __rt_schedulable(NULL, 0, 0); 6997 read_unlock(&tasklist_lock); 6998 mutex_unlock(&rt_constraints_mutex); 6999 7000 return ret; 7001} 7002 7003static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) 7004{ 7005 /* Don't accept realtime tasks when there is no way for them to run */ 7006 if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0) 7007 return 0; 7008 7009 return 1; 7010} 7011 7012#else /* !CONFIG_RT_GROUP_SCHED */ 7013static int sched_rt_global_constraints(void) 7014{ 7015 unsigned long flags; 7016 int i; 7017 7018 if (sysctl_sched_rt_period <= 0) 7019 return -EINVAL; 7020 7021 /* 7022 * There's always some RT tasks in the root group 7023 * -- migration, kstopmachine etc.. 7024 */ 7025 if (sysctl_sched_rt_runtime == 0) 7026 return -EBUSY; 7027 7028 raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); 7029 for_each_possible_cpu(i) { 7030 struct rt_rq *rt_rq = &cpu_rq(i)->rt; 7031 7032 raw_spin_lock(&rt_rq->rt_runtime_lock); 7033 rt_rq->rt_runtime = global_rt_runtime(); 7034 raw_spin_unlock(&rt_rq->rt_runtime_lock); 7035 } 7036 raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); 7037 7038 return 0; 7039} 7040#endif /* CONFIG_RT_GROUP_SCHED */ 7041 7042int sched_rr_handler(struct ctl_table *table, int write, 7043 void __user *buffer, size_t *lenp, 7044 loff_t *ppos) 7045{ 7046 int ret; 7047 static DEFINE_MUTEX(mutex); 7048 7049 mutex_lock(&mutex); 7050 ret = proc_dointvec(table, write, buffer, lenp, ppos); 7051 /* make sure that internally we keep jiffies */ 7052 /* also, writing zero resets timeslice to default */ 7053 if (!ret && write) { 7054 sched_rr_timeslice = sched_rr_timeslice <= 0 ? 7055 RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice); 7056 } 7057 mutex_unlock(&mutex); 7058 return ret; 7059} 7060 7061int sched_rt_handler(struct ctl_table *table, int write, 7062 void __user *buffer, size_t *lenp, 7063 loff_t *ppos) 7064{ 7065 int ret; 7066 int old_period, old_runtime; 7067 static DEFINE_MUTEX(mutex); 7068 7069 mutex_lock(&mutex); 7070 old_period = sysctl_sched_rt_period; 7071 old_runtime = sysctl_sched_rt_runtime; 7072 7073 ret = proc_dointvec(table, write, buffer, lenp, ppos); 7074 7075 if (!ret && write) { 7076 ret = sched_rt_global_constraints(); 7077 if (ret) { 7078 sysctl_sched_rt_period = old_period; 7079 sysctl_sched_rt_runtime = old_runtime; 7080 } else { 7081 def_rt_bandwidth.rt_runtime = global_rt_runtime(); 7082 def_rt_bandwidth.rt_period = 7083 ns_to_ktime(global_rt_period()); 7084 } 7085 } 7086 mutex_unlock(&mutex); 7087 7088 return ret; 7089} 7090 7091#ifdef CONFIG_CGROUP_SCHED 7092 7093/* return corresponding task_group object of a cgroup */ 7094static inline struct task_group *cgroup_tg(struct cgroup *cgrp) 7095{ 7096 return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id), 7097 struct task_group, css); 7098} 7099 7100static struct cgroup_subsys_state *cpu_cgroup_css_alloc(struct cgroup *cgrp) 7101{ 7102 struct task_group *tg, *parent; 7103 7104 if (!cgrp->parent) { 7105 /* This is early initialization for the top cgroup */ 7106 return &root_task_group.css; 7107 } 7108 7109 parent = cgroup_tg(cgrp->parent); 7110 tg = sched_create_group(parent); 7111 if (IS_ERR(tg)) 7112 return ERR_PTR(-ENOMEM); 7113 7114 return &tg->css; 7115} 7116 7117static int cpu_cgroup_css_online(struct cgroup *cgrp) 7118{ 7119 struct task_group *tg = cgroup_tg(cgrp); 7120 struct task_group *parent; 7121 7122 if (!cgrp->parent) 7123 return 0; 7124 7125 parent = cgroup_tg(cgrp->parent); 7126 sched_online_group(tg, parent); 7127 return 0; 7128} 7129 7130static void cpu_cgroup_css_free(struct cgroup *cgrp) 7131{ 7132 struct task_group *tg = cgroup_tg(cgrp); 7133 7134 sched_destroy_group(tg); 7135} 7136 7137static void cpu_cgroup_css_offline(struct cgroup *cgrp) 7138{ 7139 struct task_group *tg = cgroup_tg(cgrp); 7140 7141 sched_offline_group(tg); 7142} 7143 7144static int cpu_cgroup_can_attach(struct cgroup *cgrp, 7145 struct cgroup_taskset *tset) 7146{ 7147 struct task_struct *task; 7148 7149 cgroup_taskset_for_each(task, cgrp, tset) { 7150#ifdef CONFIG_RT_GROUP_SCHED 7151 if (!sched_rt_can_attach(cgroup_tg(cgrp), task)) 7152 return -EINVAL; 7153#else 7154 /* We don't support RT-tasks being in separate groups */ 7155 if (task->sched_class != &fair_sched_class) 7156 return -EINVAL; 7157#endif 7158 } 7159 return 0; 7160} 7161 7162static void cpu_cgroup_attach(struct cgroup *cgrp, 7163 struct cgroup_taskset *tset) 7164{ 7165 struct task_struct *task; 7166 7167 cgroup_taskset_for_each(task, cgrp, tset) 7168 sched_move_task(task); 7169} 7170 7171static void 7172cpu_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp, 7173 struct task_struct *task) 7174{ 7175 /* 7176 * cgroup_exit() is called in the copy_process() failure path. 7177 * Ignore this case since the task hasn't ran yet, this avoids 7178 * trying to poke a half freed task state from generic code. 7179 */ 7180 if (!(task->flags & PF_EXITING)) 7181 return; 7182 7183 sched_move_task(task); 7184} 7185 7186#ifdef CONFIG_FAIR_GROUP_SCHED 7187static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype, 7188 u64 shareval) 7189{ 7190 return sched_group_set_shares(cgroup_tg(cgrp), scale_load(shareval)); 7191} 7192 7193static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft) 7194{ 7195 struct task_group *tg = cgroup_tg(cgrp); 7196 7197 return (u64) scale_load_down(tg->shares); 7198} 7199 7200#ifdef CONFIG_CFS_BANDWIDTH 7201static DEFINE_MUTEX(cfs_constraints_mutex); 7202 7203const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ 7204const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ 7205 7206static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); 7207 7208static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) 7209{ 7210 int i, ret = 0, runtime_enabled, runtime_was_enabled; 7211 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 7212 7213 if (tg == &root_task_group) 7214 return -EINVAL; 7215 7216 /* 7217 * Ensure we have at some amount of bandwidth every period. This is 7218 * to prevent reaching a state of large arrears when throttled via 7219 * entity_tick() resulting in prolonged exit starvation. 7220 */ 7221 if (quota < min_cfs_quota_period || period < min_cfs_quota_period) 7222 return -EINVAL; 7223 7224 /* 7225 * Likewise, bound things on the otherside by preventing insane quota 7226 * periods. This also allows us to normalize in computing quota 7227 * feasibility. 7228 */ 7229 if (period > max_cfs_quota_period) 7230 return -EINVAL; 7231 7232 mutex_lock(&cfs_constraints_mutex); 7233 ret = __cfs_schedulable(tg, period, quota); 7234 if (ret) 7235 goto out_unlock; 7236 7237 runtime_enabled = quota != RUNTIME_INF; 7238 runtime_was_enabled = cfs_b->quota != RUNTIME_INF; 7239 account_cfs_bandwidth_used(runtime_enabled, runtime_was_enabled); 7240 raw_spin_lock_irq(&cfs_b->lock); 7241 cfs_b->period = ns_to_ktime(period); 7242 cfs_b->quota = quota; 7243 7244 __refill_cfs_bandwidth_runtime(cfs_b); 7245 /* restart the period timer (if active) to handle new period expiry */ 7246 if (runtime_enabled && cfs_b->timer_active) { 7247 /* force a reprogram */ 7248 cfs_b->timer_active = 0; 7249 __start_cfs_bandwidth(cfs_b); 7250 } 7251 raw_spin_unlock_irq(&cfs_b->lock); 7252 7253 for_each_possible_cpu(i) { 7254 struct cfs_rq *cfs_rq = tg->cfs_rq[i]; 7255 struct rq *rq = cfs_rq->rq; 7256 7257 raw_spin_lock_irq(&rq->lock); 7258 cfs_rq->runtime_enabled = runtime_enabled; 7259 cfs_rq->runtime_remaining = 0; 7260 7261 if (cfs_rq->throttled) 7262 unthrottle_cfs_rq(cfs_rq); 7263 raw_spin_unlock_irq(&rq->lock); 7264 } 7265out_unlock: 7266 mutex_unlock(&cfs_constraints_mutex); 7267 7268 return ret; 7269} 7270 7271int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) 7272{ 7273 u64 quota, period; 7274 7275 period = ktime_to_ns(tg->cfs_bandwidth.period); 7276 if (cfs_quota_us < 0) 7277 quota = RUNTIME_INF; 7278 else 7279 quota = (u64)cfs_quota_us * NSEC_PER_USEC; 7280 7281 return tg_set_cfs_bandwidth(tg, period, quota); 7282} 7283 7284long tg_get_cfs_quota(struct task_group *tg) 7285{ 7286 u64 quota_us; 7287 7288 if (tg->cfs_bandwidth.quota == RUNTIME_INF) 7289 return -1; 7290 7291 quota_us = tg->cfs_bandwidth.quota; 7292 do_div(quota_us, NSEC_PER_USEC); 7293 7294 return quota_us; 7295} 7296 7297int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) 7298{ 7299 u64 quota, period; 7300 7301 period = (u64)cfs_period_us * NSEC_PER_USEC; 7302 quota = tg->cfs_bandwidth.quota; 7303 7304 return tg_set_cfs_bandwidth(tg, period, quota); 7305} 7306 7307long tg_get_cfs_period(struct task_group *tg) 7308{ 7309 u64 cfs_period_us; 7310 7311 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); 7312 do_div(cfs_period_us, NSEC_PER_USEC); 7313 7314 return cfs_period_us; 7315} 7316 7317static s64 cpu_cfs_quota_read_s64(struct cgroup *cgrp, struct cftype *cft) 7318{ 7319 return tg_get_cfs_quota(cgroup_tg(cgrp)); 7320} 7321 7322static int cpu_cfs_quota_write_s64(struct cgroup *cgrp, struct cftype *cftype, 7323 s64 cfs_quota_us) 7324{ 7325 return tg_set_cfs_quota(cgroup_tg(cgrp), cfs_quota_us); 7326} 7327 7328static u64 cpu_cfs_period_read_u64(struct cgroup *cgrp, struct cftype *cft) 7329{ 7330 return tg_get_cfs_period(cgroup_tg(cgrp)); 7331} 7332 7333static int cpu_cfs_period_write_u64(struct cgroup *cgrp, struct cftype *cftype, 7334 u64 cfs_period_us) 7335{ 7336 return tg_set_cfs_period(cgroup_tg(cgrp), cfs_period_us); 7337} 7338 7339struct cfs_schedulable_data { 7340 struct task_group *tg; 7341 u64 period, quota; 7342}; 7343 7344/* 7345 * normalize group quota/period to be quota/max_period 7346 * note: units are usecs 7347 */ 7348static u64 normalize_cfs_quota(struct task_group *tg, 7349 struct cfs_schedulable_data *d) 7350{ 7351 u64 quota, period; 7352 7353 if (tg == d->tg) { 7354 period = d->period; 7355 quota = d->quota; 7356 } else { 7357 period = tg_get_cfs_period(tg); 7358 quota = tg_get_cfs_quota(tg); 7359 } 7360 7361 /* note: these should typically be equivalent */ 7362 if (quota == RUNTIME_INF || quota == -1) 7363 return RUNTIME_INF; 7364 7365 return to_ratio(period, quota); 7366} 7367 7368static int tg_cfs_schedulable_down(struct task_group *tg, void *data) 7369{ 7370 struct cfs_schedulable_data *d = data; 7371 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 7372 s64 quota = 0, parent_quota = -1; 7373 7374 if (!tg->parent) { 7375 quota = RUNTIME_INF; 7376 } else { 7377 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; 7378 7379 quota = normalize_cfs_quota(tg, d); 7380 parent_quota = parent_b->hierarchal_quota; 7381 7382 /* 7383 * ensure max(child_quota) <= parent_quota, inherit when no 7384 * limit is set 7385 */ 7386 if (quota == RUNTIME_INF) 7387 quota = parent_quota; 7388 else if (parent_quota != RUNTIME_INF && quota > parent_quota) 7389 return -EINVAL; 7390 } 7391 cfs_b->hierarchal_quota = quota; 7392 7393 return 0; 7394} 7395 7396static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) 7397{ 7398 int ret; 7399 struct cfs_schedulable_data data = { 7400 .tg = tg, 7401 .period = period, 7402 .quota = quota, 7403 }; 7404 7405 if (quota != RUNTIME_INF) { 7406 do_div(data.period, NSEC_PER_USEC); 7407 do_div(data.quota, NSEC_PER_USEC); 7408 } 7409 7410 rcu_read_lock(); 7411 ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); 7412 rcu_read_unlock(); 7413 7414 return ret; 7415} 7416 7417static int cpu_stats_show(struct cgroup *cgrp, struct cftype *cft, 7418 struct cgroup_map_cb *cb) 7419{ 7420 struct task_group *tg = cgroup_tg(cgrp); 7421 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 7422 7423 cb->fill(cb, "nr_periods", cfs_b->nr_periods); 7424 cb->fill(cb, "nr_throttled", cfs_b->nr_throttled); 7425 cb->fill(cb, "throttled_time", cfs_b->throttled_time); 7426 7427 return 0; 7428} 7429#endif /* CONFIG_CFS_BANDWIDTH */ 7430#endif /* CONFIG_FAIR_GROUP_SCHED */ 7431 7432#ifdef CONFIG_RT_GROUP_SCHED 7433static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft, 7434 s64 val) 7435{ 7436 return sched_group_set_rt_runtime(cgroup_tg(cgrp), val); 7437} 7438 7439static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft) 7440{ 7441 return sched_group_rt_runtime(cgroup_tg(cgrp)); 7442} 7443 7444static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype, 7445 u64 rt_period_us) 7446{ 7447 return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us); 7448} 7449 7450static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft) 7451{ 7452 return sched_group_rt_period(cgroup_tg(cgrp)); 7453} 7454#endif /* CONFIG_RT_GROUP_SCHED */ 7455 7456static struct cftype cpu_files[] = { 7457#ifdef CONFIG_FAIR_GROUP_SCHED 7458 { 7459 .name = "shares", 7460 .read_u64 = cpu_shares_read_u64, 7461 .write_u64 = cpu_shares_write_u64, 7462 }, 7463#endif 7464#ifdef CONFIG_CFS_BANDWIDTH 7465 { 7466 .name = "cfs_quota_us", 7467 .read_s64 = cpu_cfs_quota_read_s64, 7468 .write_s64 = cpu_cfs_quota_write_s64, 7469 }, 7470 { 7471 .name = "cfs_period_us", 7472 .read_u64 = cpu_cfs_period_read_u64, 7473 .write_u64 = cpu_cfs_period_write_u64, 7474 }, 7475 { 7476 .name = "stat", 7477 .read_map = cpu_stats_show, 7478 }, 7479#endif 7480#ifdef CONFIG_RT_GROUP_SCHED 7481 { 7482 .name = "rt_runtime_us", 7483 .read_s64 = cpu_rt_runtime_read, 7484 .write_s64 = cpu_rt_runtime_write, 7485 }, 7486 { 7487 .name = "rt_period_us", 7488 .read_u64 = cpu_rt_period_read_uint, 7489 .write_u64 = cpu_rt_period_write_uint, 7490 }, 7491#endif 7492 { } /* terminate */ 7493}; 7494 7495struct cgroup_subsys cpu_cgroup_subsys = { 7496 .name = "cpu", 7497 .css_alloc = cpu_cgroup_css_alloc, 7498 .css_free = cpu_cgroup_css_free, 7499 .css_online = cpu_cgroup_css_online, 7500 .css_offline = cpu_cgroup_css_offline, 7501 .can_attach = cpu_cgroup_can_attach, 7502 .attach = cpu_cgroup_attach, 7503 .exit = cpu_cgroup_exit, 7504 .subsys_id = cpu_cgroup_subsys_id, 7505 .base_cftypes = cpu_files, 7506 .early_init = 1, 7507}; 7508 7509#endif /* CONFIG_CGROUP_SCHED */ 7510 7511void dump_cpu_task(int cpu) 7512{ 7513 pr_info("Task dump for CPU %d:\n", cpu); 7514 sched_show_task(cpu_curr(cpu)); 7515} 7516