tick-broadcast.c revision 07bd1172902e782f288e4d44b1fde7dec0f08b6f
1/* 2 * linux/kernel/time/tick-broadcast.c 3 * 4 * This file contains functions which emulate a local clock-event 5 * device via a broadcast event source. 6 * 7 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> 8 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar 9 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner 10 * 11 * This code is licenced under the GPL version 2. For details see 12 * kernel-base/COPYING. 13 */ 14#include <linux/cpu.h> 15#include <linux/err.h> 16#include <linux/hrtimer.h> 17#include <linux/interrupt.h> 18#include <linux/percpu.h> 19#include <linux/profile.h> 20#include <linux/sched.h> 21#include <linux/smp.h> 22#include <linux/module.h> 23 24#include "tick-internal.h" 25 26/* 27 * Broadcast support for broken x86 hardware, where the local apic 28 * timer stops in C3 state. 29 */ 30 31static struct tick_device tick_broadcast_device; 32static cpumask_var_t tick_broadcast_mask; 33static cpumask_var_t tick_broadcast_on; 34static cpumask_var_t tmpmask; 35static DEFINE_RAW_SPINLOCK(tick_broadcast_lock); 36static int tick_broadcast_force; 37 38#ifdef CONFIG_TICK_ONESHOT 39static void tick_broadcast_clear_oneshot(int cpu); 40#else 41static inline void tick_broadcast_clear_oneshot(int cpu) { } 42#endif 43 44/* 45 * Debugging: see timer_list.c 46 */ 47struct tick_device *tick_get_broadcast_device(void) 48{ 49 return &tick_broadcast_device; 50} 51 52struct cpumask *tick_get_broadcast_mask(void) 53{ 54 return tick_broadcast_mask; 55} 56 57/* 58 * Start the device in periodic mode 59 */ 60static void tick_broadcast_start_periodic(struct clock_event_device *bc) 61{ 62 if (bc) 63 tick_setup_periodic(bc, 1); 64} 65 66/* 67 * Check, if the device can be utilized as broadcast device: 68 */ 69static bool tick_check_broadcast_device(struct clock_event_device *curdev, 70 struct clock_event_device *newdev) 71{ 72 if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) || 73 (newdev->features & CLOCK_EVT_FEAT_C3STOP)) 74 return false; 75 76 if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT && 77 !(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) 78 return false; 79 80 return !curdev || newdev->rating > curdev->rating; 81} 82 83/* 84 * Conditionally install/replace broadcast device 85 */ 86void tick_install_broadcast_device(struct clock_event_device *dev) 87{ 88 struct clock_event_device *cur = tick_broadcast_device.evtdev; 89 90 if (!tick_check_broadcast_device(cur, dev)) 91 return; 92 93 if (!try_module_get(dev->owner)) 94 return; 95 96 clockevents_exchange_device(cur, dev); 97 if (cur) 98 cur->event_handler = clockevents_handle_noop; 99 tick_broadcast_device.evtdev = dev; 100 if (!cpumask_empty(tick_broadcast_mask)) 101 tick_broadcast_start_periodic(dev); 102 /* 103 * Inform all cpus about this. We might be in a situation 104 * where we did not switch to oneshot mode because the per cpu 105 * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack 106 * of a oneshot capable broadcast device. Without that 107 * notification the systems stays stuck in periodic mode 108 * forever. 109 */ 110 if (dev->features & CLOCK_EVT_FEAT_ONESHOT) 111 tick_clock_notify(); 112} 113 114/* 115 * Check, if the device is the broadcast device 116 */ 117int tick_is_broadcast_device(struct clock_event_device *dev) 118{ 119 return (dev && tick_broadcast_device.evtdev == dev); 120} 121 122static void err_broadcast(const struct cpumask *mask) 123{ 124 pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n"); 125} 126 127static void tick_device_setup_broadcast_func(struct clock_event_device *dev) 128{ 129 if (!dev->broadcast) 130 dev->broadcast = tick_broadcast; 131 if (!dev->broadcast) { 132 pr_warn_once("%s depends on broadcast, but no broadcast function available\n", 133 dev->name); 134 dev->broadcast = err_broadcast; 135 } 136} 137 138/* 139 * Check, if the device is disfunctional and a place holder, which 140 * needs to be handled by the broadcast device. 141 */ 142int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) 143{ 144 struct clock_event_device *bc = tick_broadcast_device.evtdev; 145 unsigned long flags; 146 int ret; 147 148 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 149 150 /* 151 * Devices might be registered with both periodic and oneshot 152 * mode disabled. This signals, that the device needs to be 153 * operated from the broadcast device and is a placeholder for 154 * the cpu local device. 155 */ 156 if (!tick_device_is_functional(dev)) { 157 dev->event_handler = tick_handle_periodic; 158 tick_device_setup_broadcast_func(dev); 159 cpumask_set_cpu(cpu, tick_broadcast_mask); 160 tick_broadcast_start_periodic(bc); 161 ret = 1; 162 } else { 163 /* 164 * Clear the broadcast bit for this cpu if the 165 * device is not power state affected. 166 */ 167 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) 168 cpumask_clear_cpu(cpu, tick_broadcast_mask); 169 else 170 tick_device_setup_broadcast_func(dev); 171 172 /* 173 * Clear the broadcast bit if the CPU is not in 174 * periodic broadcast on state. 175 */ 176 if (!cpumask_test_cpu(cpu, tick_broadcast_on)) 177 cpumask_clear_cpu(cpu, tick_broadcast_mask); 178 179 switch (tick_broadcast_device.mode) { 180 case TICKDEV_MODE_ONESHOT: 181 /* 182 * If the system is in oneshot mode we can 183 * unconditionally clear the oneshot mask bit, 184 * because the CPU is running and therefore 185 * not in an idle state which causes the power 186 * state affected device to stop. Let the 187 * caller initialize the device. 188 */ 189 tick_broadcast_clear_oneshot(cpu); 190 ret = 0; 191 break; 192 193 case TICKDEV_MODE_PERIODIC: 194 /* 195 * If the system is in periodic mode, check 196 * whether the broadcast device can be 197 * switched off now. 198 */ 199 if (cpumask_empty(tick_broadcast_mask) && bc) 200 clockevents_shutdown(bc); 201 /* 202 * If we kept the cpu in the broadcast mask, 203 * tell the caller to leave the per cpu device 204 * in shutdown state. The periodic interrupt 205 * is delivered by the broadcast device. 206 */ 207 ret = cpumask_test_cpu(cpu, tick_broadcast_mask); 208 break; 209 default: 210 /* Nothing to do */ 211 ret = 0; 212 break; 213 } 214 } 215 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 216 return ret; 217} 218 219#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 220int tick_receive_broadcast(void) 221{ 222 struct tick_device *td = this_cpu_ptr(&tick_cpu_device); 223 struct clock_event_device *evt = td->evtdev; 224 225 if (!evt) 226 return -ENODEV; 227 228 if (!evt->event_handler) 229 return -EINVAL; 230 231 evt->event_handler(evt); 232 return 0; 233} 234#endif 235 236/* 237 * Broadcast the event to the cpus, which are set in the mask (mangled). 238 */ 239static void tick_do_broadcast(struct cpumask *mask) 240{ 241 int cpu = smp_processor_id(); 242 struct tick_device *td; 243 244 /* 245 * Check, if the current cpu is in the mask 246 */ 247 if (cpumask_test_cpu(cpu, mask)) { 248 cpumask_clear_cpu(cpu, mask); 249 td = &per_cpu(tick_cpu_device, cpu); 250 td->evtdev->event_handler(td->evtdev); 251 } 252 253 if (!cpumask_empty(mask)) { 254 /* 255 * It might be necessary to actually check whether the devices 256 * have different broadcast functions. For now, just use the 257 * one of the first device. This works as long as we have this 258 * misfeature only on x86 (lapic) 259 */ 260 td = &per_cpu(tick_cpu_device, cpumask_first(mask)); 261 td->evtdev->broadcast(mask); 262 } 263} 264 265/* 266 * Periodic broadcast: 267 * - invoke the broadcast handlers 268 */ 269static void tick_do_periodic_broadcast(void) 270{ 271 raw_spin_lock(&tick_broadcast_lock); 272 273 cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask); 274 tick_do_broadcast(tmpmask); 275 276 raw_spin_unlock(&tick_broadcast_lock); 277} 278 279/* 280 * Event handler for periodic broadcast ticks 281 */ 282static void tick_handle_periodic_broadcast(struct clock_event_device *dev) 283{ 284 ktime_t next; 285 286 tick_do_periodic_broadcast(); 287 288 /* 289 * The device is in periodic mode. No reprogramming necessary: 290 */ 291 if (dev->mode == CLOCK_EVT_MODE_PERIODIC) 292 return; 293 294 /* 295 * Setup the next period for devices, which do not have 296 * periodic mode. We read dev->next_event first and add to it 297 * when the event already expired. clockevents_program_event() 298 * sets dev->next_event only when the event is really 299 * programmed to the device. 300 */ 301 for (next = dev->next_event; ;) { 302 next = ktime_add(next, tick_period); 303 304 if (!clockevents_program_event(dev, next, false)) 305 return; 306 tick_do_periodic_broadcast(); 307 } 308} 309 310/* 311 * Powerstate information: The system enters/leaves a state, where 312 * affected devices might stop 313 */ 314static void tick_do_broadcast_on_off(unsigned long *reason) 315{ 316 struct clock_event_device *bc, *dev; 317 struct tick_device *td; 318 unsigned long flags; 319 int cpu, bc_stopped; 320 321 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 322 323 cpu = smp_processor_id(); 324 td = &per_cpu(tick_cpu_device, cpu); 325 dev = td->evtdev; 326 bc = tick_broadcast_device.evtdev; 327 328 /* 329 * Is the device not affected by the powerstate ? 330 */ 331 if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP)) 332 goto out; 333 334 if (!tick_device_is_functional(dev)) 335 goto out; 336 337 bc_stopped = cpumask_empty(tick_broadcast_mask); 338 339 switch (*reason) { 340 case CLOCK_EVT_NOTIFY_BROADCAST_ON: 341 case CLOCK_EVT_NOTIFY_BROADCAST_FORCE: 342 cpumask_set_cpu(cpu, tick_broadcast_on); 343 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) { 344 if (tick_broadcast_device.mode == 345 TICKDEV_MODE_PERIODIC) 346 clockevents_shutdown(dev); 347 } 348 if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE) 349 tick_broadcast_force = 1; 350 break; 351 case CLOCK_EVT_NOTIFY_BROADCAST_OFF: 352 if (tick_broadcast_force) 353 break; 354 cpumask_clear_cpu(cpu, tick_broadcast_on); 355 if (!tick_device_is_functional(dev)) 356 break; 357 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) { 358 if (tick_broadcast_device.mode == 359 TICKDEV_MODE_PERIODIC) 360 tick_setup_periodic(dev, 0); 361 } 362 break; 363 } 364 365 if (cpumask_empty(tick_broadcast_mask)) { 366 if (!bc_stopped) 367 clockevents_shutdown(bc); 368 } else if (bc_stopped) { 369 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 370 tick_broadcast_start_periodic(bc); 371 else 372 tick_broadcast_setup_oneshot(bc); 373 } 374out: 375 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 376} 377 378/* 379 * Powerstate information: The system enters/leaves a state, where 380 * affected devices might stop. 381 */ 382void tick_broadcast_on_off(unsigned long reason, int *oncpu) 383{ 384 if (!cpumask_test_cpu(*oncpu, cpu_online_mask)) 385 printk(KERN_ERR "tick-broadcast: ignoring broadcast for " 386 "offline CPU #%d\n", *oncpu); 387 else 388 tick_do_broadcast_on_off(&reason); 389} 390 391/* 392 * Set the periodic handler depending on broadcast on/off 393 */ 394void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast) 395{ 396 if (!broadcast) 397 dev->event_handler = tick_handle_periodic; 398 else 399 dev->event_handler = tick_handle_periodic_broadcast; 400} 401 402/* 403 * Remove a CPU from broadcasting 404 */ 405void tick_shutdown_broadcast(unsigned int *cpup) 406{ 407 struct clock_event_device *bc; 408 unsigned long flags; 409 unsigned int cpu = *cpup; 410 411 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 412 413 bc = tick_broadcast_device.evtdev; 414 cpumask_clear_cpu(cpu, tick_broadcast_mask); 415 cpumask_clear_cpu(cpu, tick_broadcast_on); 416 417 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) { 418 if (bc && cpumask_empty(tick_broadcast_mask)) 419 clockevents_shutdown(bc); 420 } 421 422 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 423} 424 425void tick_suspend_broadcast(void) 426{ 427 struct clock_event_device *bc; 428 unsigned long flags; 429 430 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 431 432 bc = tick_broadcast_device.evtdev; 433 if (bc) 434 clockevents_shutdown(bc); 435 436 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 437} 438 439int tick_resume_broadcast(void) 440{ 441 struct clock_event_device *bc; 442 unsigned long flags; 443 int broadcast = 0; 444 445 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 446 447 bc = tick_broadcast_device.evtdev; 448 449 if (bc) { 450 clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME); 451 452 switch (tick_broadcast_device.mode) { 453 case TICKDEV_MODE_PERIODIC: 454 if (!cpumask_empty(tick_broadcast_mask)) 455 tick_broadcast_start_periodic(bc); 456 broadcast = cpumask_test_cpu(smp_processor_id(), 457 tick_broadcast_mask); 458 break; 459 case TICKDEV_MODE_ONESHOT: 460 if (!cpumask_empty(tick_broadcast_mask)) 461 broadcast = tick_resume_broadcast_oneshot(bc); 462 break; 463 } 464 } 465 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 466 467 return broadcast; 468} 469 470 471#ifdef CONFIG_TICK_ONESHOT 472 473static cpumask_var_t tick_broadcast_oneshot_mask; 474static cpumask_var_t tick_broadcast_pending_mask; 475static cpumask_var_t tick_broadcast_force_mask; 476 477/* 478 * Exposed for debugging: see timer_list.c 479 */ 480struct cpumask *tick_get_broadcast_oneshot_mask(void) 481{ 482 return tick_broadcast_oneshot_mask; 483} 484 485/* 486 * Called before going idle with interrupts disabled. Checks whether a 487 * broadcast event from the other core is about to happen. We detected 488 * that in tick_broadcast_oneshot_control(). The callsite can use this 489 * to avoid a deep idle transition as we are about to get the 490 * broadcast IPI right away. 491 */ 492int tick_check_broadcast_expired(void) 493{ 494 return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask); 495} 496 497/* 498 * Set broadcast interrupt affinity 499 */ 500static void tick_broadcast_set_affinity(struct clock_event_device *bc, 501 const struct cpumask *cpumask) 502{ 503 if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ)) 504 return; 505 506 if (cpumask_equal(bc->cpumask, cpumask)) 507 return; 508 509 bc->cpumask = cpumask; 510 irq_set_affinity(bc->irq, bc->cpumask); 511} 512 513static int tick_broadcast_set_event(struct clock_event_device *bc, int cpu, 514 ktime_t expires, int force) 515{ 516 int ret; 517 518 if (bc->mode != CLOCK_EVT_MODE_ONESHOT) 519 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 520 521 ret = clockevents_program_event(bc, expires, force); 522 if (!ret) 523 tick_broadcast_set_affinity(bc, cpumask_of(cpu)); 524 return ret; 525} 526 527int tick_resume_broadcast_oneshot(struct clock_event_device *bc) 528{ 529 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 530 return 0; 531} 532 533/* 534 * Called from irq_enter() when idle was interrupted to reenable the 535 * per cpu device. 536 */ 537void tick_check_oneshot_broadcast(int cpu) 538{ 539 if (cpumask_test_cpu(cpu, tick_broadcast_oneshot_mask)) { 540 struct tick_device *td = &per_cpu(tick_cpu_device, cpu); 541 542 /* 543 * We might be in the middle of switching over from 544 * periodic to oneshot. If the CPU has not yet 545 * switched over, leave the device alone. 546 */ 547 if (td->mode == TICKDEV_MODE_ONESHOT) { 548 clockevents_set_mode(td->evtdev, 549 CLOCK_EVT_MODE_ONESHOT); 550 } 551 } 552} 553 554/* 555 * Handle oneshot mode broadcasting 556 */ 557static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) 558{ 559 struct tick_device *td; 560 ktime_t now, next_event; 561 int cpu, next_cpu = 0; 562 563 raw_spin_lock(&tick_broadcast_lock); 564again: 565 dev->next_event.tv64 = KTIME_MAX; 566 next_event.tv64 = KTIME_MAX; 567 cpumask_clear(tmpmask); 568 now = ktime_get(); 569 /* Find all expired events */ 570 for_each_cpu(cpu, tick_broadcast_oneshot_mask) { 571 td = &per_cpu(tick_cpu_device, cpu); 572 if (td->evtdev->next_event.tv64 <= now.tv64) { 573 cpumask_set_cpu(cpu, tmpmask); 574 /* 575 * Mark the remote cpu in the pending mask, so 576 * it can avoid reprogramming the cpu local 577 * timer in tick_broadcast_oneshot_control(). 578 */ 579 cpumask_set_cpu(cpu, tick_broadcast_pending_mask); 580 } else if (td->evtdev->next_event.tv64 < next_event.tv64) { 581 next_event.tv64 = td->evtdev->next_event.tv64; 582 next_cpu = cpu; 583 } 584 } 585 586 /* Take care of enforced broadcast requests */ 587 cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask); 588 cpumask_clear(tick_broadcast_force_mask); 589 590 /* 591 * Sanity check. Catch the case where we try to broadcast to 592 * offline cpus. 593 */ 594 if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask))) 595 cpumask_and(tmpmask, tmpmask, cpu_online_mask); 596 597 /* 598 * Wakeup the cpus which have an expired event. 599 */ 600 tick_do_broadcast(tmpmask); 601 602 /* 603 * Two reasons for reprogram: 604 * 605 * - The global event did not expire any CPU local 606 * events. This happens in dyntick mode, as the maximum PIT 607 * delta is quite small. 608 * 609 * - There are pending events on sleeping CPUs which were not 610 * in the event mask 611 */ 612 if (next_event.tv64 != KTIME_MAX) { 613 /* 614 * Rearm the broadcast device. If event expired, 615 * repeat the above 616 */ 617 if (tick_broadcast_set_event(dev, next_cpu, next_event, 0)) 618 goto again; 619 } 620 raw_spin_unlock(&tick_broadcast_lock); 621} 622 623/* 624 * Powerstate information: The system enters/leaves a state, where 625 * affected devices might stop 626 */ 627void tick_broadcast_oneshot_control(unsigned long reason) 628{ 629 struct clock_event_device *bc, *dev; 630 struct tick_device *td; 631 unsigned long flags; 632 ktime_t now; 633 int cpu; 634 635 /* 636 * Periodic mode does not care about the enter/exit of power 637 * states 638 */ 639 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 640 return; 641 642 /* 643 * We are called with preemtion disabled from the depth of the 644 * idle code, so we can't be moved away. 645 */ 646 cpu = smp_processor_id(); 647 td = &per_cpu(tick_cpu_device, cpu); 648 dev = td->evtdev; 649 650 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) 651 return; 652 653 bc = tick_broadcast_device.evtdev; 654 655 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 656 if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) { 657 WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask)); 658 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) { 659 clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); 660 /* 661 * We only reprogram the broadcast timer if we 662 * did not mark ourself in the force mask and 663 * if the cpu local event is earlier than the 664 * broadcast event. If the current CPU is in 665 * the force mask, then we are going to be 666 * woken by the IPI right away. 667 */ 668 if (!cpumask_test_cpu(cpu, tick_broadcast_force_mask) && 669 dev->next_event.tv64 < bc->next_event.tv64) 670 tick_broadcast_set_event(bc, cpu, dev->next_event, 1); 671 } 672 } else { 673 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) { 674 clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); 675 if (dev->next_event.tv64 == KTIME_MAX) 676 goto out; 677 /* 678 * The cpu which was handling the broadcast 679 * timer marked this cpu in the broadcast 680 * pending mask and fired the broadcast 681 * IPI. So we are going to handle the expired 682 * event anyway via the broadcast IPI 683 * handler. No need to reprogram the timer 684 * with an already expired event. 685 */ 686 if (cpumask_test_and_clear_cpu(cpu, 687 tick_broadcast_pending_mask)) 688 goto out; 689 690 /* 691 * If the pending bit is not set, then we are 692 * either the CPU handling the broadcast 693 * interrupt or we got woken by something else. 694 * 695 * We are not longer in the broadcast mask, so 696 * if the cpu local expiry time is already 697 * reached, we would reprogram the cpu local 698 * timer with an already expired event. 699 * 700 * This can lead to a ping-pong when we return 701 * to idle and therefor rearm the broadcast 702 * timer before the cpu local timer was able 703 * to fire. This happens because the forced 704 * reprogramming makes sure that the event 705 * will happen in the future and depending on 706 * the min_delta setting this might be far 707 * enough out that the ping-pong starts. 708 * 709 * If the cpu local next_event has expired 710 * then we know that the broadcast timer 711 * next_event has expired as well and 712 * broadcast is about to be handled. So we 713 * avoid reprogramming and enforce that the 714 * broadcast handler, which did not run yet, 715 * will invoke the cpu local handler. 716 * 717 * We cannot call the handler directly from 718 * here, because we might be in a NOHZ phase 719 * and we did not go through the irq_enter() 720 * nohz fixups. 721 */ 722 now = ktime_get(); 723 if (dev->next_event.tv64 <= now.tv64) { 724 cpumask_set_cpu(cpu, tick_broadcast_force_mask); 725 goto out; 726 } 727 /* 728 * We got woken by something else. Reprogram 729 * the cpu local timer device. 730 */ 731 tick_program_event(dev->next_event, 1); 732 } 733 } 734out: 735 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 736} 737 738/* 739 * Reset the one shot broadcast for a cpu 740 * 741 * Called with tick_broadcast_lock held 742 */ 743static void tick_broadcast_clear_oneshot(int cpu) 744{ 745 cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask); 746} 747 748static void tick_broadcast_init_next_event(struct cpumask *mask, 749 ktime_t expires) 750{ 751 struct tick_device *td; 752 int cpu; 753 754 for_each_cpu(cpu, mask) { 755 td = &per_cpu(tick_cpu_device, cpu); 756 if (td->evtdev) 757 td->evtdev->next_event = expires; 758 } 759} 760 761/** 762 * tick_broadcast_setup_oneshot - setup the broadcast device 763 */ 764void tick_broadcast_setup_oneshot(struct clock_event_device *bc) 765{ 766 int cpu = smp_processor_id(); 767 768 /* Set it up only once ! */ 769 if (bc->event_handler != tick_handle_oneshot_broadcast) { 770 int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC; 771 772 bc->event_handler = tick_handle_oneshot_broadcast; 773 774 /* Take the do_timer update */ 775 if (!tick_nohz_full_cpu(cpu)) 776 tick_do_timer_cpu = cpu; 777 778 /* 779 * We must be careful here. There might be other CPUs 780 * waiting for periodic broadcast. We need to set the 781 * oneshot_mask bits for those and program the 782 * broadcast device to fire. 783 */ 784 cpumask_copy(tmpmask, tick_broadcast_mask); 785 cpumask_clear_cpu(cpu, tmpmask); 786 cpumask_or(tick_broadcast_oneshot_mask, 787 tick_broadcast_oneshot_mask, tmpmask); 788 789 if (was_periodic && !cpumask_empty(tmpmask)) { 790 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 791 tick_broadcast_init_next_event(tmpmask, 792 tick_next_period); 793 tick_broadcast_set_event(bc, cpu, tick_next_period, 1); 794 } else 795 bc->next_event.tv64 = KTIME_MAX; 796 } else { 797 /* 798 * The first cpu which switches to oneshot mode sets 799 * the bit for all other cpus which are in the general 800 * (periodic) broadcast mask. So the bit is set and 801 * would prevent the first broadcast enter after this 802 * to program the bc device. 803 */ 804 tick_broadcast_clear_oneshot(cpu); 805 } 806} 807 808/* 809 * Select oneshot operating mode for the broadcast device 810 */ 811void tick_broadcast_switch_to_oneshot(void) 812{ 813 struct clock_event_device *bc; 814 unsigned long flags; 815 816 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 817 818 tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT; 819 bc = tick_broadcast_device.evtdev; 820 if (bc) 821 tick_broadcast_setup_oneshot(bc); 822 823 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 824} 825 826 827/* 828 * Remove a dead CPU from broadcasting 829 */ 830void tick_shutdown_broadcast_oneshot(unsigned int *cpup) 831{ 832 unsigned long flags; 833 unsigned int cpu = *cpup; 834 835 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 836 837 /* 838 * Clear the broadcast masks for the dead cpu, but do not stop 839 * the broadcast device! 840 */ 841 cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask); 842 cpumask_clear_cpu(cpu, tick_broadcast_pending_mask); 843 cpumask_clear_cpu(cpu, tick_broadcast_force_mask); 844 845 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 846} 847 848/* 849 * Check, whether the broadcast device is in one shot mode 850 */ 851int tick_broadcast_oneshot_active(void) 852{ 853 return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT; 854} 855 856/* 857 * Check whether the broadcast device supports oneshot. 858 */ 859bool tick_broadcast_oneshot_available(void) 860{ 861 struct clock_event_device *bc = tick_broadcast_device.evtdev; 862 863 return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false; 864} 865 866#endif 867 868void __init tick_broadcast_init(void) 869{ 870 zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT); 871 zalloc_cpumask_var(&tick_broadcast_on, GFP_NOWAIT); 872 zalloc_cpumask_var(&tmpmask, GFP_NOWAIT); 873#ifdef CONFIG_TICK_ONESHOT 874 zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT); 875 zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT); 876 zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT); 877#endif 878} 879