tick-broadcast.c revision b352bc1cbc29134a356b5c16ee2281807a7b984e
1/* 2 * linux/kernel/time/tick-broadcast.c 3 * 4 * This file contains functions which emulate a local clock-event 5 * device via a broadcast event source. 6 * 7 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> 8 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar 9 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner 10 * 11 * This code is licenced under the GPL version 2. For details see 12 * kernel-base/COPYING. 13 */ 14#include <linux/cpu.h> 15#include <linux/err.h> 16#include <linux/hrtimer.h> 17#include <linux/interrupt.h> 18#include <linux/percpu.h> 19#include <linux/profile.h> 20#include <linux/sched.h> 21#include <linux/smp.h> 22 23#include "tick-internal.h" 24 25/* 26 * Broadcast support for broken x86 hardware, where the local apic 27 * timer stops in C3 state. 28 */ 29 30static struct tick_device tick_broadcast_device; 31static cpumask_var_t tick_broadcast_mask; 32static cpumask_var_t tmpmask; 33static DEFINE_RAW_SPINLOCK(tick_broadcast_lock); 34static int tick_broadcast_force; 35 36#ifdef CONFIG_TICK_ONESHOT 37static void tick_broadcast_clear_oneshot(int cpu); 38#else 39static inline void tick_broadcast_clear_oneshot(int cpu) { } 40#endif 41 42/* 43 * Debugging: see timer_list.c 44 */ 45struct tick_device *tick_get_broadcast_device(void) 46{ 47 return &tick_broadcast_device; 48} 49 50struct cpumask *tick_get_broadcast_mask(void) 51{ 52 return tick_broadcast_mask; 53} 54 55/* 56 * Start the device in periodic mode 57 */ 58static void tick_broadcast_start_periodic(struct clock_event_device *bc) 59{ 60 if (bc) 61 tick_setup_periodic(bc, 1); 62} 63 64/* 65 * Check, if the device can be utilized as broadcast device: 66 */ 67int tick_check_broadcast_device(struct clock_event_device *dev) 68{ 69 if ((tick_broadcast_device.evtdev && 70 tick_broadcast_device.evtdev->rating >= dev->rating) || 71 (dev->features & CLOCK_EVT_FEAT_C3STOP)) 72 return 0; 73 74 clockevents_exchange_device(tick_broadcast_device.evtdev, dev); 75 tick_broadcast_device.evtdev = dev; 76 if (!cpumask_empty(tick_broadcast_mask)) 77 tick_broadcast_start_periodic(dev); 78 return 1; 79} 80 81/* 82 * Check, if the device is the broadcast device 83 */ 84int tick_is_broadcast_device(struct clock_event_device *dev) 85{ 86 return (dev && tick_broadcast_device.evtdev == dev); 87} 88 89static void err_broadcast(const struct cpumask *mask) 90{ 91 pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n"); 92} 93 94static void tick_device_setup_broadcast_func(struct clock_event_device *dev) 95{ 96 if (!dev->broadcast) 97 dev->broadcast = tick_broadcast; 98 if (!dev->broadcast) { 99 pr_warn_once("%s depends on broadcast, but no broadcast function available\n", 100 dev->name); 101 dev->broadcast = err_broadcast; 102 } 103} 104 105/* 106 * Check, if the device is disfunctional and a place holder, which 107 * needs to be handled by the broadcast device. 108 */ 109int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) 110{ 111 unsigned long flags; 112 int ret = 0; 113 114 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 115 116 /* 117 * Devices might be registered with both periodic and oneshot 118 * mode disabled. This signals, that the device needs to be 119 * operated from the broadcast device and is a placeholder for 120 * the cpu local device. 121 */ 122 if (!tick_device_is_functional(dev)) { 123 dev->event_handler = tick_handle_periodic; 124 tick_device_setup_broadcast_func(dev); 125 cpumask_set_cpu(cpu, tick_broadcast_mask); 126 tick_broadcast_start_periodic(tick_broadcast_device.evtdev); 127 ret = 1; 128 } else { 129 /* 130 * When the new device is not affected by the stop 131 * feature and the cpu is marked in the broadcast mask 132 * then clear the broadcast bit. 133 */ 134 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) { 135 int cpu = smp_processor_id(); 136 cpumask_clear_cpu(cpu, tick_broadcast_mask); 137 tick_broadcast_clear_oneshot(cpu); 138 } else { 139 tick_device_setup_broadcast_func(dev); 140 } 141 } 142 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 143 return ret; 144} 145 146#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 147int tick_receive_broadcast(void) 148{ 149 struct tick_device *td = this_cpu_ptr(&tick_cpu_device); 150 struct clock_event_device *evt = td->evtdev; 151 152 if (!evt) 153 return -ENODEV; 154 155 if (!evt->event_handler) 156 return -EINVAL; 157 158 evt->event_handler(evt); 159 return 0; 160} 161#endif 162 163/* 164 * Broadcast the event to the cpus, which are set in the mask (mangled). 165 */ 166static void tick_do_broadcast(struct cpumask *mask) 167{ 168 int cpu = smp_processor_id(); 169 struct tick_device *td; 170 171 /* 172 * Check, if the current cpu is in the mask 173 */ 174 if (cpumask_test_cpu(cpu, mask)) { 175 cpumask_clear_cpu(cpu, mask); 176 td = &per_cpu(tick_cpu_device, cpu); 177 td->evtdev->event_handler(td->evtdev); 178 } 179 180 if (!cpumask_empty(mask)) { 181 /* 182 * It might be necessary to actually check whether the devices 183 * have different broadcast functions. For now, just use the 184 * one of the first device. This works as long as we have this 185 * misfeature only on x86 (lapic) 186 */ 187 td = &per_cpu(tick_cpu_device, cpumask_first(mask)); 188 td->evtdev->broadcast(mask); 189 } 190} 191 192/* 193 * Periodic broadcast: 194 * - invoke the broadcast handlers 195 */ 196static void tick_do_periodic_broadcast(void) 197{ 198 raw_spin_lock(&tick_broadcast_lock); 199 200 cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask); 201 tick_do_broadcast(tmpmask); 202 203 raw_spin_unlock(&tick_broadcast_lock); 204} 205 206/* 207 * Event handler for periodic broadcast ticks 208 */ 209static void tick_handle_periodic_broadcast(struct clock_event_device *dev) 210{ 211 ktime_t next; 212 213 tick_do_periodic_broadcast(); 214 215 /* 216 * The device is in periodic mode. No reprogramming necessary: 217 */ 218 if (dev->mode == CLOCK_EVT_MODE_PERIODIC) 219 return; 220 221 /* 222 * Setup the next period for devices, which do not have 223 * periodic mode. We read dev->next_event first and add to it 224 * when the event already expired. clockevents_program_event() 225 * sets dev->next_event only when the event is really 226 * programmed to the device. 227 */ 228 for (next = dev->next_event; ;) { 229 next = ktime_add(next, tick_period); 230 231 if (!clockevents_program_event(dev, next, false)) 232 return; 233 tick_do_periodic_broadcast(); 234 } 235} 236 237/* 238 * Powerstate information: The system enters/leaves a state, where 239 * affected devices might stop 240 */ 241static void tick_do_broadcast_on_off(unsigned long *reason) 242{ 243 struct clock_event_device *bc, *dev; 244 struct tick_device *td; 245 unsigned long flags; 246 int cpu, bc_stopped; 247 248 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 249 250 cpu = smp_processor_id(); 251 td = &per_cpu(tick_cpu_device, cpu); 252 dev = td->evtdev; 253 bc = tick_broadcast_device.evtdev; 254 255 /* 256 * Is the device not affected by the powerstate ? 257 */ 258 if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP)) 259 goto out; 260 261 if (!tick_device_is_functional(dev)) 262 goto out; 263 264 bc_stopped = cpumask_empty(tick_broadcast_mask); 265 266 switch (*reason) { 267 case CLOCK_EVT_NOTIFY_BROADCAST_ON: 268 case CLOCK_EVT_NOTIFY_BROADCAST_FORCE: 269 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) { 270 if (tick_broadcast_device.mode == 271 TICKDEV_MODE_PERIODIC) 272 clockevents_shutdown(dev); 273 } 274 if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE) 275 tick_broadcast_force = 1; 276 break; 277 case CLOCK_EVT_NOTIFY_BROADCAST_OFF: 278 if (!tick_broadcast_force && 279 cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) { 280 if (tick_broadcast_device.mode == 281 TICKDEV_MODE_PERIODIC) 282 tick_setup_periodic(dev, 0); 283 } 284 break; 285 } 286 287 if (cpumask_empty(tick_broadcast_mask)) { 288 if (!bc_stopped) 289 clockevents_shutdown(bc); 290 } else if (bc_stopped) { 291 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 292 tick_broadcast_start_periodic(bc); 293 else 294 tick_broadcast_setup_oneshot(bc); 295 } 296out: 297 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 298} 299 300/* 301 * Powerstate information: The system enters/leaves a state, where 302 * affected devices might stop. 303 */ 304void tick_broadcast_on_off(unsigned long reason, int *oncpu) 305{ 306 if (!cpumask_test_cpu(*oncpu, cpu_online_mask)) 307 printk(KERN_ERR "tick-broadcast: ignoring broadcast for " 308 "offline CPU #%d\n", *oncpu); 309 else 310 tick_do_broadcast_on_off(&reason); 311} 312 313/* 314 * Set the periodic handler depending on broadcast on/off 315 */ 316void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast) 317{ 318 if (!broadcast) 319 dev->event_handler = tick_handle_periodic; 320 else 321 dev->event_handler = tick_handle_periodic_broadcast; 322} 323 324/* 325 * Remove a CPU from broadcasting 326 */ 327void tick_shutdown_broadcast(unsigned int *cpup) 328{ 329 struct clock_event_device *bc; 330 unsigned long flags; 331 unsigned int cpu = *cpup; 332 333 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 334 335 bc = tick_broadcast_device.evtdev; 336 cpumask_clear_cpu(cpu, tick_broadcast_mask); 337 338 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) { 339 if (bc && cpumask_empty(tick_broadcast_mask)) 340 clockevents_shutdown(bc); 341 } 342 343 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 344} 345 346void tick_suspend_broadcast(void) 347{ 348 struct clock_event_device *bc; 349 unsigned long flags; 350 351 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 352 353 bc = tick_broadcast_device.evtdev; 354 if (bc) 355 clockevents_shutdown(bc); 356 357 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 358} 359 360int tick_resume_broadcast(void) 361{ 362 struct clock_event_device *bc; 363 unsigned long flags; 364 int broadcast = 0; 365 366 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 367 368 bc = tick_broadcast_device.evtdev; 369 370 if (bc) { 371 clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME); 372 373 switch (tick_broadcast_device.mode) { 374 case TICKDEV_MODE_PERIODIC: 375 if (!cpumask_empty(tick_broadcast_mask)) 376 tick_broadcast_start_periodic(bc); 377 broadcast = cpumask_test_cpu(smp_processor_id(), 378 tick_broadcast_mask); 379 break; 380 case TICKDEV_MODE_ONESHOT: 381 if (!cpumask_empty(tick_broadcast_mask)) 382 broadcast = tick_resume_broadcast_oneshot(bc); 383 break; 384 } 385 } 386 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 387 388 return broadcast; 389} 390 391 392#ifdef CONFIG_TICK_ONESHOT 393 394static cpumask_var_t tick_broadcast_oneshot_mask; 395 396/* 397 * Exposed for debugging: see timer_list.c 398 */ 399struct cpumask *tick_get_broadcast_oneshot_mask(void) 400{ 401 return tick_broadcast_oneshot_mask; 402} 403 404static int tick_broadcast_set_event(ktime_t expires, int force) 405{ 406 struct clock_event_device *bc = tick_broadcast_device.evtdev; 407 408 if (bc->mode != CLOCK_EVT_MODE_ONESHOT) 409 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 410 411 return clockevents_program_event(bc, expires, force); 412} 413 414int tick_resume_broadcast_oneshot(struct clock_event_device *bc) 415{ 416 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 417 return 0; 418} 419 420/* 421 * Called from irq_enter() when idle was interrupted to reenable the 422 * per cpu device. 423 */ 424void tick_check_oneshot_broadcast(int cpu) 425{ 426 if (cpumask_test_cpu(cpu, tick_broadcast_oneshot_mask)) { 427 struct tick_device *td = &per_cpu(tick_cpu_device, cpu); 428 429 clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT); 430 } 431} 432 433/* 434 * Handle oneshot mode broadcasting 435 */ 436static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) 437{ 438 struct tick_device *td; 439 ktime_t now, next_event; 440 int cpu; 441 442 raw_spin_lock(&tick_broadcast_lock); 443again: 444 dev->next_event.tv64 = KTIME_MAX; 445 next_event.tv64 = KTIME_MAX; 446 cpumask_clear(tmpmask); 447 now = ktime_get(); 448 /* Find all expired events */ 449 for_each_cpu(cpu, tick_broadcast_oneshot_mask) { 450 td = &per_cpu(tick_cpu_device, cpu); 451 if (td->evtdev->next_event.tv64 <= now.tv64) 452 cpumask_set_cpu(cpu, tmpmask); 453 else if (td->evtdev->next_event.tv64 < next_event.tv64) 454 next_event.tv64 = td->evtdev->next_event.tv64; 455 } 456 457 /* 458 * Wakeup the cpus which have an expired event. 459 */ 460 tick_do_broadcast(tmpmask); 461 462 /* 463 * Two reasons for reprogram: 464 * 465 * - The global event did not expire any CPU local 466 * events. This happens in dyntick mode, as the maximum PIT 467 * delta is quite small. 468 * 469 * - There are pending events on sleeping CPUs which were not 470 * in the event mask 471 */ 472 if (next_event.tv64 != KTIME_MAX) { 473 /* 474 * Rearm the broadcast device. If event expired, 475 * repeat the above 476 */ 477 if (tick_broadcast_set_event(next_event, 0)) 478 goto again; 479 } 480 raw_spin_unlock(&tick_broadcast_lock); 481} 482 483/* 484 * Powerstate information: The system enters/leaves a state, where 485 * affected devices might stop 486 */ 487void tick_broadcast_oneshot_control(unsigned long reason) 488{ 489 struct clock_event_device *bc, *dev; 490 struct tick_device *td; 491 unsigned long flags; 492 int cpu; 493 494 /* 495 * Periodic mode does not care about the enter/exit of power 496 * states 497 */ 498 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 499 return; 500 501 /* 502 * We are called with preemtion disabled from the depth of the 503 * idle code, so we can't be moved away. 504 */ 505 cpu = smp_processor_id(); 506 td = &per_cpu(tick_cpu_device, cpu); 507 dev = td->evtdev; 508 509 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) 510 return; 511 512 bc = tick_broadcast_device.evtdev; 513 514 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 515 if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) { 516 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) { 517 clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); 518 if (dev->next_event.tv64 < bc->next_event.tv64) 519 tick_broadcast_set_event(dev->next_event, 1); 520 } 521 } else { 522 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) { 523 clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); 524 if (dev->next_event.tv64 != KTIME_MAX) 525 tick_program_event(dev->next_event, 1); 526 } 527 } 528 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 529} 530 531/* 532 * Reset the one shot broadcast for a cpu 533 * 534 * Called with tick_broadcast_lock held 535 */ 536static void tick_broadcast_clear_oneshot(int cpu) 537{ 538 cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask); 539} 540 541static void tick_broadcast_init_next_event(struct cpumask *mask, 542 ktime_t expires) 543{ 544 struct tick_device *td; 545 int cpu; 546 547 for_each_cpu(cpu, mask) { 548 td = &per_cpu(tick_cpu_device, cpu); 549 if (td->evtdev) 550 td->evtdev->next_event = expires; 551 } 552} 553 554/** 555 * tick_broadcast_setup_oneshot - setup the broadcast device 556 */ 557void tick_broadcast_setup_oneshot(struct clock_event_device *bc) 558{ 559 int cpu = smp_processor_id(); 560 561 /* Set it up only once ! */ 562 if (bc->event_handler != tick_handle_oneshot_broadcast) { 563 int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC; 564 565 bc->event_handler = tick_handle_oneshot_broadcast; 566 567 /* Take the do_timer update */ 568 tick_do_timer_cpu = cpu; 569 570 /* 571 * We must be careful here. There might be other CPUs 572 * waiting for periodic broadcast. We need to set the 573 * oneshot_mask bits for those and program the 574 * broadcast device to fire. 575 */ 576 cpumask_copy(tmpmask, tick_broadcast_mask); 577 cpumask_clear_cpu(cpu, tmpmask); 578 cpumask_or(tick_broadcast_oneshot_mask, 579 tick_broadcast_oneshot_mask, tmpmask); 580 581 if (was_periodic && !cpumask_empty(tmpmask)) { 582 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 583 tick_broadcast_init_next_event(tmpmask, 584 tick_next_period); 585 tick_broadcast_set_event(tick_next_period, 1); 586 } else 587 bc->next_event.tv64 = KTIME_MAX; 588 } else { 589 /* 590 * The first cpu which switches to oneshot mode sets 591 * the bit for all other cpus which are in the general 592 * (periodic) broadcast mask. So the bit is set and 593 * would prevent the first broadcast enter after this 594 * to program the bc device. 595 */ 596 tick_broadcast_clear_oneshot(cpu); 597 } 598} 599 600/* 601 * Select oneshot operating mode for the broadcast device 602 */ 603void tick_broadcast_switch_to_oneshot(void) 604{ 605 struct clock_event_device *bc; 606 unsigned long flags; 607 608 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 609 610 tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT; 611 bc = tick_broadcast_device.evtdev; 612 if (bc) 613 tick_broadcast_setup_oneshot(bc); 614 615 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 616} 617 618 619/* 620 * Remove a dead CPU from broadcasting 621 */ 622void tick_shutdown_broadcast_oneshot(unsigned int *cpup) 623{ 624 unsigned long flags; 625 unsigned int cpu = *cpup; 626 627 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 628 629 /* 630 * Clear the broadcast mask flag for the dead cpu, but do not 631 * stop the broadcast device! 632 */ 633 cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask); 634 635 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 636} 637 638/* 639 * Check, whether the broadcast device is in one shot mode 640 */ 641int tick_broadcast_oneshot_active(void) 642{ 643 return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT; 644} 645 646/* 647 * Check whether the broadcast device supports oneshot. 648 */ 649bool tick_broadcast_oneshot_available(void) 650{ 651 struct clock_event_device *bc = tick_broadcast_device.evtdev; 652 653 return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false; 654} 655 656#endif 657 658void __init tick_broadcast_init(void) 659{ 660 alloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT); 661 alloc_cpumask_var(&tmpmask, GFP_NOWAIT); 662#ifdef CONFIG_TICK_ONESHOT 663 alloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT); 664#endif 665} 666