tick-broadcast.c revision 6321dd60c76b2e12383bc06046288b15397ed3a0
1/* 2 * linux/kernel/time/tick-broadcast.c 3 * 4 * This file contains functions which emulate a local clock-event 5 * device via a broadcast event source. 6 * 7 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> 8 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar 9 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner 10 * 11 * This code is licenced under the GPL version 2. For details see 12 * kernel-base/COPYING. 13 */ 14#include <linux/cpu.h> 15#include <linux/err.h> 16#include <linux/hrtimer.h> 17#include <linux/irq.h> 18#include <linux/percpu.h> 19#include <linux/profile.h> 20#include <linux/sched.h> 21#include <linux/tick.h> 22 23#include "tick-internal.h" 24 25/* 26 * Broadcast support for broken x86 hardware, where the local apic 27 * timer stops in C3 state. 28 */ 29 30struct tick_device tick_broadcast_device; 31static cpumask_t tick_broadcast_mask; 32static DEFINE_SPINLOCK(tick_broadcast_lock); 33 34/* 35 * Debugging: see timer_list.c 36 */ 37struct tick_device *tick_get_broadcast_device(void) 38{ 39 return &tick_broadcast_device; 40} 41 42cpumask_t *tick_get_broadcast_mask(void) 43{ 44 return &tick_broadcast_mask; 45} 46 47/* 48 * Start the device in periodic mode 49 */ 50static void tick_broadcast_start_periodic(struct clock_event_device *bc) 51{ 52 if (bc && bc->mode == CLOCK_EVT_MODE_SHUTDOWN) 53 tick_setup_periodic(bc, 1); 54} 55 56/* 57 * Check, if the device can be utilized as broadcast device: 58 */ 59int tick_check_broadcast_device(struct clock_event_device *dev) 60{ 61 if (tick_broadcast_device.evtdev || 62 (dev->features & CLOCK_EVT_FEAT_C3STOP)) 63 return 0; 64 65 clockevents_exchange_device(NULL, dev); 66 tick_broadcast_device.evtdev = dev; 67 if (!cpus_empty(tick_broadcast_mask)) 68 tick_broadcast_start_periodic(dev); 69 return 1; 70} 71 72/* 73 * Check, if the device is the broadcast device 74 */ 75int tick_is_broadcast_device(struct clock_event_device *dev) 76{ 77 return (dev && tick_broadcast_device.evtdev == dev); 78} 79 80/* 81 * Check, if the device is disfunctional and a place holder, which 82 * needs to be handled by the broadcast device. 83 */ 84int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) 85{ 86 unsigned long flags; 87 int ret = 0; 88 89 spin_lock_irqsave(&tick_broadcast_lock, flags); 90 91 /* 92 * Devices might be registered with both periodic and oneshot 93 * mode disabled. This signals, that the device needs to be 94 * operated from the broadcast device and is a placeholder for 95 * the cpu local device. 96 */ 97 if (!tick_device_is_functional(dev)) { 98 dev->event_handler = tick_handle_periodic; 99 cpu_set(cpu, tick_broadcast_mask); 100 tick_broadcast_start_periodic(tick_broadcast_device.evtdev); 101 ret = 1; 102 } 103 104 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 105 return ret; 106} 107 108/* 109 * Broadcast the event to the cpus, which are set in the mask 110 */ 111int tick_do_broadcast(cpumask_t mask) 112{ 113 int ret = 0, cpu = smp_processor_id(); 114 struct tick_device *td; 115 116 /* 117 * Check, if the current cpu is in the mask 118 */ 119 if (cpu_isset(cpu, mask)) { 120 cpu_clear(cpu, mask); 121 td = &per_cpu(tick_cpu_device, cpu); 122 td->evtdev->event_handler(td->evtdev); 123 ret = 1; 124 } 125 126 if (!cpus_empty(mask)) { 127 /* 128 * It might be necessary to actually check whether the devices 129 * have different broadcast functions. For now, just use the 130 * one of the first device. This works as long as we have this 131 * misfeature only on x86 (lapic) 132 */ 133 cpu = first_cpu(mask); 134 td = &per_cpu(tick_cpu_device, cpu); 135 td->evtdev->broadcast(mask); 136 ret = 1; 137 } 138 return ret; 139} 140 141/* 142 * Periodic broadcast: 143 * - invoke the broadcast handlers 144 */ 145static void tick_do_periodic_broadcast(void) 146{ 147 cpumask_t mask; 148 149 spin_lock(&tick_broadcast_lock); 150 151 cpus_and(mask, cpu_online_map, tick_broadcast_mask); 152 tick_do_broadcast(mask); 153 154 spin_unlock(&tick_broadcast_lock); 155} 156 157/* 158 * Event handler for periodic broadcast ticks 159 */ 160static void tick_handle_periodic_broadcast(struct clock_event_device *dev) 161{ 162 dev->next_event.tv64 = KTIME_MAX; 163 164 tick_do_periodic_broadcast(); 165 166 /* 167 * The device is in periodic mode. No reprogramming necessary: 168 */ 169 if (dev->mode == CLOCK_EVT_MODE_PERIODIC) 170 return; 171 172 /* 173 * Setup the next period for devices, which do not have 174 * periodic mode: 175 */ 176 for (;;) { 177 ktime_t next = ktime_add(dev->next_event, tick_period); 178 179 if (!clockevents_program_event(dev, next, ktime_get())) 180 return; 181 tick_do_periodic_broadcast(); 182 } 183} 184 185/* 186 * Powerstate information: The system enters/leaves a state, where 187 * affected devices might stop 188 */ 189static void tick_do_broadcast_on_off(void *why) 190{ 191 struct clock_event_device *bc, *dev; 192 struct tick_device *td; 193 unsigned long flags, *reason = why; 194 int cpu; 195 196 spin_lock_irqsave(&tick_broadcast_lock, flags); 197 198 cpu = smp_processor_id(); 199 td = &per_cpu(tick_cpu_device, cpu); 200 dev = td->evtdev; 201 bc = tick_broadcast_device.evtdev; 202 203 /* 204 * Is the device in broadcast mode forever or is it not 205 * affected by the powerstate ? 206 */ 207 if (!dev || !tick_device_is_functional(dev) || 208 !(dev->features & CLOCK_EVT_FEAT_C3STOP)) 209 goto out; 210 211 if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_ON) { 212 if (!cpu_isset(cpu, tick_broadcast_mask)) { 213 cpu_set(cpu, tick_broadcast_mask); 214 if (td->mode == TICKDEV_MODE_PERIODIC) 215 clockevents_set_mode(dev, 216 CLOCK_EVT_MODE_SHUTDOWN); 217 } 218 } else { 219 if (cpu_isset(cpu, tick_broadcast_mask)) { 220 cpu_clear(cpu, tick_broadcast_mask); 221 if (td->mode == TICKDEV_MODE_PERIODIC) 222 tick_setup_periodic(dev, 0); 223 } 224 } 225 226 if (cpus_empty(tick_broadcast_mask)) 227 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN); 228 else { 229 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 230 tick_broadcast_start_periodic(bc); 231 else 232 tick_broadcast_setup_oneshot(bc); 233 } 234out: 235 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 236} 237 238/* 239 * Powerstate information: The system enters/leaves a state, where 240 * affected devices might stop. 241 */ 242void tick_broadcast_on_off(unsigned long reason, int *oncpu) 243{ 244 int cpu = get_cpu(); 245 246 if (cpu == *oncpu) 247 tick_do_broadcast_on_off(&reason); 248 else 249 smp_call_function_single(*oncpu, tick_do_broadcast_on_off, 250 &reason, 1, 1); 251 put_cpu(); 252} 253 254/* 255 * Set the periodic handler depending on broadcast on/off 256 */ 257void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast) 258{ 259 if (!broadcast) 260 dev->event_handler = tick_handle_periodic; 261 else 262 dev->event_handler = tick_handle_periodic_broadcast; 263} 264 265/* 266 * Remove a CPU from broadcasting 267 */ 268void tick_shutdown_broadcast(unsigned int *cpup) 269{ 270 struct clock_event_device *bc; 271 unsigned long flags; 272 unsigned int cpu = *cpup; 273 274 spin_lock_irqsave(&tick_broadcast_lock, flags); 275 276 bc = tick_broadcast_device.evtdev; 277 cpu_clear(cpu, tick_broadcast_mask); 278 279 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) { 280 if (bc && cpus_empty(tick_broadcast_mask)) 281 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN); 282 } 283 284 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 285} 286 287void tick_suspend_broadcast(void) 288{ 289 struct clock_event_device *bc; 290 unsigned long flags; 291 292 spin_lock_irqsave(&tick_broadcast_lock, flags); 293 294 bc = tick_broadcast_device.evtdev; 295 if (bc && tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 296 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN); 297 298 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 299} 300 301int tick_resume_broadcast(void) 302{ 303 struct clock_event_device *bc; 304 unsigned long flags; 305 int broadcast = 0; 306 307 spin_lock_irqsave(&tick_broadcast_lock, flags); 308 309 bc = tick_broadcast_device.evtdev; 310 if (bc) { 311 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC && 312 !cpus_empty(tick_broadcast_mask)) 313 tick_broadcast_start_periodic(bc); 314 315 broadcast = cpu_isset(smp_processor_id(), tick_broadcast_mask); 316 } 317 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 318 319 return broadcast; 320} 321 322 323#ifdef CONFIG_TICK_ONESHOT 324 325static cpumask_t tick_broadcast_oneshot_mask; 326 327/* 328 * Debugging: see timer_list.c 329 */ 330cpumask_t *tick_get_broadcast_oneshot_mask(void) 331{ 332 return &tick_broadcast_oneshot_mask; 333} 334 335static int tick_broadcast_set_event(ktime_t expires, int force) 336{ 337 struct clock_event_device *bc = tick_broadcast_device.evtdev; 338 ktime_t now = ktime_get(); 339 int res; 340 341 for(;;) { 342 res = clockevents_program_event(bc, expires, now); 343 if (!res || !force) 344 return res; 345 now = ktime_get(); 346 expires = ktime_add(now, ktime_set(0, bc->min_delta_ns)); 347 } 348} 349 350/* 351 * Reprogram the broadcast device: 352 * 353 * Called with tick_broadcast_lock held and interrupts disabled. 354 */ 355static int tick_broadcast_reprogram(void) 356{ 357 ktime_t expires = { .tv64 = KTIME_MAX }; 358 struct tick_device *td; 359 int cpu; 360 361 /* 362 * Find the event which expires next: 363 */ 364 for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS; 365 cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) { 366 td = &per_cpu(tick_cpu_device, cpu); 367 if (td->evtdev->next_event.tv64 < expires.tv64) 368 expires = td->evtdev->next_event; 369 } 370 371 if (expires.tv64 == KTIME_MAX) 372 return 0; 373 374 return tick_broadcast_set_event(expires, 0); 375} 376 377/* 378 * Handle oneshot mode broadcasting 379 */ 380static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) 381{ 382 struct tick_device *td; 383 cpumask_t mask; 384 ktime_t now; 385 int cpu; 386 387 spin_lock(&tick_broadcast_lock); 388again: 389 dev->next_event.tv64 = KTIME_MAX; 390 mask = CPU_MASK_NONE; 391 now = ktime_get(); 392 /* Find all expired events */ 393 for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS; 394 cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) { 395 td = &per_cpu(tick_cpu_device, cpu); 396 if (td->evtdev->next_event.tv64 <= now.tv64) 397 cpu_set(cpu, mask); 398 } 399 400 /* 401 * Wakeup the cpus which have an expired event. The broadcast 402 * device is reprogrammed in the return from idle code. 403 */ 404 if (!tick_do_broadcast(mask)) { 405 /* 406 * The global event did not expire any CPU local 407 * events. This happens in dyntick mode, as the 408 * maximum PIT delta is quite small. 409 */ 410 if (tick_broadcast_reprogram()) 411 goto again; 412 } 413 spin_unlock(&tick_broadcast_lock); 414} 415 416/* 417 * Powerstate information: The system enters/leaves a state, where 418 * affected devices might stop 419 */ 420void tick_broadcast_oneshot_control(unsigned long reason) 421{ 422 struct clock_event_device *bc, *dev; 423 struct tick_device *td; 424 unsigned long flags; 425 int cpu; 426 427 spin_lock_irqsave(&tick_broadcast_lock, flags); 428 429 /* 430 * Periodic mode does not care about the enter/exit of power 431 * states 432 */ 433 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 434 goto out; 435 436 bc = tick_broadcast_device.evtdev; 437 cpu = smp_processor_id(); 438 td = &per_cpu(tick_cpu_device, cpu); 439 dev = td->evtdev; 440 441 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) 442 goto out; 443 444 if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) { 445 if (!cpu_isset(cpu, tick_broadcast_oneshot_mask)) { 446 cpu_set(cpu, tick_broadcast_oneshot_mask); 447 clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); 448 if (dev->next_event.tv64 < bc->next_event.tv64) 449 tick_broadcast_set_event(dev->next_event, 1); 450 } 451 } else { 452 if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) { 453 cpu_clear(cpu, tick_broadcast_oneshot_mask); 454 clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); 455 if (dev->next_event.tv64 != KTIME_MAX) 456 tick_program_event(dev->next_event, 1); 457 } 458 } 459 460out: 461 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 462} 463 464/** 465 * tick_broadcast_setup_highres - setup the broadcast device for highres 466 */ 467void tick_broadcast_setup_oneshot(struct clock_event_device *bc) 468{ 469 if (bc->mode != CLOCK_EVT_MODE_ONESHOT) { 470 bc->event_handler = tick_handle_oneshot_broadcast; 471 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 472 bc->next_event.tv64 = KTIME_MAX; 473 } 474} 475 476/* 477 * Select oneshot operating mode for the broadcast device 478 */ 479void tick_broadcast_switch_to_oneshot(void) 480{ 481 struct clock_event_device *bc; 482 unsigned long flags; 483 484 spin_lock_irqsave(&tick_broadcast_lock, flags); 485 486 tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT; 487 bc = tick_broadcast_device.evtdev; 488 if (bc) 489 tick_broadcast_setup_oneshot(bc); 490 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 491} 492 493 494/* 495 * Remove a dead CPU from broadcasting 496 */ 497void tick_shutdown_broadcast_oneshot(unsigned int *cpup) 498{ 499 struct clock_event_device *bc; 500 unsigned long flags; 501 unsigned int cpu = *cpup; 502 503 spin_lock_irqsave(&tick_broadcast_lock, flags); 504 505 bc = tick_broadcast_device.evtdev; 506 cpu_clear(cpu, tick_broadcast_oneshot_mask); 507 508 if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT) { 509 if (bc && cpus_empty(tick_broadcast_oneshot_mask)) 510 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN); 511 } 512 513 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 514} 515 516#endif 517