tick-broadcast.c revision 4a93232dab0a07074bcc5291a0f1f39919916f31 
1/*
2 * linux/kernel/time/tick-broadcast.c
3 *
4 * This file contains functions which emulate a local clock-event
5 * device via a broadcast event source.
6 *
7 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
8 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
9 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
10 *
11 * This code is licenced under the GPL version 2. For details see
12 * kernel-base/COPYING.
13 */
14#include <linux/cpu.h>
15#include <linux/err.h>
16#include <linux/hrtimer.h>
17#include <linux/irq.h>
18#include <linux/percpu.h>
19#include <linux/profile.h>
20#include <linux/sched.h>
21#include <linux/tick.h>
22
23#include "tick-internal.h"
24
25/*
26 * Broadcast support for broken x86 hardware, where the local apic
27 * timer stops in C3 state.
28 */
29
30struct tick_device tick_broadcast_device;
31static cpumask_t tick_broadcast_mask;
32static DEFINE_SPINLOCK(tick_broadcast_lock);
33
34#ifdef CONFIG_TICK_ONESHOT
35static void tick_broadcast_clear_oneshot(int cpu);
36#else
37static inline void tick_broadcast_clear_oneshot(int cpu) { }
38#endif
39
40/*
41 * Debugging: see timer_list.c
42 */
43struct tick_device *tick_get_broadcast_device(void)
44{
45	return &tick_broadcast_device;
46}
47
48cpumask_t *tick_get_broadcast_mask(void)
49{
50	return &tick_broadcast_mask;
51}
52
53/*
54 * Start the device in periodic mode
55 */
56static void tick_broadcast_start_periodic(struct clock_event_device *bc)
57{
58	if (bc)
59		tick_setup_periodic(bc, 1);
60}
61
62/*
63 * Check, if the device can be utilized as broadcast device:
64 */
65int tick_check_broadcast_device(struct clock_event_device *dev)
66{
67	if ((tick_broadcast_device.evtdev &&
68	     tick_broadcast_device.evtdev->rating >= dev->rating) ||
69	     (dev->features & CLOCK_EVT_FEAT_C3STOP))
70		return 0;
71
72	clockevents_exchange_device(NULL, dev);
73	tick_broadcast_device.evtdev = dev;
74	if (!cpus_empty(tick_broadcast_mask))
75		tick_broadcast_start_periodic(dev);
76	return 1;
77}
78
79/*
80 * Check, if the device is the broadcast device
81 */
82int tick_is_broadcast_device(struct clock_event_device *dev)
83{
84	return (dev && tick_broadcast_device.evtdev == dev);
85}
86
87/*
88 * Check, if the device is disfunctional and a place holder, which
89 * needs to be handled by the broadcast device.
90 */
91int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
92{
93	unsigned long flags;
94	int ret = 0;
95
96	spin_lock_irqsave(&tick_broadcast_lock, flags);
97
98	/*
99	 * Devices might be registered with both periodic and oneshot
100	 * mode disabled. This signals, that the device needs to be
101	 * operated from the broadcast device and is a placeholder for
102	 * the cpu local device.
103	 */
104	if (!tick_device_is_functional(dev)) {
105		dev->event_handler = tick_handle_periodic;
106		cpu_set(cpu, tick_broadcast_mask);
107		tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
108		ret = 1;
109	} else {
110		/*
111		 * When the new device is not affected by the stop
112		 * feature and the cpu is marked in the broadcast mask
113		 * then clear the broadcast bit.
114		 */
115		if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
116			int cpu = smp_processor_id();
117
118			cpu_clear(cpu, tick_broadcast_mask);
119			tick_broadcast_clear_oneshot(cpu);
120		}
121	}
122	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
123	return ret;
124}
125
126/*
127 * Broadcast the event to the cpus, which are set in the mask
128 */
129int tick_do_broadcast(cpumask_t mask)
130{
131	int ret = 0, cpu = smp_processor_id();
132	struct tick_device *td;
133
134	/*
135	 * Check, if the current cpu is in the mask
136	 */
137	if (cpu_isset(cpu, mask)) {
138		cpu_clear(cpu, mask);
139		td = &per_cpu(tick_cpu_device, cpu);
140		td->evtdev->event_handler(td->evtdev);
141		ret = 1;
142	}
143
144	if (!cpus_empty(mask)) {
145		/*
146		 * It might be necessary to actually check whether the devices
147		 * have different broadcast functions. For now, just use the
148		 * one of the first device. This works as long as we have this
149		 * misfeature only on x86 (lapic)
150		 */
151		cpu = first_cpu(mask);
152		td = &per_cpu(tick_cpu_device, cpu);
153		td->evtdev->broadcast(mask);
154		ret = 1;
155	}
156	return ret;
157}
158
159/*
160 * Periodic broadcast:
161 * - invoke the broadcast handlers
162 */
163static void tick_do_periodic_broadcast(void)
164{
165	cpumask_t mask;
166
167	spin_lock(&tick_broadcast_lock);
168
169	cpus_and(mask, cpu_online_map, tick_broadcast_mask);
170	tick_do_broadcast(mask);
171
172	spin_unlock(&tick_broadcast_lock);
173}
174
175/*
176 * Event handler for periodic broadcast ticks
177 */
178static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
179{
180	tick_do_periodic_broadcast();
181
182	/*
183	 * The device is in periodic mode. No reprogramming necessary:
184	 */
185	if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
186		return;
187
188	/*
189	 * Setup the next period for devices, which do not have
190	 * periodic mode:
191	 */
192	for (;;) {
193		ktime_t next = ktime_add(dev->next_event, tick_period);
194
195		if (!clockevents_program_event(dev, next, ktime_get()))
196			return;
197		tick_do_periodic_broadcast();
198	}
199}
200
201/*
202 * Powerstate information: The system enters/leaves a state, where
203 * affected devices might stop
204 */
205static void tick_do_broadcast_on_off(void *why)
206{
207	struct clock_event_device *bc, *dev;
208	struct tick_device *td;
209	unsigned long flags, *reason = why;
210	int cpu;
211
212	spin_lock_irqsave(&tick_broadcast_lock, flags);
213
214	cpu = smp_processor_id();
215	td = &per_cpu(tick_cpu_device, cpu);
216	dev = td->evtdev;
217	bc = tick_broadcast_device.evtdev;
218
219	/*
220	 * Is the device in broadcast mode forever or is it not
221	 * affected by the powerstate ?
222	 */
223	if (!dev || !tick_device_is_functional(dev) ||
224	    !(dev->features & CLOCK_EVT_FEAT_C3STOP))
225		goto out;
226
227	if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_ON) {
228		if (!cpu_isset(cpu, tick_broadcast_mask)) {
229			cpu_set(cpu, tick_broadcast_mask);
230			if (td->mode == TICKDEV_MODE_PERIODIC)
231				clockevents_set_mode(dev,
232						     CLOCK_EVT_MODE_SHUTDOWN);
233		}
234	} else {
235		if (cpu_isset(cpu, tick_broadcast_mask)) {
236			cpu_clear(cpu, tick_broadcast_mask);
237			if (td->mode == TICKDEV_MODE_PERIODIC)
238				tick_setup_periodic(dev, 0);
239		}
240	}
241
242	if (cpus_empty(tick_broadcast_mask))
243		clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
244	else {
245		if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
246			tick_broadcast_start_periodic(bc);
247		else
248			tick_broadcast_setup_oneshot(bc);
249	}
250out:
251	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
252}
253
254/*
255 * Powerstate information: The system enters/leaves a state, where
256 * affected devices might stop.
257 */
258void tick_broadcast_on_off(unsigned long reason, int *oncpu)
259{
260	int cpu = get_cpu();
261
262	if (!cpu_isset(*oncpu, cpu_online_map)) {
263		printk(KERN_ERR "tick-braodcast: ignoring broadcast for "
264		       "offline CPU #%d\n", *oncpu);
265	} else {
266
267		if (cpu == *oncpu)
268			tick_do_broadcast_on_off(&reason);
269		else
270			smp_call_function_single(*oncpu,
271						 tick_do_broadcast_on_off,
272						 &reason, 1, 1);
273	}
274	put_cpu();
275}
276
277/*
278 * Set the periodic handler depending on broadcast on/off
279 */
280void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
281{
282	if (!broadcast)
283		dev->event_handler = tick_handle_periodic;
284	else
285		dev->event_handler = tick_handle_periodic_broadcast;
286}
287
288/*
289 * Remove a CPU from broadcasting
290 */
291void tick_shutdown_broadcast(unsigned int *cpup)
292{
293	struct clock_event_device *bc;
294	unsigned long flags;
295	unsigned int cpu = *cpup;
296
297	spin_lock_irqsave(&tick_broadcast_lock, flags);
298
299	bc = tick_broadcast_device.evtdev;
300	cpu_clear(cpu, tick_broadcast_mask);
301
302	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
303		if (bc && cpus_empty(tick_broadcast_mask))
304			clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
305	}
306
307	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
308}
309
310void tick_suspend_broadcast(void)
311{
312	struct clock_event_device *bc;
313	unsigned long flags;
314
315	spin_lock_irqsave(&tick_broadcast_lock, flags);
316
317	bc = tick_broadcast_device.evtdev;
318	if (bc)
319		clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
320
321	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
322}
323
324int tick_resume_broadcast(void)
325{
326	struct clock_event_device *bc;
327	unsigned long flags;
328	int broadcast = 0;
329
330	spin_lock_irqsave(&tick_broadcast_lock, flags);
331
332	bc = tick_broadcast_device.evtdev;
333
334	if (bc) {
335		clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);
336
337		switch (tick_broadcast_device.mode) {
338		case TICKDEV_MODE_PERIODIC:
339			if(!cpus_empty(tick_broadcast_mask))
340				tick_broadcast_start_periodic(bc);
341			broadcast = cpu_isset(smp_processor_id(),
342					      tick_broadcast_mask);
343			break;
344		case TICKDEV_MODE_ONESHOT:
345			broadcast = tick_resume_broadcast_oneshot(bc);
346			break;
347		}
348	}
349	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
350
351	return broadcast;
352}
353
354
355#ifdef CONFIG_TICK_ONESHOT
356
357static cpumask_t tick_broadcast_oneshot_mask;
358
359/*
360 * Debugging: see timer_list.c
361 */
362cpumask_t *tick_get_broadcast_oneshot_mask(void)
363{
364	return &tick_broadcast_oneshot_mask;
365}
366
367static int tick_broadcast_set_event(ktime_t expires, int force)
368{
369	struct clock_event_device *bc = tick_broadcast_device.evtdev;
370	ktime_t now = ktime_get();
371	int res;
372
373	for(;;) {
374		res = clockevents_program_event(bc, expires, now);
375		if (!res || !force)
376			return res;
377		now = ktime_get();
378		expires = ktime_add(now, ktime_set(0, bc->min_delta_ns));
379	}
380}
381
382int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
383{
384	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
385	return 0;
386}
387
388/*
389 * Reprogram the broadcast device:
390 *
391 * Called with tick_broadcast_lock held and interrupts disabled.
392 */
393static int tick_broadcast_reprogram(void)
394{
395	ktime_t expires = { .tv64 = KTIME_MAX };
396	struct tick_device *td;
397	int cpu;
398
399	/*
400	 * Find the event which expires next:
401	 */
402	for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
403	     cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
404		td = &per_cpu(tick_cpu_device, cpu);
405		if (td->evtdev->next_event.tv64 < expires.tv64)
406			expires = td->evtdev->next_event;
407	}
408
409	if (expires.tv64 == KTIME_MAX)
410		return 0;
411
412	return tick_broadcast_set_event(expires, 0);
413}
414
415/*
416 * Handle oneshot mode broadcasting
417 */
418static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
419{
420	struct tick_device *td;
421	cpumask_t mask;
422	ktime_t now;
423	int cpu;
424
425	spin_lock(&tick_broadcast_lock);
426again:
427	dev->next_event.tv64 = KTIME_MAX;
428	mask = CPU_MASK_NONE;
429	now = ktime_get();
430	/* Find all expired events */
431	for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
432	     cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
433		td = &per_cpu(tick_cpu_device, cpu);
434		if (td->evtdev->next_event.tv64 <= now.tv64)
435			cpu_set(cpu, mask);
436	}
437
438	/*
439	 * Wakeup the cpus which have an expired event. The broadcast
440	 * device is reprogrammed in the return from idle code.
441	 */
442	if (!tick_do_broadcast(mask)) {
443		/*
444		 * The global event did not expire any CPU local
445		 * events. This happens in dyntick mode, as the
446		 * maximum PIT delta is quite small.
447		 */
448		if (tick_broadcast_reprogram())
449			goto again;
450	}
451	spin_unlock(&tick_broadcast_lock);
452}
453
454/*
455 * Powerstate information: The system enters/leaves a state, where
456 * affected devices might stop
457 */
458void tick_broadcast_oneshot_control(unsigned long reason)
459{
460	struct clock_event_device *bc, *dev;
461	struct tick_device *td;
462	unsigned long flags;
463	int cpu;
464
465	spin_lock_irqsave(&tick_broadcast_lock, flags);
466
467	/*
468	 * Periodic mode does not care about the enter/exit of power
469	 * states
470	 */
471	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
472		goto out;
473
474	bc = tick_broadcast_device.evtdev;
475	cpu = smp_processor_id();
476	td = &per_cpu(tick_cpu_device, cpu);
477	dev = td->evtdev;
478
479	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
480		goto out;
481
482	if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
483		if (!cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
484			cpu_set(cpu, tick_broadcast_oneshot_mask);
485			clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
486			if (dev->next_event.tv64 < bc->next_event.tv64)
487				tick_broadcast_set_event(dev->next_event, 1);
488		}
489	} else {
490		if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
491			cpu_clear(cpu, tick_broadcast_oneshot_mask);
492			clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
493			if (dev->next_event.tv64 != KTIME_MAX)
494				tick_program_event(dev->next_event, 1);
495		}
496	}
497
498out:
499	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
500}
501
502/*
503 * Reset the one shot broadcast for a cpu
504 *
505 * Called with tick_broadcast_lock held
506 */
507static void tick_broadcast_clear_oneshot(int cpu)
508{
509	cpu_clear(cpu, tick_broadcast_oneshot_mask);
510}
511
512/**
513 * tick_broadcast_setup_highres - setup the broadcast device for highres
514 */
515void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
516{
517	bc->event_handler = tick_handle_oneshot_broadcast;
518	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
519	bc->next_event.tv64 = KTIME_MAX;
520}
521
522/*
523 * Select oneshot operating mode for the broadcast device
524 */
525void tick_broadcast_switch_to_oneshot(void)
526{
527	struct clock_event_device *bc;
528	unsigned long flags;
529
530	spin_lock_irqsave(&tick_broadcast_lock, flags);
531
532	tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
533	bc = tick_broadcast_device.evtdev;
534	if (bc)
535		tick_broadcast_setup_oneshot(bc);
536	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
537}
538
539
540/*
541 * Remove a dead CPU from broadcasting
542 */
543void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
544{
545	unsigned long flags;
546	unsigned int cpu = *cpup;
547
548	spin_lock_irqsave(&tick_broadcast_lock, flags);
549
550	/*
551	 * Clear the broadcast mask flag for the dead cpu, but do not
552	 * stop the broadcast device!
553	 */
554	cpu_clear(cpu, tick_broadcast_oneshot_mask);
555
556	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
557}
558
559#endif
560