tick-broadcast.c revision 1fb9b7d29d8e85ba3196eaa7ab871bf76fc98d36 
1/*
2 * linux/kernel/time/tick-broadcast.c
3 *
4 * This file contains functions which emulate a local clock-event
5 * device via a broadcast event source.
6 *
7 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
8 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
9 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
10 *
11 * This code is licenced under the GPL version 2. For details see
12 * kernel-base/COPYING.
13 */
14#include <linux/cpu.h>
15#include <linux/err.h>
16#include <linux/hrtimer.h>
17#include <linux/interrupt.h>
18#include <linux/percpu.h>
19#include <linux/profile.h>
20#include <linux/sched.h>
21#include <linux/tick.h>
22
23#include "tick-internal.h"
24
25/*
26 * Broadcast support for broken x86 hardware, where the local apic
27 * timer stops in C3 state.
28 */
29
30struct tick_device tick_broadcast_device;
31static cpumask_t tick_broadcast_mask;
32static DEFINE_SPINLOCK(tick_broadcast_lock);
33static int tick_broadcast_force;
34
35#ifdef CONFIG_TICK_ONESHOT
36static void tick_broadcast_clear_oneshot(int cpu);
37#else
38static inline void tick_broadcast_clear_oneshot(int cpu) { }
39#endif
40
41/*
42 * Debugging: see timer_list.c
43 */
44struct tick_device *tick_get_broadcast_device(void)
45{
46	return &tick_broadcast_device;
47}
48
49cpumask_t *tick_get_broadcast_mask(void)
50{
51	return &tick_broadcast_mask;
52}
53
54/*
55 * Start the device in periodic mode
56 */
57static void tick_broadcast_start_periodic(struct clock_event_device *bc)
58{
59	if (bc)
60		tick_setup_periodic(bc, 1);
61}
62
63/*
64 * Check, if the device can be utilized as broadcast device:
65 */
66int tick_check_broadcast_device(struct clock_event_device *dev)
67{
68	if ((tick_broadcast_device.evtdev &&
69	     tick_broadcast_device.evtdev->rating >= dev->rating) ||
70	     (dev->features & CLOCK_EVT_FEAT_C3STOP))
71		return 0;
72
73	clockevents_exchange_device(NULL, dev);
74	tick_broadcast_device.evtdev = dev;
75	if (!cpus_empty(tick_broadcast_mask))
76		tick_broadcast_start_periodic(dev);
77	return 1;
78}
79
80/*
81 * Check, if the device is the broadcast device
82 */
83int tick_is_broadcast_device(struct clock_event_device *dev)
84{
85	return (dev && tick_broadcast_device.evtdev == dev);
86}
87
88/*
89 * Check, if the device is disfunctional and a place holder, which
90 * needs to be handled by the broadcast device.
91 */
92int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
93{
94	unsigned long flags;
95	int ret = 0;
96
97	spin_lock_irqsave(&tick_broadcast_lock, flags);
98
99	/*
100	 * Devices might be registered with both periodic and oneshot
101	 * mode disabled. This signals, that the device needs to be
102	 * operated from the broadcast device and is a placeholder for
103	 * the cpu local device.
104	 */
105	if (!tick_device_is_functional(dev)) {
106		dev->event_handler = tick_handle_periodic;
107		cpu_set(cpu, tick_broadcast_mask);
108		tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
109		ret = 1;
110	} else {
111		/*
112		 * When the new device is not affected by the stop
113		 * feature and the cpu is marked in the broadcast mask
114		 * then clear the broadcast bit.
115		 */
116		if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
117			int cpu = smp_processor_id();
118
119			cpu_clear(cpu, tick_broadcast_mask);
120			tick_broadcast_clear_oneshot(cpu);
121		}
122	}
123	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
124	return ret;
125}
126
127/*
128 * Broadcast the event to the cpus, which are set in the mask
129 */
130static void tick_do_broadcast(cpumask_t mask)
131{
132	int cpu = smp_processor_id();
133	struct tick_device *td;
134
135	/*
136	 * Check, if the current cpu is in the mask
137	 */
138	if (cpu_isset(cpu, mask)) {
139		cpu_clear(cpu, mask);
140		td = &per_cpu(tick_cpu_device, cpu);
141		td->evtdev->event_handler(td->evtdev);
142	}
143
144	if (!cpus_empty(mask)) {
145		/*
146		 * It might be necessary to actually check whether the devices
147		 * have different broadcast functions. For now, just use the
148		 * one of the first device. This works as long as we have this
149		 * misfeature only on x86 (lapic)
150		 */
151		cpu = first_cpu(mask);
152		td = &per_cpu(tick_cpu_device, cpu);
153		td->evtdev->broadcast(mask);
154	}
155}
156
157/*
158 * Periodic broadcast:
159 * - invoke the broadcast handlers
160 */
161static void tick_do_periodic_broadcast(void)
162{
163	cpumask_t mask;
164
165	spin_lock(&tick_broadcast_lock);
166
167	cpus_and(mask, cpu_online_map, tick_broadcast_mask);
168	tick_do_broadcast(mask);
169
170	spin_unlock(&tick_broadcast_lock);
171}
172
173/*
174 * Event handler for periodic broadcast ticks
175 */
176static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
177{
178	ktime_t next;
179
180	tick_do_periodic_broadcast();
181
182	/*
183	 * The device is in periodic mode. No reprogramming necessary:
184	 */
185	if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
186		return;
187
188	/*
189	 * Setup the next period for devices, which do not have
190	 * periodic mode. We read dev->next_event first and add to it
191	 * when the event alrady expired. clockevents_program_event()
192	 * sets dev->next_event only when the event is really
193	 * programmed to the device.
194	 */
195	for (next = dev->next_event; ;) {
196		next = ktime_add(next, tick_period);
197
198		if (!clockevents_program_event(dev, next, ktime_get()))
199			return;
200		tick_do_periodic_broadcast();
201	}
202}
203
204/*
205 * Powerstate information: The system enters/leaves a state, where
206 * affected devices might stop
207 */
208static void tick_do_broadcast_on_off(void *why)
209{
210	struct clock_event_device *bc, *dev;
211	struct tick_device *td;
212	unsigned long flags, *reason = why;
213	int cpu, bc_stopped;
214
215	spin_lock_irqsave(&tick_broadcast_lock, flags);
216
217	cpu = smp_processor_id();
218	td = &per_cpu(tick_cpu_device, cpu);
219	dev = td->evtdev;
220	bc = tick_broadcast_device.evtdev;
221
222	/*
223	 * Is the device not affected by the powerstate ?
224	 */
225	if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
226		goto out;
227
228	if (!tick_device_is_functional(dev))
229		goto out;
230
231	bc_stopped = cpus_empty(tick_broadcast_mask);
232
233	switch (*reason) {
234	case CLOCK_EVT_NOTIFY_BROADCAST_ON:
235	case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
236		if (!cpu_isset(cpu, tick_broadcast_mask)) {
237			cpu_set(cpu, tick_broadcast_mask);
238			if (td->mode == TICKDEV_MODE_PERIODIC)
239				clockevents_set_mode(dev,
240						     CLOCK_EVT_MODE_SHUTDOWN);
241		}
242		if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
243			tick_broadcast_force = 1;
244		break;
245	case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
246		if (!tick_broadcast_force &&
247		    cpu_isset(cpu, tick_broadcast_mask)) {
248			cpu_clear(cpu, tick_broadcast_mask);
249			if (td->mode == TICKDEV_MODE_PERIODIC)
250				tick_setup_periodic(dev, 0);
251		}
252		break;
253	}
254
255	if (cpus_empty(tick_broadcast_mask)) {
256		if (!bc_stopped)
257			clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
258	} else if (bc_stopped) {
259		if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
260			tick_broadcast_start_periodic(bc);
261		else
262			tick_broadcast_setup_oneshot(bc);
263	}
264out:
265	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
266}
267
268/*
269 * Powerstate information: The system enters/leaves a state, where
270 * affected devices might stop.
271 */
272void tick_broadcast_on_off(unsigned long reason, int *oncpu)
273{
274	if (!cpu_isset(*oncpu, cpu_online_map))
275		printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
276		       "offline CPU #%d\n", *oncpu);
277	else
278		smp_call_function_single(*oncpu, tick_do_broadcast_on_off,
279					 &reason, 1);
280}
281
282/*
283 * Set the periodic handler depending on broadcast on/off
284 */
285void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
286{
287	if (!broadcast)
288		dev->event_handler = tick_handle_periodic;
289	else
290		dev->event_handler = tick_handle_periodic_broadcast;
291}
292
293/*
294 * Remove a CPU from broadcasting
295 */
296void tick_shutdown_broadcast(unsigned int *cpup)
297{
298	struct clock_event_device *bc;
299	unsigned long flags;
300	unsigned int cpu = *cpup;
301
302	spin_lock_irqsave(&tick_broadcast_lock, flags);
303
304	bc = tick_broadcast_device.evtdev;
305	cpu_clear(cpu, tick_broadcast_mask);
306
307	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
308		if (bc && cpus_empty(tick_broadcast_mask))
309			clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
310	}
311
312	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
313}
314
315void tick_suspend_broadcast(void)
316{
317	struct clock_event_device *bc;
318	unsigned long flags;
319
320	spin_lock_irqsave(&tick_broadcast_lock, flags);
321
322	bc = tick_broadcast_device.evtdev;
323	if (bc)
324		clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
325
326	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
327}
328
329int tick_resume_broadcast(void)
330{
331	struct clock_event_device *bc;
332	unsigned long flags;
333	int broadcast = 0;
334
335	spin_lock_irqsave(&tick_broadcast_lock, flags);
336
337	bc = tick_broadcast_device.evtdev;
338
339	if (bc) {
340		clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);
341
342		switch (tick_broadcast_device.mode) {
343		case TICKDEV_MODE_PERIODIC:
344			if(!cpus_empty(tick_broadcast_mask))
345				tick_broadcast_start_periodic(bc);
346			broadcast = cpu_isset(smp_processor_id(),
347					      tick_broadcast_mask);
348			break;
349		case TICKDEV_MODE_ONESHOT:
350			broadcast = tick_resume_broadcast_oneshot(bc);
351			break;
352		}
353	}
354	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
355
356	return broadcast;
357}
358
359
360#ifdef CONFIG_TICK_ONESHOT
361
362static cpumask_t tick_broadcast_oneshot_mask;
363
364/*
365 * Debugging: see timer_list.c
366 */
367cpumask_t *tick_get_broadcast_oneshot_mask(void)
368{
369	return &tick_broadcast_oneshot_mask;
370}
371
372static int tick_broadcast_set_event(ktime_t expires, int force)
373{
374	struct clock_event_device *bc = tick_broadcast_device.evtdev;
375
376	return tick_dev_program_event(bc, expires, force);
377}
378
379int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
380{
381	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
382	return 0;
383}
384
385/*
386 * Handle oneshot mode broadcasting
387 */
388static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
389{
390	struct tick_device *td;
391	cpumask_t mask;
392	ktime_t now, next_event;
393	int cpu;
394
395	spin_lock(&tick_broadcast_lock);
396again:
397	dev->next_event.tv64 = KTIME_MAX;
398	next_event.tv64 = KTIME_MAX;
399	mask = CPU_MASK_NONE;
400	now = ktime_get();
401	/* Find all expired events */
402	for_each_cpu_mask_nr(cpu, tick_broadcast_oneshot_mask) {
403		td = &per_cpu(tick_cpu_device, cpu);
404		if (td->evtdev->next_event.tv64 <= now.tv64)
405			cpu_set(cpu, mask);
406		else if (td->evtdev->next_event.tv64 < next_event.tv64)
407			next_event.tv64 = td->evtdev->next_event.tv64;
408	}
409
410	/*
411	 * Wakeup the cpus which have an expired event.
412	 */
413	tick_do_broadcast(mask);
414
415	/*
416	 * Two reasons for reprogram:
417	 *
418	 * - The global event did not expire any CPU local
419	 * events. This happens in dyntick mode, as the maximum PIT
420	 * delta is quite small.
421	 *
422	 * - There are pending events on sleeping CPUs which were not
423	 * in the event mask
424	 */
425	if (next_event.tv64 != KTIME_MAX) {
426		/*
427		 * Rearm the broadcast device. If event expired,
428		 * repeat the above
429		 */
430		if (tick_broadcast_set_event(next_event, 0))
431			goto again;
432	}
433	spin_unlock(&tick_broadcast_lock);
434}
435
436/*
437 * Powerstate information: The system enters/leaves a state, where
438 * affected devices might stop
439 */
440void tick_broadcast_oneshot_control(unsigned long reason)
441{
442	struct clock_event_device *bc, *dev;
443	struct tick_device *td;
444	unsigned long flags;
445	int cpu;
446
447	spin_lock_irqsave(&tick_broadcast_lock, flags);
448
449	/*
450	 * Periodic mode does not care about the enter/exit of power
451	 * states
452	 */
453	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
454		goto out;
455
456	bc = tick_broadcast_device.evtdev;
457	cpu = smp_processor_id();
458	td = &per_cpu(tick_cpu_device, cpu);
459	dev = td->evtdev;
460
461	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
462		goto out;
463
464	if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
465		if (!cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
466			cpu_set(cpu, tick_broadcast_oneshot_mask);
467			clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
468			if (dev->next_event.tv64 < bc->next_event.tv64)
469				tick_broadcast_set_event(dev->next_event, 1);
470		}
471	} else {
472		if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
473			cpu_clear(cpu, tick_broadcast_oneshot_mask);
474			clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
475			if (dev->next_event.tv64 != KTIME_MAX)
476				tick_program_event(dev->next_event, 1);
477		}
478	}
479
480out:
481	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
482}
483
484/*
485 * Reset the one shot broadcast for a cpu
486 *
487 * Called with tick_broadcast_lock held
488 */
489static void tick_broadcast_clear_oneshot(int cpu)
490{
491	cpu_clear(cpu, tick_broadcast_oneshot_mask);
492}
493
494/**
495 * tick_broadcast_setup_oneshot - setup the broadcast device
496 */
497void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
498{
499	/* Set it up only once ! */
500	if (bc->event_handler != tick_handle_oneshot_broadcast) {
501		bc->event_handler = tick_handle_oneshot_broadcast;
502		clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
503		bc->next_event.tv64 = KTIME_MAX;
504	}
505}
506
507/*
508 * Select oneshot operating mode for the broadcast device
509 */
510void tick_broadcast_switch_to_oneshot(void)
511{
512	struct clock_event_device *bc;
513	unsigned long flags;
514
515	spin_lock_irqsave(&tick_broadcast_lock, flags);
516
517	tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
518	bc = tick_broadcast_device.evtdev;
519	if (bc)
520		tick_broadcast_setup_oneshot(bc);
521	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
522}
523
524
525/*
526 * Remove a dead CPU from broadcasting
527 */
528void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
529{
530	unsigned long flags;
531	unsigned int cpu = *cpup;
532
533	spin_lock_irqsave(&tick_broadcast_lock, flags);
534
535	/*
536	 * Clear the broadcast mask flag for the dead cpu, but do not
537	 * stop the broadcast device!
538	 */
539	cpu_clear(cpu, tick_broadcast_oneshot_mask);
540
541	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
542}
543
544#endif
545