tick-broadcast.c revision 302745699c1b675b5d2a1af87271de10e4d96b6a 
1/*
2 * linux/kernel/time/tick-broadcast.c
3 *
4 * This file contains functions which emulate a local clock-event
5 * device via a broadcast event source.
6 *
7 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
8 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
9 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
10 *
11 * This code is licenced under the GPL version 2. For details see
12 * kernel-base/COPYING.
13 */
14#include <linux/cpu.h>
15#include <linux/err.h>
16#include <linux/hrtimer.h>
17#include <linux/interrupt.h>
18#include <linux/percpu.h>
19#include <linux/profile.h>
20#include <linux/sched.h>
21#include <linux/tick.h>
22
23#include "tick-internal.h"
24
25/*
26 * Broadcast support for broken x86 hardware, where the local apic
27 * timer stops in C3 state.
28 */
29
30struct tick_device tick_broadcast_device;
31static cpumask_t tick_broadcast_mask;
32static DEFINE_SPINLOCK(tick_broadcast_lock);
33static int tick_broadcast_force;
34
35#ifdef CONFIG_TICK_ONESHOT
36static void tick_broadcast_clear_oneshot(int cpu);
37#else
38static inline void tick_broadcast_clear_oneshot(int cpu) { }
39#endif
40
41/*
42 * Debugging: see timer_list.c
43 */
44struct tick_device *tick_get_broadcast_device(void)
45{
46	return &tick_broadcast_device;
47}
48
49cpumask_t *tick_get_broadcast_mask(void)
50{
51	return &tick_broadcast_mask;
52}
53
54/*
55 * Start the device in periodic mode
56 */
57static void tick_broadcast_start_periodic(struct clock_event_device *bc)
58{
59	if (bc)
60		tick_setup_periodic(bc, 1);
61}
62
63/*
64 * Check, if the device can be utilized as broadcast device:
65 */
66int tick_check_broadcast_device(struct clock_event_device *dev)
67{
68	if ((tick_broadcast_device.evtdev &&
69	     tick_broadcast_device.evtdev->rating >= dev->rating) ||
70	     (dev->features & CLOCK_EVT_FEAT_C3STOP))
71		return 0;
72
73	clockevents_exchange_device(NULL, dev);
74	tick_broadcast_device.evtdev = dev;
75	if (!cpus_empty(tick_broadcast_mask))
76		tick_broadcast_start_periodic(dev);
77	return 1;
78}
79
80/*
81 * Check, if the device is the broadcast device
82 */
83int tick_is_broadcast_device(struct clock_event_device *dev)
84{
85	return (dev && tick_broadcast_device.evtdev == dev);
86}
87
88/*
89 * Check, if the device is disfunctional and a place holder, which
90 * needs to be handled by the broadcast device.
91 */
92int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
93{
94	unsigned long flags;
95	int ret = 0;
96
97	spin_lock_irqsave(&tick_broadcast_lock, flags);
98
99	/*
100	 * Devices might be registered with both periodic and oneshot
101	 * mode disabled. This signals, that the device needs to be
102	 * operated from the broadcast device and is a placeholder for
103	 * the cpu local device.
104	 */
105	if (!tick_device_is_functional(dev)) {
106		dev->event_handler = tick_handle_periodic;
107		cpu_set(cpu, tick_broadcast_mask);
108		tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
109		ret = 1;
110	} else {
111		/*
112		 * When the new device is not affected by the stop
113		 * feature and the cpu is marked in the broadcast mask
114		 * then clear the broadcast bit.
115		 */
116		if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
117			int cpu = smp_processor_id();
118
119			cpu_clear(cpu, tick_broadcast_mask);
120			tick_broadcast_clear_oneshot(cpu);
121		}
122	}
123	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
124	return ret;
125}
126
127/*
128 * Broadcast the event to the cpus, which are set in the mask
129 */
130static void tick_do_broadcast(cpumask_t mask)
131{
132	int cpu = smp_processor_id();
133	struct tick_device *td;
134
135	/*
136	 * Check, if the current cpu is in the mask
137	 */
138	if (cpu_isset(cpu, mask)) {
139		cpu_clear(cpu, mask);
140		td = &per_cpu(tick_cpu_device, cpu);
141		td->evtdev->event_handler(td->evtdev);
142	}
143
144	if (!cpus_empty(mask)) {
145		/*
146		 * It might be necessary to actually check whether the devices
147		 * have different broadcast functions. For now, just use the
148		 * one of the first device. This works as long as we have this
149		 * misfeature only on x86 (lapic)
150		 */
151		cpu = first_cpu(mask);
152		td = &per_cpu(tick_cpu_device, cpu);
153		td->evtdev->broadcast(mask);
154	}
155}
156
157/*
158 * Periodic broadcast:
159 * - invoke the broadcast handlers
160 */
161static void tick_do_periodic_broadcast(void)
162{
163	cpumask_t mask;
164
165	spin_lock(&tick_broadcast_lock);
166
167	cpus_and(mask, cpu_online_map, tick_broadcast_mask);
168	tick_do_broadcast(mask);
169
170	spin_unlock(&tick_broadcast_lock);
171}
172
173/*
174 * Event handler for periodic broadcast ticks
175 */
176static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
177{
178	ktime_t next;
179
180	tick_do_periodic_broadcast();
181
182	/*
183	 * The device is in periodic mode. No reprogramming necessary:
184	 */
185	if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
186		return;
187
188	/*
189	 * Setup the next period for devices, which do not have
190	 * periodic mode. We read dev->next_event first and add to it
191	 * when the event alrady expired. clockevents_program_event()
192	 * sets dev->next_event only when the event is really
193	 * programmed to the device.
194	 */
195	for (next = dev->next_event; ;) {
196		next = ktime_add(next, tick_period);
197
198		if (!clockevents_program_event(dev, next, ktime_get()))
199			return;
200		tick_do_periodic_broadcast();
201	}
202}
203
204/*
205 * Powerstate information: The system enters/leaves a state, where
206 * affected devices might stop
207 */
208static void tick_do_broadcast_on_off(void *why)
209{
210	struct clock_event_device *bc, *dev;
211	struct tick_device *td;
212	unsigned long flags, *reason = why;
213	int cpu, bc_stopped;
214
215	spin_lock_irqsave(&tick_broadcast_lock, flags);
216
217	cpu = smp_processor_id();
218	td = &per_cpu(tick_cpu_device, cpu);
219	dev = td->evtdev;
220	bc = tick_broadcast_device.evtdev;
221
222	/*
223	 * Is the device not affected by the powerstate ?
224	 */
225	if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
226		goto out;
227
228	if (!tick_device_is_functional(dev))
229		goto out;
230
231	bc_stopped = cpus_empty(tick_broadcast_mask);
232
233	switch (*reason) {
234	case CLOCK_EVT_NOTIFY_BROADCAST_ON:
235	case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
236		if (!cpu_isset(cpu, tick_broadcast_mask)) {
237			cpu_set(cpu, tick_broadcast_mask);
238			if (bc->mode == TICKDEV_MODE_PERIODIC)
239				clockevents_shutdown(dev);
240		}
241		if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
242			tick_broadcast_force = 1;
243		break;
244	case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
245		if (!tick_broadcast_force &&
246		    cpu_isset(cpu, tick_broadcast_mask)) {
247			cpu_clear(cpu, tick_broadcast_mask);
248			if (bc->mode == TICKDEV_MODE_PERIODIC)
249				tick_setup_periodic(dev, 0);
250		}
251		break;
252	}
253
254	if (cpus_empty(tick_broadcast_mask)) {
255		if (!bc_stopped)
256			clockevents_shutdown(bc);
257	} else if (bc_stopped) {
258		if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
259			tick_broadcast_start_periodic(bc);
260		else
261			tick_broadcast_setup_oneshot(bc);
262	}
263out:
264	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
265}
266
267/*
268 * Powerstate information: The system enters/leaves a state, where
269 * affected devices might stop.
270 */
271void tick_broadcast_on_off(unsigned long reason, int *oncpu)
272{
273	if (!cpu_isset(*oncpu, cpu_online_map))
274		printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
275		       "offline CPU #%d\n", *oncpu);
276	else
277		smp_call_function_single(*oncpu, tick_do_broadcast_on_off,
278					 &reason, 1);
279}
280
281/*
282 * Set the periodic handler depending on broadcast on/off
283 */
284void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
285{
286	if (!broadcast)
287		dev->event_handler = tick_handle_periodic;
288	else
289		dev->event_handler = tick_handle_periodic_broadcast;
290}
291
292/*
293 * Remove a CPU from broadcasting
294 */
295void tick_shutdown_broadcast(unsigned int *cpup)
296{
297	struct clock_event_device *bc;
298	unsigned long flags;
299	unsigned int cpu = *cpup;
300
301	spin_lock_irqsave(&tick_broadcast_lock, flags);
302
303	bc = tick_broadcast_device.evtdev;
304	cpu_clear(cpu, tick_broadcast_mask);
305
306	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
307		if (bc && cpus_empty(tick_broadcast_mask))
308			clockevents_shutdown(bc);
309	}
310
311	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
312}
313
314void tick_suspend_broadcast(void)
315{
316	struct clock_event_device *bc;
317	unsigned long flags;
318
319	spin_lock_irqsave(&tick_broadcast_lock, flags);
320
321	bc = tick_broadcast_device.evtdev;
322	if (bc)
323		clockevents_shutdown(bc);
324
325	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
326}
327
328int tick_resume_broadcast(void)
329{
330	struct clock_event_device *bc;
331	unsigned long flags;
332	int broadcast = 0;
333
334	spin_lock_irqsave(&tick_broadcast_lock, flags);
335
336	bc = tick_broadcast_device.evtdev;
337
338	if (bc) {
339		clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);
340
341		switch (tick_broadcast_device.mode) {
342		case TICKDEV_MODE_PERIODIC:
343			if(!cpus_empty(tick_broadcast_mask))
344				tick_broadcast_start_periodic(bc);
345			broadcast = cpu_isset(smp_processor_id(),
346					      tick_broadcast_mask);
347			break;
348		case TICKDEV_MODE_ONESHOT:
349			broadcast = tick_resume_broadcast_oneshot(bc);
350			break;
351		}
352	}
353	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
354
355	return broadcast;
356}
357
358
359#ifdef CONFIG_TICK_ONESHOT
360
361static cpumask_t tick_broadcast_oneshot_mask;
362
363/*
364 * Debugging: see timer_list.c
365 */
366cpumask_t *tick_get_broadcast_oneshot_mask(void)
367{
368	return &tick_broadcast_oneshot_mask;
369}
370
371static int tick_broadcast_set_event(ktime_t expires, int force)
372{
373	struct clock_event_device *bc = tick_broadcast_device.evtdev;
374
375	return tick_dev_program_event(bc, expires, force);
376}
377
378int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
379{
380	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
381	return 0;
382}
383
384/*
385 * Handle oneshot mode broadcasting
386 */
387static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
388{
389	struct tick_device *td;
390	cpumask_t mask;
391	ktime_t now, next_event;
392	int cpu;
393
394	spin_lock(&tick_broadcast_lock);
395again:
396	dev->next_event.tv64 = KTIME_MAX;
397	next_event.tv64 = KTIME_MAX;
398	mask = CPU_MASK_NONE;
399	now = ktime_get();
400	/* Find all expired events */
401	for_each_cpu_mask_nr(cpu, tick_broadcast_oneshot_mask) {
402		td = &per_cpu(tick_cpu_device, cpu);
403		if (td->evtdev->next_event.tv64 <= now.tv64)
404			cpu_set(cpu, mask);
405		else if (td->evtdev->next_event.tv64 < next_event.tv64)
406			next_event.tv64 = td->evtdev->next_event.tv64;
407	}
408
409	/*
410	 * Wakeup the cpus which have an expired event.
411	 */
412	tick_do_broadcast(mask);
413
414	/*
415	 * Two reasons for reprogram:
416	 *
417	 * - The global event did not expire any CPU local
418	 * events. This happens in dyntick mode, as the maximum PIT
419	 * delta is quite small.
420	 *
421	 * - There are pending events on sleeping CPUs which were not
422	 * in the event mask
423	 */
424	if (next_event.tv64 != KTIME_MAX) {
425		/*
426		 * Rearm the broadcast device. If event expired,
427		 * repeat the above
428		 */
429		if (tick_broadcast_set_event(next_event, 0))
430			goto again;
431	}
432	spin_unlock(&tick_broadcast_lock);
433}
434
435/*
436 * Powerstate information: The system enters/leaves a state, where
437 * affected devices might stop
438 */
439void tick_broadcast_oneshot_control(unsigned long reason)
440{
441	struct clock_event_device *bc, *dev;
442	struct tick_device *td;
443	unsigned long flags;
444	int cpu;
445
446	spin_lock_irqsave(&tick_broadcast_lock, flags);
447
448	/*
449	 * Periodic mode does not care about the enter/exit of power
450	 * states
451	 */
452	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
453		goto out;
454
455	bc = tick_broadcast_device.evtdev;
456	cpu = smp_processor_id();
457	td = &per_cpu(tick_cpu_device, cpu);
458	dev = td->evtdev;
459
460	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
461		goto out;
462
463	if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
464		if (!cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
465			cpu_set(cpu, tick_broadcast_oneshot_mask);
466			clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
467			if (dev->next_event.tv64 < bc->next_event.tv64)
468				tick_broadcast_set_event(dev->next_event, 1);
469		}
470	} else {
471		if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
472			cpu_clear(cpu, tick_broadcast_oneshot_mask);
473			clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
474			if (dev->next_event.tv64 != KTIME_MAX)
475				tick_program_event(dev->next_event, 1);
476		}
477	}
478
479out:
480	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
481}
482
483/*
484 * Reset the one shot broadcast for a cpu
485 *
486 * Called with tick_broadcast_lock held
487 */
488static void tick_broadcast_clear_oneshot(int cpu)
489{
490	cpu_clear(cpu, tick_broadcast_oneshot_mask);
491}
492
493static void tick_broadcast_init_next_event(cpumask_t *mask, ktime_t expires)
494{
495	struct tick_device *td;
496	int cpu;
497
498	for_each_cpu_mask_nr(cpu, *mask) {
499		td = &per_cpu(tick_cpu_device, cpu);
500		if (td->evtdev)
501			td->evtdev->next_event = expires;
502	}
503}
504
505/**
506 * tick_broadcast_setup_oneshot - setup the broadcast device
507 */
508void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
509{
510	/* Set it up only once ! */
511	if (bc->event_handler != tick_handle_oneshot_broadcast) {
512		int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC;
513		int cpu = smp_processor_id();
514		cpumask_t mask;
515
516		bc->event_handler = tick_handle_oneshot_broadcast;
517		clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
518
519		/* Take the do_timer update */
520		tick_do_timer_cpu = cpu;
521
522		/*
523		 * We must be careful here. There might be other CPUs
524		 * waiting for periodic broadcast. We need to set the
525		 * oneshot_mask bits for those and program the
526		 * broadcast device to fire.
527		 */
528		mask = tick_broadcast_mask;
529		cpu_clear(cpu, mask);
530		cpus_or(tick_broadcast_oneshot_mask,
531			tick_broadcast_oneshot_mask, mask);
532
533		if (was_periodic && !cpus_empty(mask)) {
534			tick_broadcast_init_next_event(&mask, tick_next_period);
535			tick_broadcast_set_event(tick_next_period, 1);
536		} else
537			bc->next_event.tv64 = KTIME_MAX;
538	}
539}
540
541/*
542 * Select oneshot operating mode for the broadcast device
543 */
544void tick_broadcast_switch_to_oneshot(void)
545{
546	struct clock_event_device *bc;
547	unsigned long flags;
548
549	spin_lock_irqsave(&tick_broadcast_lock, flags);
550
551	tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
552	bc = tick_broadcast_device.evtdev;
553	if (bc)
554		tick_broadcast_setup_oneshot(bc);
555	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
556}
557
558
559/*
560 * Remove a dead CPU from broadcasting
561 */
562void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
563{
564	unsigned long flags;
565	unsigned int cpu = *cpup;
566
567	spin_lock_irqsave(&tick_broadcast_lock, flags);
568
569	/*
570	 * Clear the broadcast mask flag for the dead cpu, but do not
571	 * stop the broadcast device!
572	 */
573	cpu_clear(cpu, tick_broadcast_oneshot_mask);
574
575	spin_unlock_irqrestore(&tick_broadcast_lock, flags);
576}
577
578#endif
579