tick-broadcast.c revision ea8deb8dfa6b0e8d1b3d1051585706739b46656c
1/*
2 * linux/kernel/time/tick-broadcast.c
3 *
4 * This file contains functions which emulate a local clock-event
5 * device via a broadcast event source.
6 *
7 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
8 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
9 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
10 *
11 * This code is licenced under the GPL version 2. For details see
12 * kernel-base/COPYING.
13 */
14#include <linux/cpu.h>
15#include <linux/err.h>
16#include <linux/hrtimer.h>
17#include <linux/interrupt.h>
18#include <linux/percpu.h>
19#include <linux/profile.h>
20#include <linux/sched.h>
21#include <linux/smp.h>
22
23#include "tick-internal.h"
24
25/*
26 * Broadcast support for broken x86 hardware, where the local apic
27 * timer stops in C3 state.
28 */
29
30static struct tick_device tick_broadcast_device;
31static cpumask_var_t tick_broadcast_mask;
32static cpumask_var_t tmpmask;
33static DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
34static int tick_broadcast_force;
35
36#ifdef CONFIG_TICK_ONESHOT
37static void tick_broadcast_clear_oneshot(int cpu);
38#else
39static inline void tick_broadcast_clear_oneshot(int cpu) { }
40#endif
41
42/*
43 * Debugging: see timer_list.c
44 */
45struct tick_device *tick_get_broadcast_device(void)
46{
47	return &tick_broadcast_device;
48}
49
50struct cpumask *tick_get_broadcast_mask(void)
51{
52	return tick_broadcast_mask;
53}
54
55/*
56 * Start the device in periodic mode
57 */
58static void tick_broadcast_start_periodic(struct clock_event_device *bc)
59{
60	if (bc)
61		tick_setup_periodic(bc, 1);
62}
63
64/*
65 * Check, if the device can be utilized as broadcast device:
66 */
67int tick_check_broadcast_device(struct clock_event_device *dev)
68{
69	struct clock_event_device *cur = tick_broadcast_device.evtdev;
70
71	if ((dev->features & CLOCK_EVT_FEAT_DUMMY) ||
72	    (tick_broadcast_device.evtdev &&
73	     tick_broadcast_device.evtdev->rating >= dev->rating) ||
74	     (dev->features & CLOCK_EVT_FEAT_C3STOP))
75		return 0;
76
77	clockevents_exchange_device(tick_broadcast_device.evtdev, dev);
78	if (cur)
79		cur->event_handler = clockevents_handle_noop;
80	tick_broadcast_device.evtdev = dev;
81	if (!cpumask_empty(tick_broadcast_mask))
82		tick_broadcast_start_periodic(dev);
83	/*
84	 * Inform all cpus about this. We might be in a situation
85	 * where we did not switch to oneshot mode because the per cpu
86	 * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack
87	 * of a oneshot capable broadcast device. Without that
88	 * notification the systems stays stuck in periodic mode
89	 * forever.
90	 */
91	if (dev->features & CLOCK_EVT_FEAT_ONESHOT)
92		tick_clock_notify();
93	return 1;
94}
95
96/*
97 * Check, if the device is the broadcast device
98 */
99int tick_is_broadcast_device(struct clock_event_device *dev)
100{
101	return (dev && tick_broadcast_device.evtdev == dev);
102}
103
104static void err_broadcast(const struct cpumask *mask)
105{
106	pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
107}
108
109static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
110{
111	if (!dev->broadcast)
112		dev->broadcast = tick_broadcast;
113	if (!dev->broadcast) {
114		pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
115			     dev->name);
116		dev->broadcast = err_broadcast;
117	}
118}
119
120/*
121 * Check, if the device is disfunctional and a place holder, which
122 * needs to be handled by the broadcast device.
123 */
124int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
125{
126	unsigned long flags;
127	int ret = 0;
128
129	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
130
131	/*
132	 * Devices might be registered with both periodic and oneshot
133	 * mode disabled. This signals, that the device needs to be
134	 * operated from the broadcast device and is a placeholder for
135	 * the cpu local device.
136	 */
137	if (!tick_device_is_functional(dev)) {
138		dev->event_handler = tick_handle_periodic;
139		tick_device_setup_broadcast_func(dev);
140		cpumask_set_cpu(cpu, tick_broadcast_mask);
141		tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
142		ret = 1;
143	} else {
144		/*
145		 * When the new device is not affected by the stop
146		 * feature and the cpu is marked in the broadcast mask
147		 * then clear the broadcast bit.
148		 */
149		if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
150			int cpu = smp_processor_id();
151			cpumask_clear_cpu(cpu, tick_broadcast_mask);
152			tick_broadcast_clear_oneshot(cpu);
153		} else {
154			tick_device_setup_broadcast_func(dev);
155		}
156	}
157	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
158	return ret;
159}
160
161#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
162int tick_receive_broadcast(void)
163{
164	struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
165	struct clock_event_device *evt = td->evtdev;
166
167	if (!evt)
168		return -ENODEV;
169
170	if (!evt->event_handler)
171		return -EINVAL;
172
173	evt->event_handler(evt);
174	return 0;
175}
176#endif
177
178/*
179 * Broadcast the event to the cpus, which are set in the mask (mangled).
180 */
181static void tick_do_broadcast(struct cpumask *mask)
182{
183	int cpu = smp_processor_id();
184	struct tick_device *td;
185
186	/*
187	 * Check, if the current cpu is in the mask
188	 */
189	if (cpumask_test_cpu(cpu, mask)) {
190		cpumask_clear_cpu(cpu, mask);
191		td = &per_cpu(tick_cpu_device, cpu);
192		td->evtdev->event_handler(td->evtdev);
193	}
194
195	if (!cpumask_empty(mask)) {
196		/*
197		 * It might be necessary to actually check whether the devices
198		 * have different broadcast functions. For now, just use the
199		 * one of the first device. This works as long as we have this
200		 * misfeature only on x86 (lapic)
201		 */
202		td = &per_cpu(tick_cpu_device, cpumask_first(mask));
203		td->evtdev->broadcast(mask);
204	}
205}
206
207/*
208 * Periodic broadcast:
209 * - invoke the broadcast handlers
210 */
211static void tick_do_periodic_broadcast(void)
212{
213	raw_spin_lock(&tick_broadcast_lock);
214
215	cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
216	tick_do_broadcast(tmpmask);
217
218	raw_spin_unlock(&tick_broadcast_lock);
219}
220
221/*
222 * Event handler for periodic broadcast ticks
223 */
224static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
225{
226	ktime_t next;
227
228	tick_do_periodic_broadcast();
229
230	/*
231	 * The device is in periodic mode. No reprogramming necessary:
232	 */
233	if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
234		return;
235
236	/*
237	 * Setup the next period for devices, which do not have
238	 * periodic mode. We read dev->next_event first and add to it
239	 * when the event already expired. clockevents_program_event()
240	 * sets dev->next_event only when the event is really
241	 * programmed to the device.
242	 */
243	for (next = dev->next_event; ;) {
244		next = ktime_add(next, tick_period);
245
246		if (!clockevents_program_event(dev, next, false))
247			return;
248		tick_do_periodic_broadcast();
249	}
250}
251
252/*
253 * Powerstate information: The system enters/leaves a state, where
254 * affected devices might stop
255 */
256static void tick_do_broadcast_on_off(unsigned long *reason)
257{
258	struct clock_event_device *bc, *dev;
259	struct tick_device *td;
260	unsigned long flags;
261	int cpu, bc_stopped;
262
263	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
264
265	cpu = smp_processor_id();
266	td = &per_cpu(tick_cpu_device, cpu);
267	dev = td->evtdev;
268	bc = tick_broadcast_device.evtdev;
269
270	/*
271	 * Is the device not affected by the powerstate ?
272	 */
273	if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
274		goto out;
275
276	if (!tick_device_is_functional(dev))
277		goto out;
278
279	bc_stopped = cpumask_empty(tick_broadcast_mask);
280
281	switch (*reason) {
282	case CLOCK_EVT_NOTIFY_BROADCAST_ON:
283	case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
284		if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
285			if (tick_broadcast_device.mode ==
286			    TICKDEV_MODE_PERIODIC)
287				clockevents_shutdown(dev);
288		}
289		if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
290			tick_broadcast_force = 1;
291		break;
292	case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
293		if (!tick_broadcast_force &&
294		    cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
295			if (tick_broadcast_device.mode ==
296			    TICKDEV_MODE_PERIODIC)
297				tick_setup_periodic(dev, 0);
298		}
299		break;
300	}
301
302	if (cpumask_empty(tick_broadcast_mask)) {
303		if (!bc_stopped)
304			clockevents_shutdown(bc);
305	} else if (bc_stopped) {
306		if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
307			tick_broadcast_start_periodic(bc);
308		else
309			tick_broadcast_setup_oneshot(bc);
310	}
311out:
312	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
313}
314
315/*
316 * Powerstate information: The system enters/leaves a state, where
317 * affected devices might stop.
318 */
319void tick_broadcast_on_off(unsigned long reason, int *oncpu)
320{
321	if (!cpumask_test_cpu(*oncpu, cpu_online_mask))
322		printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
323		       "offline CPU #%d\n", *oncpu);
324	else
325		tick_do_broadcast_on_off(&reason);
326}
327
328/*
329 * Set the periodic handler depending on broadcast on/off
330 */
331void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
332{
333	if (!broadcast)
334		dev->event_handler = tick_handle_periodic;
335	else
336		dev->event_handler = tick_handle_periodic_broadcast;
337}
338
339/*
340 * Remove a CPU from broadcasting
341 */
342void tick_shutdown_broadcast(unsigned int *cpup)
343{
344	struct clock_event_device *bc;
345	unsigned long flags;
346	unsigned int cpu = *cpup;
347
348	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
349
350	bc = tick_broadcast_device.evtdev;
351	cpumask_clear_cpu(cpu, tick_broadcast_mask);
352
353	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
354		if (bc && cpumask_empty(tick_broadcast_mask))
355			clockevents_shutdown(bc);
356	}
357
358	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
359}
360
361void tick_suspend_broadcast(void)
362{
363	struct clock_event_device *bc;
364	unsigned long flags;
365
366	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
367
368	bc = tick_broadcast_device.evtdev;
369	if (bc)
370		clockevents_shutdown(bc);
371
372	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
373}
374
375int tick_resume_broadcast(void)
376{
377	struct clock_event_device *bc;
378	unsigned long flags;
379	int broadcast = 0;
380
381	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
382
383	bc = tick_broadcast_device.evtdev;
384
385	if (bc) {
386		clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);
387
388		switch (tick_broadcast_device.mode) {
389		case TICKDEV_MODE_PERIODIC:
390			if (!cpumask_empty(tick_broadcast_mask))
391				tick_broadcast_start_periodic(bc);
392			broadcast = cpumask_test_cpu(smp_processor_id(),
393						     tick_broadcast_mask);
394			break;
395		case TICKDEV_MODE_ONESHOT:
396			if (!cpumask_empty(tick_broadcast_mask))
397				broadcast = tick_resume_broadcast_oneshot(bc);
398			break;
399		}
400	}
401	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
402
403	return broadcast;
404}
405
406
407#ifdef CONFIG_TICK_ONESHOT
408
409static cpumask_var_t tick_broadcast_oneshot_mask;
410static cpumask_var_t tick_broadcast_pending_mask;
411static cpumask_var_t tick_broadcast_force_mask;
412
413/*
414 * Exposed for debugging: see timer_list.c
415 */
416struct cpumask *tick_get_broadcast_oneshot_mask(void)
417{
418	return tick_broadcast_oneshot_mask;
419}
420
421/*
422 * Called before going idle with interrupts disabled. Checks whether a
423 * broadcast event from the other core is about to happen. We detected
424 * that in tick_broadcast_oneshot_control(). The callsite can use this
425 * to avoid a deep idle transition as we are about to get the
426 * broadcast IPI right away.
427 */
428int tick_check_broadcast_expired(void)
429{
430	return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
431}
432
433/*
434 * Set broadcast interrupt affinity
435 */
436static void tick_broadcast_set_affinity(struct clock_event_device *bc,
437					const struct cpumask *cpumask)
438{
439	if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
440		return;
441
442	if (cpumask_equal(bc->cpumask, cpumask))
443		return;
444
445	bc->cpumask = cpumask;
446	irq_set_affinity(bc->irq, bc->cpumask);
447}
448
449static int tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
450				    ktime_t expires, int force)
451{
452	int ret;
453
454	if (bc->mode != CLOCK_EVT_MODE_ONESHOT)
455		clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
456
457	ret = clockevents_program_event(bc, expires, force);
458	if (!ret)
459		tick_broadcast_set_affinity(bc, cpumask_of(cpu));
460	return ret;
461}
462
463int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
464{
465	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
466	return 0;
467}
468
469/*
470 * Called from irq_enter() when idle was interrupted to reenable the
471 * per cpu device.
472 */
473void tick_check_oneshot_broadcast(int cpu)
474{
475	if (cpumask_test_cpu(cpu, tick_broadcast_oneshot_mask)) {
476		struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
477
478		clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT);
479	}
480}
481
482/*
483 * Handle oneshot mode broadcasting
484 */
485static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
486{
487	struct tick_device *td;
488	ktime_t now, next_event;
489	int cpu, next_cpu = 0;
490
491	raw_spin_lock(&tick_broadcast_lock);
492again:
493	dev->next_event.tv64 = KTIME_MAX;
494	next_event.tv64 = KTIME_MAX;
495	cpumask_clear(tmpmask);
496	now = ktime_get();
497	/* Find all expired events */
498	for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
499		td = &per_cpu(tick_cpu_device, cpu);
500		if (td->evtdev->next_event.tv64 <= now.tv64) {
501			cpumask_set_cpu(cpu, tmpmask);
502			/*
503			 * Mark the remote cpu in the pending mask, so
504			 * it can avoid reprogramming the cpu local
505			 * timer in tick_broadcast_oneshot_control().
506			 */
507			cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
508		} else if (td->evtdev->next_event.tv64 < next_event.tv64) {
509			next_event.tv64 = td->evtdev->next_event.tv64;
510			next_cpu = cpu;
511		}
512	}
513
514	/*
515	 * Remove the current cpu from the pending mask. The event is
516	 * delivered immediately in tick_do_broadcast() !
517	 */
518	cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);
519
520	/* Take care of enforced broadcast requests */
521	cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
522	cpumask_clear(tick_broadcast_force_mask);
523
524	/*
525	 * Wakeup the cpus which have an expired event.
526	 */
527	tick_do_broadcast(tmpmask);
528
529	/*
530	 * Two reasons for reprogram:
531	 *
532	 * - The global event did not expire any CPU local
533	 * events. This happens in dyntick mode, as the maximum PIT
534	 * delta is quite small.
535	 *
536	 * - There are pending events on sleeping CPUs which were not
537	 * in the event mask
538	 */
539	if (next_event.tv64 != KTIME_MAX) {
540		/*
541		 * Rearm the broadcast device. If event expired,
542		 * repeat the above
543		 */
544		if (tick_broadcast_set_event(dev, next_cpu, next_event, 0))
545			goto again;
546	}
547	raw_spin_unlock(&tick_broadcast_lock);
548}
549
550/*
551 * Powerstate information: The system enters/leaves a state, where
552 * affected devices might stop
553 */
554void tick_broadcast_oneshot_control(unsigned long reason)
555{
556	struct clock_event_device *bc, *dev;
557	struct tick_device *td;
558	unsigned long flags;
559	ktime_t now;
560	int cpu;
561
562	/*
563	 * Periodic mode does not care about the enter/exit of power
564	 * states
565	 */
566	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
567		return;
568
569	/*
570	 * We are called with preemtion disabled from the depth of the
571	 * idle code, so we can't be moved away.
572	 */
573	cpu = smp_processor_id();
574	td = &per_cpu(tick_cpu_device, cpu);
575	dev = td->evtdev;
576
577	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
578		return;
579
580	bc = tick_broadcast_device.evtdev;
581
582	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
583	if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
584		if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
585			WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));
586			clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
587			/*
588			 * We only reprogram the broadcast timer if we
589			 * did not mark ourself in the force mask and
590			 * if the cpu local event is earlier than the
591			 * broadcast event. If the current CPU is in
592			 * the force mask, then we are going to be
593			 * woken by the IPI right away.
594			 */
595			if (!cpumask_test_cpu(cpu, tick_broadcast_force_mask) &&
596			    dev->next_event.tv64 < bc->next_event.tv64)
597				tick_broadcast_set_event(bc, cpu, dev->next_event, 1);
598		}
599	} else {
600		if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
601			clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
602			/*
603			 * The cpu which was handling the broadcast
604			 * timer marked this cpu in the broadcast
605			 * pending mask and fired the broadcast
606			 * IPI. So we are going to handle the expired
607			 * event anyway via the broadcast IPI
608			 * handler. No need to reprogram the timer
609			 * with an already expired event.
610			 */
611			if (cpumask_test_and_clear_cpu(cpu,
612				       tick_broadcast_pending_mask))
613				goto out;
614
615			/*
616			 * Bail out if there is no next event.
617			 */
618			if (dev->next_event.tv64 == KTIME_MAX)
619				goto out;
620			/*
621			 * If the pending bit is not set, then we are
622			 * either the CPU handling the broadcast
623			 * interrupt or we got woken by something else.
624			 *
625			 * We are not longer in the broadcast mask, so
626			 * if the cpu local expiry time is already
627			 * reached, we would reprogram the cpu local
628			 * timer with an already expired event.
629			 *
630			 * This can lead to a ping-pong when we return
631			 * to idle and therefor rearm the broadcast
632			 * timer before the cpu local timer was able
633			 * to fire. This happens because the forced
634			 * reprogramming makes sure that the event
635			 * will happen in the future and depending on
636			 * the min_delta setting this might be far
637			 * enough out that the ping-pong starts.
638			 *
639			 * If the cpu local next_event has expired
640			 * then we know that the broadcast timer
641			 * next_event has expired as well and
642			 * broadcast is about to be handled. So we
643			 * avoid reprogramming and enforce that the
644			 * broadcast handler, which did not run yet,
645			 * will invoke the cpu local handler.
646			 *
647			 * We cannot call the handler directly from
648			 * here, because we might be in a NOHZ phase
649			 * and we did not go through the irq_enter()
650			 * nohz fixups.
651			 */
652			now = ktime_get();
653			if (dev->next_event.tv64 <= now.tv64) {
654				cpumask_set_cpu(cpu, tick_broadcast_force_mask);
655				goto out;
656			}
657			/*
658			 * We got woken by something else. Reprogram
659			 * the cpu local timer device.
660			 */
661			tick_program_event(dev->next_event, 1);
662		}
663	}
664out:
665	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
666}
667
668/*
669 * Reset the one shot broadcast for a cpu
670 *
671 * Called with tick_broadcast_lock held
672 */
673static void tick_broadcast_clear_oneshot(int cpu)
674{
675	cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
676}
677
678static void tick_broadcast_init_next_event(struct cpumask *mask,
679					   ktime_t expires)
680{
681	struct tick_device *td;
682	int cpu;
683
684	for_each_cpu(cpu, mask) {
685		td = &per_cpu(tick_cpu_device, cpu);
686		if (td->evtdev)
687			td->evtdev->next_event = expires;
688	}
689}
690
691/**
692 * tick_broadcast_setup_oneshot - setup the broadcast device
693 */
694void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
695{
696	int cpu = smp_processor_id();
697
698	/* Set it up only once ! */
699	if (bc->event_handler != tick_handle_oneshot_broadcast) {
700		int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC;
701
702		bc->event_handler = tick_handle_oneshot_broadcast;
703
704		/*
705		 * We must be careful here. There might be other CPUs
706		 * waiting for periodic broadcast. We need to set the
707		 * oneshot_mask bits for those and program the
708		 * broadcast device to fire.
709		 */
710		cpumask_copy(tmpmask, tick_broadcast_mask);
711		cpumask_clear_cpu(cpu, tmpmask);
712		cpumask_or(tick_broadcast_oneshot_mask,
713			   tick_broadcast_oneshot_mask, tmpmask);
714
715		if (was_periodic && !cpumask_empty(tmpmask)) {
716			clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
717			tick_broadcast_init_next_event(tmpmask,
718						       tick_next_period);
719			tick_broadcast_set_event(bc, cpu, tick_next_period, 1);
720		} else
721			bc->next_event.tv64 = KTIME_MAX;
722	} else {
723		/*
724		 * The first cpu which switches to oneshot mode sets
725		 * the bit for all other cpus which are in the general
726		 * (periodic) broadcast mask. So the bit is set and
727		 * would prevent the first broadcast enter after this
728		 * to program the bc device.
729		 */
730		tick_broadcast_clear_oneshot(cpu);
731	}
732}
733
734/*
735 * Select oneshot operating mode for the broadcast device
736 */
737void tick_broadcast_switch_to_oneshot(void)
738{
739	struct clock_event_device *bc;
740	unsigned long flags;
741
742	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
743
744	tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
745	bc = tick_broadcast_device.evtdev;
746	if (bc)
747		tick_broadcast_setup_oneshot(bc);
748
749	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
750}
751
752
753/*
754 * Remove a dead CPU from broadcasting
755 */
756void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
757{
758	unsigned long flags;
759	unsigned int cpu = *cpup;
760
761	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
762
763	/*
764	 * Clear the broadcast mask flag for the dead cpu, but do not
765	 * stop the broadcast device!
766	 */
767	cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
768
769	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
770}
771
772/*
773 * Check, whether the broadcast device is in one shot mode
774 */
775int tick_broadcast_oneshot_active(void)
776{
777	return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
778}
779
780/*
781 * Check whether the broadcast device supports oneshot.
782 */
783bool tick_broadcast_oneshot_available(void)
784{
785	struct clock_event_device *bc = tick_broadcast_device.evtdev;
786
787	return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
788}
789
790#endif
791
792void __init tick_broadcast_init(void)
793{
794	zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
795	zalloc_cpumask_var(&tmpmask, GFP_NOWAIT);
796#ifdef CONFIG_TICK_ONESHOT
797	zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
798	zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
799	zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
800#endif
801}
802