cpuidle.c revision 0aeb9cac6f8a6fc68acfb07d30b62ad6106a6384 
1/*
2 * cpuidle.c - core cpuidle infrastructure
3 *
4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
5 *               Shaohua Li <shaohua.li@intel.com>
6 *               Adam Belay <abelay@novell.com>
7 *
8 * This code is licenced under the GPL.
9 */
10
11#include <linux/kernel.h>
12#include <linux/mutex.h>
13#include <linux/sched.h>
14#include <linux/notifier.h>
15#include <linux/pm_qos.h>
16#include <linux/cpu.h>
17#include <linux/cpuidle.h>
18#include <linux/ktime.h>
19#include <linux/hrtimer.h>
20#include <linux/module.h>
21#include <trace/events/power.h>
22
23#include "cpuidle.h"
24
25DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
26
27DEFINE_MUTEX(cpuidle_lock);
28LIST_HEAD(cpuidle_detected_devices);
29
30static int enabled_devices;
31static int off __read_mostly;
32static int initialized __read_mostly;
33
34int cpuidle_disabled(void)
35{
36	return off;
37}
38void disable_cpuidle(void)
39{
40	off = 1;
41}
42
43#if defined(CONFIG_ARCH_HAS_CPU_IDLE_WAIT)
44static void cpuidle_kick_cpus(void)
45{
46	cpu_idle_wait();
47}
48#elif defined(CONFIG_SMP)
49# error "Arch needs cpu_idle_wait() equivalent here"
50#else /* !CONFIG_ARCH_HAS_CPU_IDLE_WAIT && !CONFIG_SMP */
51static void cpuidle_kick_cpus(void) {}
52#endif
53
54static int __cpuidle_register_device(struct cpuidle_device *dev);
55
56static inline int cpuidle_enter(struct cpuidle_device *dev,
57				struct cpuidle_driver *drv, int index)
58{
59	struct cpuidle_state *target_state = &drv->states[index];
60	return target_state->enter(dev, drv, index);
61}
62
63static inline int cpuidle_enter_tk(struct cpuidle_device *dev,
64			       struct cpuidle_driver *drv, int index)
65{
66	return cpuidle_wrap_enter(dev, drv, index, cpuidle_enter);
67}
68
69typedef int (*cpuidle_enter_t)(struct cpuidle_device *dev,
70			       struct cpuidle_driver *drv, int index);
71
72static cpuidle_enter_t cpuidle_enter_ops;
73
74/**
75 * cpuidle_play_dead - cpu off-lining
76 *
77 * Returns in case of an error or no driver
78 */
79int cpuidle_play_dead(void)
80{
81	struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
82	struct cpuidle_driver *drv = cpuidle_get_driver();
83	int i, dead_state = -1;
84	int power_usage = -1;
85
86	if (!drv)
87		return -ENODEV;
88
89	/* Find lowest-power state that supports long-term idle */
90	for (i = CPUIDLE_DRIVER_STATE_START; i < drv->state_count; i++) {
91		struct cpuidle_state *s = &drv->states[i];
92
93		if (s->power_usage < power_usage && s->enter_dead) {
94			power_usage = s->power_usage;
95			dead_state = i;
96		}
97	}
98
99	if (dead_state != -1)
100		return drv->states[dead_state].enter_dead(dev, dead_state);
101
102	return -ENODEV;
103}
104
105/**
106 * cpuidle_idle_call - the main idle loop
107 *
108 * NOTE: no locks or semaphores should be used here
109 * return non-zero on failure
110 */
111int cpuidle_idle_call(void)
112{
113	struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
114	struct cpuidle_driver *drv = cpuidle_get_driver();
115	int next_state, entered_state;
116
117	if (off)
118		return -ENODEV;
119
120	if (!initialized)
121		return -ENODEV;
122
123	/* check if the device is ready */
124	if (!dev || !dev->enabled)
125		return -EBUSY;
126
127	/* ask the governor for the next state */
128	next_state = cpuidle_curr_governor->select(drv, dev);
129	if (need_resched()) {
130		local_irq_enable();
131		return 0;
132	}
133
134	trace_power_start_rcuidle(POWER_CSTATE, next_state, dev->cpu);
135	trace_cpu_idle_rcuidle(next_state, dev->cpu);
136
137	entered_state = cpuidle_enter_ops(dev, drv, next_state);
138
139	trace_power_end_rcuidle(dev->cpu);
140	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
141
142	if (entered_state >= 0) {
143		/* Update cpuidle counters */
144		/* This can be moved to within driver enter routine
145		 * but that results in multiple copies of same code.
146		 */
147		dev->states_usage[entered_state].time +=
148				(unsigned long long)dev->last_residency;
149		dev->states_usage[entered_state].usage++;
150	} else {
151		dev->last_residency = 0;
152	}
153
154	/* give the governor an opportunity to reflect on the outcome */
155	if (cpuidle_curr_governor->reflect)
156		cpuidle_curr_governor->reflect(dev, entered_state);
157
158	return 0;
159}
160
161/**
162 * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
163 */
164void cpuidle_install_idle_handler(void)
165{
166	if (enabled_devices) {
167		/* Make sure all changes finished before we switch to new idle */
168		smp_wmb();
169		initialized = 1;
170	}
171}
172
173/**
174 * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
175 */
176void cpuidle_uninstall_idle_handler(void)
177{
178	if (enabled_devices) {
179		initialized = 0;
180		cpuidle_kick_cpus();
181	}
182}
183
184/**
185 * cpuidle_pause_and_lock - temporarily disables CPUIDLE
186 */
187void cpuidle_pause_and_lock(void)
188{
189	mutex_lock(&cpuidle_lock);
190	cpuidle_uninstall_idle_handler();
191}
192
193EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
194
195/**
196 * cpuidle_resume_and_unlock - resumes CPUIDLE operation
197 */
198void cpuidle_resume_and_unlock(void)
199{
200	cpuidle_install_idle_handler();
201	mutex_unlock(&cpuidle_lock);
202}
203
204EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
205
206/**
207 * cpuidle_wrap_enter - performs timekeeping and irqen around enter function
208 * @dev: pointer to a valid cpuidle_device object
209 * @drv: pointer to a valid cpuidle_driver object
210 * @index: index of the target cpuidle state.
211 */
212int cpuidle_wrap_enter(struct cpuidle_device *dev,
213				struct cpuidle_driver *drv, int index,
214				int (*enter)(struct cpuidle_device *dev,
215					struct cpuidle_driver *drv, int index))
216{
217	ktime_t time_start, time_end;
218	s64 diff;
219
220	time_start = ktime_get();
221
222	index = enter(dev, drv, index);
223
224	time_end = ktime_get();
225
226	local_irq_enable();
227
228	diff = ktime_to_us(ktime_sub(time_end, time_start));
229	if (diff > INT_MAX)
230		diff = INT_MAX;
231
232	dev->last_residency = (int) diff;
233
234	return index;
235}
236
237#ifdef CONFIG_ARCH_HAS_CPU_RELAX
238static int poll_idle(struct cpuidle_device *dev,
239		struct cpuidle_driver *drv, int index)
240{
241	ktime_t	t1, t2;
242	s64 diff;
243
244	t1 = ktime_get();
245	local_irq_enable();
246	while (!need_resched())
247		cpu_relax();
248
249	t2 = ktime_get();
250	diff = ktime_to_us(ktime_sub(t2, t1));
251	if (diff > INT_MAX)
252		diff = INT_MAX;
253
254	dev->last_residency = (int) diff;
255
256	return index;
257}
258
259static void poll_idle_init(struct cpuidle_driver *drv)
260{
261	struct cpuidle_state *state = &drv->states[0];
262
263	snprintf(state->name, CPUIDLE_NAME_LEN, "POLL");
264	snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
265	state->exit_latency = 0;
266	state->target_residency = 0;
267	state->power_usage = -1;
268	state->flags = 0;
269	state->enter = poll_idle;
270	state->disable = 0;
271}
272#else
273static void poll_idle_init(struct cpuidle_driver *drv) {}
274#endif /* CONFIG_ARCH_HAS_CPU_RELAX */
275
276/**
277 * cpuidle_enable_device - enables idle PM for a CPU
278 * @dev: the CPU
279 *
280 * This function must be called between cpuidle_pause_and_lock and
281 * cpuidle_resume_and_unlock when used externally.
282 */
283int cpuidle_enable_device(struct cpuidle_device *dev)
284{
285	int ret, i;
286	struct cpuidle_driver *drv = cpuidle_get_driver();
287
288	if (dev->enabled)
289		return 0;
290	if (!drv || !cpuidle_curr_governor)
291		return -EIO;
292	if (!dev->state_count)
293		dev->state_count = drv->state_count;
294
295	if (dev->registered == 0) {
296		ret = __cpuidle_register_device(dev);
297		if (ret)
298			return ret;
299	}
300
301	cpuidle_enter_ops = drv->en_core_tk_irqen ?
302		cpuidle_enter_tk : cpuidle_enter;
303
304	poll_idle_init(drv);
305
306	if ((ret = cpuidle_add_state_sysfs(dev)))
307		return ret;
308
309	if (cpuidle_curr_governor->enable &&
310	    (ret = cpuidle_curr_governor->enable(drv, dev)))
311		goto fail_sysfs;
312
313	for (i = 0; i < dev->state_count; i++) {
314		dev->states_usage[i].usage = 0;
315		dev->states_usage[i].time = 0;
316	}
317	dev->last_residency = 0;
318
319	smp_wmb();
320
321	dev->enabled = 1;
322
323	enabled_devices++;
324	return 0;
325
326fail_sysfs:
327	cpuidle_remove_state_sysfs(dev);
328
329	return ret;
330}
331
332EXPORT_SYMBOL_GPL(cpuidle_enable_device);
333
334/**
335 * cpuidle_disable_device - disables idle PM for a CPU
336 * @dev: the CPU
337 *
338 * This function must be called between cpuidle_pause_and_lock and
339 * cpuidle_resume_and_unlock when used externally.
340 */
341void cpuidle_disable_device(struct cpuidle_device *dev)
342{
343	if (!dev->enabled)
344		return;
345	if (!cpuidle_get_driver() || !cpuidle_curr_governor)
346		return;
347
348	dev->enabled = 0;
349
350	if (cpuidle_curr_governor->disable)
351		cpuidle_curr_governor->disable(cpuidle_get_driver(), dev);
352
353	cpuidle_remove_state_sysfs(dev);
354	enabled_devices--;
355}
356
357EXPORT_SYMBOL_GPL(cpuidle_disable_device);
358
359/**
360 * __cpuidle_register_device - internal register function called before register
361 * and enable routines
362 * @dev: the cpu
363 *
364 * cpuidle_lock mutex must be held before this is called
365 */
366static int __cpuidle_register_device(struct cpuidle_device *dev)
367{
368	int ret;
369	struct device *cpu_dev = get_cpu_device((unsigned long)dev->cpu);
370	struct cpuidle_driver *cpuidle_driver = cpuidle_get_driver();
371
372	if (!dev)
373		return -EINVAL;
374	if (!try_module_get(cpuidle_driver->owner))
375		return -EINVAL;
376
377	init_completion(&dev->kobj_unregister);
378
379	per_cpu(cpuidle_devices, dev->cpu) = dev;
380	list_add(&dev->device_list, &cpuidle_detected_devices);
381	if ((ret = cpuidle_add_sysfs(cpu_dev))) {
382		module_put(cpuidle_driver->owner);
383		return ret;
384	}
385
386	dev->registered = 1;
387	return 0;
388}
389
390/**
391 * cpuidle_register_device - registers a CPU's idle PM feature
392 * @dev: the cpu
393 */
394int cpuidle_register_device(struct cpuidle_device *dev)
395{
396	int ret;
397
398	mutex_lock(&cpuidle_lock);
399
400	if ((ret = __cpuidle_register_device(dev))) {
401		mutex_unlock(&cpuidle_lock);
402		return ret;
403	}
404
405	cpuidle_enable_device(dev);
406	cpuidle_install_idle_handler();
407
408	mutex_unlock(&cpuidle_lock);
409
410	return 0;
411
412}
413
414EXPORT_SYMBOL_GPL(cpuidle_register_device);
415
416/**
417 * cpuidle_unregister_device - unregisters a CPU's idle PM feature
418 * @dev: the cpu
419 */
420void cpuidle_unregister_device(struct cpuidle_device *dev)
421{
422	struct device *cpu_dev = get_cpu_device((unsigned long)dev->cpu);
423	struct cpuidle_driver *cpuidle_driver = cpuidle_get_driver();
424
425	if (dev->registered == 0)
426		return;
427
428	cpuidle_pause_and_lock();
429
430	cpuidle_disable_device(dev);
431
432	cpuidle_remove_sysfs(cpu_dev);
433	list_del(&dev->device_list);
434	wait_for_completion(&dev->kobj_unregister);
435	per_cpu(cpuidle_devices, dev->cpu) = NULL;
436
437	cpuidle_resume_and_unlock();
438
439	module_put(cpuidle_driver->owner);
440}
441
442EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
443
444#ifdef CONFIG_SMP
445
446static void smp_callback(void *v)
447{
448	/* we already woke the CPU up, nothing more to do */
449}
450
451/*
452 * This function gets called when a part of the kernel has a new latency
453 * requirement.  This means we need to get all processors out of their C-state,
454 * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that
455 * wakes them all right up.
456 */
457static int cpuidle_latency_notify(struct notifier_block *b,
458		unsigned long l, void *v)
459{
460	smp_call_function(smp_callback, NULL, 1);
461	return NOTIFY_OK;
462}
463
464static struct notifier_block cpuidle_latency_notifier = {
465	.notifier_call = cpuidle_latency_notify,
466};
467
468static inline void latency_notifier_init(struct notifier_block *n)
469{
470	pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
471}
472
473#else /* CONFIG_SMP */
474
475#define latency_notifier_init(x) do { } while (0)
476
477#endif /* CONFIG_SMP */
478
479/**
480 * cpuidle_init - core initializer
481 */
482static int __init cpuidle_init(void)
483{
484	int ret;
485
486	if (cpuidle_disabled())
487		return -ENODEV;
488
489	ret = cpuidle_add_interface(cpu_subsys.dev_root);
490	if (ret)
491		return ret;
492
493	latency_notifier_init(&cpuidle_latency_notifier);
494
495	return 0;
496}
497
498module_param(off, int, 0444);
499core_initcall(cpuidle_init);
500