cpuidle.c revision 30cdd69e2a266505ca8229c944d361ff350a6959 
1/*
2 * cpuidle.c - core cpuidle infrastructure
3 *
4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
5 *               Shaohua Li <shaohua.li@intel.com>
6 *               Adam Belay <abelay@novell.com>
7 *
8 * This code is licenced under the GPL.
9 */
10
11#include <linux/clockchips.h>
12#include <linux/kernel.h>
13#include <linux/mutex.h>
14#include <linux/sched.h>
15#include <linux/notifier.h>
16#include <linux/pm_qos.h>
17#include <linux/cpu.h>
18#include <linux/cpuidle.h>
19#include <linux/ktime.h>
20#include <linux/hrtimer.h>
21#include <linux/module.h>
22#include <trace/events/power.h>
23
24#include "cpuidle.h"
25
26DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
27DEFINE_PER_CPU(struct cpuidle_device, cpuidle_dev);
28
29DEFINE_MUTEX(cpuidle_lock);
30LIST_HEAD(cpuidle_detected_devices);
31
32static int enabled_devices;
33static int off __read_mostly;
34static int initialized __read_mostly;
35
36int cpuidle_disabled(void)
37{
38	return off;
39}
40void disable_cpuidle(void)
41{
42	off = 1;
43}
44
45/**
46 * cpuidle_play_dead - cpu off-lining
47 *
48 * Returns in case of an error or no driver
49 */
50int cpuidle_play_dead(void)
51{
52	struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
53	struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
54	int i;
55
56	if (!drv)
57		return -ENODEV;
58
59	/* Find lowest-power state that supports long-term idle */
60	for (i = drv->state_count - 1; i >= CPUIDLE_DRIVER_STATE_START; i--)
61		if (drv->states[i].enter_dead)
62			return drv->states[i].enter_dead(dev, i);
63
64	return -ENODEV;
65}
66
67/**
68 * cpuidle_enabled - check if the cpuidle framework is ready
69 * @dev: cpuidle device for this cpu
70 * @drv: cpuidle driver for this cpu
71 *
72 * Return 0 on success, otherwise:
73 * -NODEV : the cpuidle framework is not available
74 * -EBUSY : the cpuidle framework is not initialized
75 */
76int cpuidle_enabled(struct cpuidle_driver *drv, struct cpuidle_device *dev)
77{
78	if (off || !initialized)
79		return -ENODEV;
80
81	if (!drv || !dev || !dev->enabled)
82		return -EBUSY;
83
84	return 0;
85}
86
87/**
88 * cpuidle_enter_state - enter the state and update stats
89 * @dev: cpuidle device for this cpu
90 * @drv: cpuidle driver for this cpu
91 * @next_state: index into drv->states of the state to enter
92 */
93int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
94			int index)
95{
96	int entered_state;
97
98	struct cpuidle_state *target_state = &drv->states[index];
99	ktime_t time_start, time_end;
100	s64 diff;
101
102	time_start = ktime_get();
103
104	entered_state = target_state->enter(dev, drv, index);
105
106	time_end = ktime_get();
107
108	local_irq_enable();
109
110	diff = ktime_to_us(ktime_sub(time_end, time_start));
111	if (diff > INT_MAX)
112		diff = INT_MAX;
113
114	dev->last_residency = (int) diff;
115
116	if (entered_state >= 0) {
117		/* Update cpuidle counters */
118		/* This can be moved to within driver enter routine
119		 * but that results in multiple copies of same code.
120		 */
121		dev->states_usage[entered_state].time += dev->last_residency;
122		dev->states_usage[entered_state].usage++;
123	} else {
124		dev->last_residency = 0;
125	}
126
127	return entered_state;
128}
129
130/**
131 * cpuidle_select - ask the cpuidle framework to choose an idle state
132 *
133 * @drv: the cpuidle driver
134 * @dev: the cpuidle device
135 *
136 * Returns the index of the idle state.
137 */
138int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev)
139{
140	return cpuidle_curr_governor->select(drv, dev);
141}
142
143/**
144 * cpuidle_enter - enter into the specified idle state
145 *
146 * @drv:   the cpuidle driver tied with the cpu
147 * @dev:   the cpuidle device
148 * @index: the index in the idle state table
149 *
150 * Returns the index in the idle state, < 0 in case of error.
151 * The error code depends on the backend driver
152 */
153int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
154		  int index)
155{
156	if (cpuidle_state_is_coupled(dev, drv, index))
157		return cpuidle_enter_state_coupled(dev, drv, index);
158	return cpuidle_enter_state(dev, drv, index);
159}
160
161/**
162 * cpuidle_reflect - tell the underlying governor what was the state
163 * we were in
164 *
165 * @dev  : the cpuidle device
166 * @index: the index in the idle state table
167 *
168 */
169void cpuidle_reflect(struct cpuidle_device *dev, int index)
170{
171	if (cpuidle_curr_governor->reflect)
172		cpuidle_curr_governor->reflect(dev, index);
173}
174
175/**
176 * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
177 */
178void cpuidle_install_idle_handler(void)
179{
180	if (enabled_devices) {
181		/* Make sure all changes finished before we switch to new idle */
182		smp_wmb();
183		initialized = 1;
184	}
185}
186
187/**
188 * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
189 */
190void cpuidle_uninstall_idle_handler(void)
191{
192	if (enabled_devices) {
193		initialized = 0;
194		kick_all_cpus_sync();
195	}
196}
197
198/**
199 * cpuidle_pause_and_lock - temporarily disables CPUIDLE
200 */
201void cpuidle_pause_and_lock(void)
202{
203	mutex_lock(&cpuidle_lock);
204	cpuidle_uninstall_idle_handler();
205}
206
207EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
208
209/**
210 * cpuidle_resume_and_unlock - resumes CPUIDLE operation
211 */
212void cpuidle_resume_and_unlock(void)
213{
214	cpuidle_install_idle_handler();
215	mutex_unlock(&cpuidle_lock);
216}
217
218EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
219
220/* Currently used in suspend/resume path to suspend cpuidle */
221void cpuidle_pause(void)
222{
223	mutex_lock(&cpuidle_lock);
224	cpuidle_uninstall_idle_handler();
225	mutex_unlock(&cpuidle_lock);
226}
227
228/* Currently used in suspend/resume path to resume cpuidle */
229void cpuidle_resume(void)
230{
231	mutex_lock(&cpuidle_lock);
232	cpuidle_install_idle_handler();
233	mutex_unlock(&cpuidle_lock);
234}
235
236/**
237 * cpuidle_enable_device - enables idle PM for a CPU
238 * @dev: the CPU
239 *
240 * This function must be called between cpuidle_pause_and_lock and
241 * cpuidle_resume_and_unlock when used externally.
242 */
243int cpuidle_enable_device(struct cpuidle_device *dev)
244{
245	int ret;
246	struct cpuidle_driver *drv;
247
248	if (!dev)
249		return -EINVAL;
250
251	if (dev->enabled)
252		return 0;
253
254	drv = cpuidle_get_cpu_driver(dev);
255
256	if (!drv || !cpuidle_curr_governor)
257		return -EIO;
258
259	if (!dev->registered)
260		return -EINVAL;
261
262	if (!dev->state_count)
263		dev->state_count = drv->state_count;
264
265	ret = cpuidle_add_device_sysfs(dev);
266	if (ret)
267		return ret;
268
269	if (cpuidle_curr_governor->enable &&
270	    (ret = cpuidle_curr_governor->enable(drv, dev)))
271		goto fail_sysfs;
272
273	smp_wmb();
274
275	dev->enabled = 1;
276
277	enabled_devices++;
278	return 0;
279
280fail_sysfs:
281	cpuidle_remove_device_sysfs(dev);
282
283	return ret;
284}
285
286EXPORT_SYMBOL_GPL(cpuidle_enable_device);
287
288/**
289 * cpuidle_disable_device - disables idle PM for a CPU
290 * @dev: the CPU
291 *
292 * This function must be called between cpuidle_pause_and_lock and
293 * cpuidle_resume_and_unlock when used externally.
294 */
295void cpuidle_disable_device(struct cpuidle_device *dev)
296{
297	struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
298
299	if (!dev || !dev->enabled)
300		return;
301
302	if (!drv || !cpuidle_curr_governor)
303		return;
304
305	dev->enabled = 0;
306
307	if (cpuidle_curr_governor->disable)
308		cpuidle_curr_governor->disable(drv, dev);
309
310	cpuidle_remove_device_sysfs(dev);
311	enabled_devices--;
312}
313
314EXPORT_SYMBOL_GPL(cpuidle_disable_device);
315
316static void __cpuidle_unregister_device(struct cpuidle_device *dev)
317{
318	struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
319
320	list_del(&dev->device_list);
321	per_cpu(cpuidle_devices, dev->cpu) = NULL;
322	module_put(drv->owner);
323}
324
325static void __cpuidle_device_init(struct cpuidle_device *dev)
326{
327	memset(dev->states_usage, 0, sizeof(dev->states_usage));
328	dev->last_residency = 0;
329}
330
331/**
332 * __cpuidle_register_device - internal register function called before register
333 * and enable routines
334 * @dev: the cpu
335 *
336 * cpuidle_lock mutex must be held before this is called
337 */
338static int __cpuidle_register_device(struct cpuidle_device *dev)
339{
340	int ret;
341	struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
342
343	if (!try_module_get(drv->owner))
344		return -EINVAL;
345
346	per_cpu(cpuidle_devices, dev->cpu) = dev;
347	list_add(&dev->device_list, &cpuidle_detected_devices);
348
349	ret = cpuidle_coupled_register_device(dev);
350	if (ret)
351		__cpuidle_unregister_device(dev);
352	else
353		dev->registered = 1;
354
355	return ret;
356}
357
358/**
359 * cpuidle_register_device - registers a CPU's idle PM feature
360 * @dev: the cpu
361 */
362int cpuidle_register_device(struct cpuidle_device *dev)
363{
364	int ret = -EBUSY;
365
366	if (!dev)
367		return -EINVAL;
368
369	mutex_lock(&cpuidle_lock);
370
371	if (dev->registered)
372		goto out_unlock;
373
374	__cpuidle_device_init(dev);
375
376	ret = __cpuidle_register_device(dev);
377	if (ret)
378		goto out_unlock;
379
380	ret = cpuidle_add_sysfs(dev);
381	if (ret)
382		goto out_unregister;
383
384	ret = cpuidle_enable_device(dev);
385	if (ret)
386		goto out_sysfs;
387
388	cpuidle_install_idle_handler();
389
390out_unlock:
391	mutex_unlock(&cpuidle_lock);
392
393	return ret;
394
395out_sysfs:
396	cpuidle_remove_sysfs(dev);
397out_unregister:
398	__cpuidle_unregister_device(dev);
399	goto out_unlock;
400}
401
402EXPORT_SYMBOL_GPL(cpuidle_register_device);
403
404/**
405 * cpuidle_unregister_device - unregisters a CPU's idle PM feature
406 * @dev: the cpu
407 */
408void cpuidle_unregister_device(struct cpuidle_device *dev)
409{
410	if (!dev || dev->registered == 0)
411		return;
412
413	cpuidle_pause_and_lock();
414
415	cpuidle_disable_device(dev);
416
417	cpuidle_remove_sysfs(dev);
418
419	__cpuidle_unregister_device(dev);
420
421	cpuidle_coupled_unregister_device(dev);
422
423	cpuidle_resume_and_unlock();
424}
425
426EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
427
428/**
429 * cpuidle_unregister: unregister a driver and the devices. This function
430 * can be used only if the driver has been previously registered through
431 * the cpuidle_register function.
432 *
433 * @drv: a valid pointer to a struct cpuidle_driver
434 */
435void cpuidle_unregister(struct cpuidle_driver *drv)
436{
437	int cpu;
438	struct cpuidle_device *device;
439
440	for_each_cpu(cpu, drv->cpumask) {
441		device = &per_cpu(cpuidle_dev, cpu);
442		cpuidle_unregister_device(device);
443	}
444
445	cpuidle_unregister_driver(drv);
446}
447EXPORT_SYMBOL_GPL(cpuidle_unregister);
448
449/**
450 * cpuidle_register: registers the driver and the cpu devices with the
451 * coupled_cpus passed as parameter. This function is used for all common
452 * initialization pattern there are in the arch specific drivers. The
453 * devices is globally defined in this file.
454 *
455 * @drv         : a valid pointer to a struct cpuidle_driver
456 * @coupled_cpus: a cpumask for the coupled states
457 *
458 * Returns 0 on success, < 0 otherwise
459 */
460int cpuidle_register(struct cpuidle_driver *drv,
461		     const struct cpumask *const coupled_cpus)
462{
463	int ret, cpu;
464	struct cpuidle_device *device;
465
466	ret = cpuidle_register_driver(drv);
467	if (ret) {
468		pr_err("failed to register cpuidle driver\n");
469		return ret;
470	}
471
472	for_each_cpu(cpu, drv->cpumask) {
473		device = &per_cpu(cpuidle_dev, cpu);
474		device->cpu = cpu;
475
476#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED
477		/*
478		 * On multiplatform for ARM, the coupled idle states could be
479		 * enabled in the kernel even if the cpuidle driver does not
480		 * use it. Note, coupled_cpus is a struct copy.
481		 */
482		if (coupled_cpus)
483			device->coupled_cpus = *coupled_cpus;
484#endif
485		ret = cpuidle_register_device(device);
486		if (!ret)
487			continue;
488
489		pr_err("Failed to register cpuidle device for cpu%d\n", cpu);
490
491		cpuidle_unregister(drv);
492		break;
493	}
494
495	return ret;
496}
497EXPORT_SYMBOL_GPL(cpuidle_register);
498
499#ifdef CONFIG_SMP
500
501static void smp_callback(void *v)
502{
503	/* we already woke the CPU up, nothing more to do */
504}
505
506/*
507 * This function gets called when a part of the kernel has a new latency
508 * requirement.  This means we need to get all processors out of their C-state,
509 * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that
510 * wakes them all right up.
511 */
512static int cpuidle_latency_notify(struct notifier_block *b,
513		unsigned long l, void *v)
514{
515	smp_call_function(smp_callback, NULL, 1);
516	return NOTIFY_OK;
517}
518
519static struct notifier_block cpuidle_latency_notifier = {
520	.notifier_call = cpuidle_latency_notify,
521};
522
523static inline void latency_notifier_init(struct notifier_block *n)
524{
525	pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
526}
527
528#else /* CONFIG_SMP */
529
530#define latency_notifier_init(x) do { } while (0)
531
532#endif /* CONFIG_SMP */
533
534/**
535 * cpuidle_init - core initializer
536 */
537static int __init cpuidle_init(void)
538{
539	int ret;
540
541	if (cpuidle_disabled())
542		return -ENODEV;
543
544	ret = cpuidle_add_interface(cpu_subsys.dev_root);
545	if (ret)
546		return ret;
547
548	latency_notifier_init(&cpuidle_latency_notifier);
549
550	return 0;
551}
552
553module_param(off, int, 0444);
554core_initcall(cpuidle_init);
555