cpuidle.c revision 2ed903c5485bad0eafdd3d59ff993598736e4f31
1/* 2 * cpuidle.c - core cpuidle infrastructure 3 * 4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> 5 * Shaohua Li <shaohua.li@intel.com> 6 * Adam Belay <abelay@novell.com> 7 * 8 * This code is licenced under the GPL. 9 */ 10 11#include <linux/clockchips.h> 12#include <linux/kernel.h> 13#include <linux/mutex.h> 14#include <linux/sched.h> 15#include <linux/notifier.h> 16#include <linux/pm_qos.h> 17#include <linux/cpu.h> 18#include <linux/cpuidle.h> 19#include <linux/ktime.h> 20#include <linux/hrtimer.h> 21#include <linux/module.h> 22#include <trace/events/power.h> 23 24#include "cpuidle.h" 25 26DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices); 27DEFINE_PER_CPU(struct cpuidle_device, cpuidle_dev); 28 29DEFINE_MUTEX(cpuidle_lock); 30LIST_HEAD(cpuidle_detected_devices); 31 32static int enabled_devices; 33static int off __read_mostly; 34static int initialized __read_mostly; 35static bool use_deepest_state __read_mostly; 36 37int cpuidle_disabled(void) 38{ 39 return off; 40} 41void disable_cpuidle(void) 42{ 43 off = 1; 44} 45 46/** 47 * cpuidle_play_dead - cpu off-lining 48 * 49 * Returns in case of an error or no driver 50 */ 51int cpuidle_play_dead(void) 52{ 53 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices); 54 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 55 int i; 56 57 if (!drv) 58 return -ENODEV; 59 60 /* Find lowest-power state that supports long-term idle */ 61 for (i = drv->state_count - 1; i >= CPUIDLE_DRIVER_STATE_START; i--) 62 if (drv->states[i].enter_dead) 63 return drv->states[i].enter_dead(dev, i); 64 65 return -ENODEV; 66} 67 68/** 69 * cpuidle_use_deepest_state - Enable/disable the "deepest idle" mode. 70 * @enable: Whether enable or disable the feature. 71 * 72 * If the "deepest idle" mode is enabled, cpuidle will ignore the governor and 73 * always use the state with the greatest exit latency (out of the states that 74 * are not disabled). 75 * 76 * This function can only be called after cpuidle_pause() to avoid races. 77 */ 78void cpuidle_use_deepest_state(bool enable) 79{ 80 use_deepest_state = enable; 81} 82 83/** 84 * cpuidle_find_deepest_state - Find the state of the greatest exit latency. 85 * @drv: cpuidle driver for a given CPU. 86 * @dev: cpuidle device for a given CPU. 87 */ 88static int cpuidle_find_deepest_state(struct cpuidle_driver *drv, 89 struct cpuidle_device *dev) 90{ 91 unsigned int latency_req = 0; 92 int i, ret = CPUIDLE_DRIVER_STATE_START - 1; 93 94 for (i = CPUIDLE_DRIVER_STATE_START; i < drv->state_count; i++) { 95 struct cpuidle_state *s = &drv->states[i]; 96 struct cpuidle_state_usage *su = &dev->states_usage[i]; 97 98 if (s->disabled || su->disable || s->exit_latency <= latency_req) 99 continue; 100 101 latency_req = s->exit_latency; 102 ret = i; 103 } 104 return ret; 105} 106 107/** 108 * cpuidle_enter_state - enter the state and update stats 109 * @dev: cpuidle device for this cpu 110 * @drv: cpuidle driver for this cpu 111 * @next_state: index into drv->states of the state to enter 112 */ 113int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, 114 int index) 115{ 116 int entered_state; 117 118 struct cpuidle_state *target_state = &drv->states[index]; 119 ktime_t time_start, time_end; 120 s64 diff; 121 122 trace_cpu_idle_rcuidle(index, dev->cpu); 123 time_start = ktime_get(); 124 125 entered_state = target_state->enter(dev, drv, index); 126 127 time_end = ktime_get(); 128 trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu); 129 130 if (!cpuidle_state_is_coupled(dev, drv, entered_state)) 131 local_irq_enable(); 132 133 diff = ktime_to_us(ktime_sub(time_end, time_start)); 134 if (diff > INT_MAX) 135 diff = INT_MAX; 136 137 dev->last_residency = (int) diff; 138 139 if (entered_state >= 0) { 140 /* Update cpuidle counters */ 141 /* This can be moved to within driver enter routine 142 * but that results in multiple copies of same code. 143 */ 144 dev->states_usage[entered_state].time += dev->last_residency; 145 dev->states_usage[entered_state].usage++; 146 } else { 147 dev->last_residency = 0; 148 } 149 150 return entered_state; 151} 152 153/** 154 * cpuidle_select - ask the cpuidle framework to choose an idle state 155 * 156 * @drv: the cpuidle driver 157 * @dev: the cpuidle device 158 * 159 * Returns the index of the idle state. 160 */ 161int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev) 162{ 163 if (off || !initialized) 164 return -ENODEV; 165 166 if (!drv || !dev || !dev->enabled) 167 return -EBUSY; 168 169 if (unlikely(use_deepest_state)) 170 return cpuidle_find_deepest_state(drv, dev); 171 172 return cpuidle_curr_governor->select(drv, dev); 173} 174 175/** 176 * cpuidle_enter - enter into the specified idle state 177 * 178 * @drv: the cpuidle driver tied with the cpu 179 * @dev: the cpuidle device 180 * @index: the index in the idle state table 181 * 182 * Returns the index in the idle state, < 0 in case of error. 183 * The error code depends on the backend driver 184 */ 185int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev, 186 int index) 187{ 188 if (cpuidle_state_is_coupled(dev, drv, index)) 189 return cpuidle_enter_state_coupled(dev, drv, index); 190 return cpuidle_enter_state(dev, drv, index); 191} 192 193/** 194 * cpuidle_reflect - tell the underlying governor what was the state 195 * we were in 196 * 197 * @dev : the cpuidle device 198 * @index: the index in the idle state table 199 * 200 */ 201void cpuidle_reflect(struct cpuidle_device *dev, int index) 202{ 203 if (cpuidle_curr_governor->reflect && !unlikely(use_deepest_state)) 204 cpuidle_curr_governor->reflect(dev, index); 205} 206 207/** 208 * cpuidle_install_idle_handler - installs the cpuidle idle loop handler 209 */ 210void cpuidle_install_idle_handler(void) 211{ 212 if (enabled_devices) { 213 /* Make sure all changes finished before we switch to new idle */ 214 smp_wmb(); 215 initialized = 1; 216 } 217} 218 219/** 220 * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler 221 */ 222void cpuidle_uninstall_idle_handler(void) 223{ 224 if (enabled_devices) { 225 initialized = 0; 226 wake_up_all_idle_cpus(); 227 } 228} 229 230/** 231 * cpuidle_pause_and_lock - temporarily disables CPUIDLE 232 */ 233void cpuidle_pause_and_lock(void) 234{ 235 mutex_lock(&cpuidle_lock); 236 cpuidle_uninstall_idle_handler(); 237} 238 239EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock); 240 241/** 242 * cpuidle_resume_and_unlock - resumes CPUIDLE operation 243 */ 244void cpuidle_resume_and_unlock(void) 245{ 246 cpuidle_install_idle_handler(); 247 mutex_unlock(&cpuidle_lock); 248} 249 250EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock); 251 252/* Currently used in suspend/resume path to suspend cpuidle */ 253void cpuidle_pause(void) 254{ 255 mutex_lock(&cpuidle_lock); 256 cpuidle_uninstall_idle_handler(); 257 mutex_unlock(&cpuidle_lock); 258} 259 260/* Currently used in suspend/resume path to resume cpuidle */ 261void cpuidle_resume(void) 262{ 263 mutex_lock(&cpuidle_lock); 264 cpuidle_install_idle_handler(); 265 mutex_unlock(&cpuidle_lock); 266} 267 268/** 269 * cpuidle_enable_device - enables idle PM for a CPU 270 * @dev: the CPU 271 * 272 * This function must be called between cpuidle_pause_and_lock and 273 * cpuidle_resume_and_unlock when used externally. 274 */ 275int cpuidle_enable_device(struct cpuidle_device *dev) 276{ 277 int ret; 278 struct cpuidle_driver *drv; 279 280 if (!dev) 281 return -EINVAL; 282 283 if (dev->enabled) 284 return 0; 285 286 drv = cpuidle_get_cpu_driver(dev); 287 288 if (!drv || !cpuidle_curr_governor) 289 return -EIO; 290 291 if (!dev->registered) 292 return -EINVAL; 293 294 if (!dev->state_count) 295 dev->state_count = drv->state_count; 296 297 ret = cpuidle_add_device_sysfs(dev); 298 if (ret) 299 return ret; 300 301 if (cpuidle_curr_governor->enable && 302 (ret = cpuidle_curr_governor->enable(drv, dev))) 303 goto fail_sysfs; 304 305 smp_wmb(); 306 307 dev->enabled = 1; 308 309 enabled_devices++; 310 return 0; 311 312fail_sysfs: 313 cpuidle_remove_device_sysfs(dev); 314 315 return ret; 316} 317 318EXPORT_SYMBOL_GPL(cpuidle_enable_device); 319 320/** 321 * cpuidle_disable_device - disables idle PM for a CPU 322 * @dev: the CPU 323 * 324 * This function must be called between cpuidle_pause_and_lock and 325 * cpuidle_resume_and_unlock when used externally. 326 */ 327void cpuidle_disable_device(struct cpuidle_device *dev) 328{ 329 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 330 331 if (!dev || !dev->enabled) 332 return; 333 334 if (!drv || !cpuidle_curr_governor) 335 return; 336 337 dev->enabled = 0; 338 339 if (cpuidle_curr_governor->disable) 340 cpuidle_curr_governor->disable(drv, dev); 341 342 cpuidle_remove_device_sysfs(dev); 343 enabled_devices--; 344} 345 346EXPORT_SYMBOL_GPL(cpuidle_disable_device); 347 348static void __cpuidle_unregister_device(struct cpuidle_device *dev) 349{ 350 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 351 352 list_del(&dev->device_list); 353 per_cpu(cpuidle_devices, dev->cpu) = NULL; 354 module_put(drv->owner); 355} 356 357static void __cpuidle_device_init(struct cpuidle_device *dev) 358{ 359 memset(dev->states_usage, 0, sizeof(dev->states_usage)); 360 dev->last_residency = 0; 361} 362 363/** 364 * __cpuidle_register_device - internal register function called before register 365 * and enable routines 366 * @dev: the cpu 367 * 368 * cpuidle_lock mutex must be held before this is called 369 */ 370static int __cpuidle_register_device(struct cpuidle_device *dev) 371{ 372 int ret; 373 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 374 375 if (!try_module_get(drv->owner)) 376 return -EINVAL; 377 378 per_cpu(cpuidle_devices, dev->cpu) = dev; 379 list_add(&dev->device_list, &cpuidle_detected_devices); 380 381 ret = cpuidle_coupled_register_device(dev); 382 if (ret) 383 __cpuidle_unregister_device(dev); 384 else 385 dev->registered = 1; 386 387 return ret; 388} 389 390/** 391 * cpuidle_register_device - registers a CPU's idle PM feature 392 * @dev: the cpu 393 */ 394int cpuidle_register_device(struct cpuidle_device *dev) 395{ 396 int ret = -EBUSY; 397 398 if (!dev) 399 return -EINVAL; 400 401 mutex_lock(&cpuidle_lock); 402 403 if (dev->registered) 404 goto out_unlock; 405 406 __cpuidle_device_init(dev); 407 408 ret = __cpuidle_register_device(dev); 409 if (ret) 410 goto out_unlock; 411 412 ret = cpuidle_add_sysfs(dev); 413 if (ret) 414 goto out_unregister; 415 416 ret = cpuidle_enable_device(dev); 417 if (ret) 418 goto out_sysfs; 419 420 cpuidle_install_idle_handler(); 421 422out_unlock: 423 mutex_unlock(&cpuidle_lock); 424 425 return ret; 426 427out_sysfs: 428 cpuidle_remove_sysfs(dev); 429out_unregister: 430 __cpuidle_unregister_device(dev); 431 goto out_unlock; 432} 433 434EXPORT_SYMBOL_GPL(cpuidle_register_device); 435 436/** 437 * cpuidle_unregister_device - unregisters a CPU's idle PM feature 438 * @dev: the cpu 439 */ 440void cpuidle_unregister_device(struct cpuidle_device *dev) 441{ 442 if (!dev || dev->registered == 0) 443 return; 444 445 cpuidle_pause_and_lock(); 446 447 cpuidle_disable_device(dev); 448 449 cpuidle_remove_sysfs(dev); 450 451 __cpuidle_unregister_device(dev); 452 453 cpuidle_coupled_unregister_device(dev); 454 455 cpuidle_resume_and_unlock(); 456} 457 458EXPORT_SYMBOL_GPL(cpuidle_unregister_device); 459 460/** 461 * cpuidle_unregister: unregister a driver and the devices. This function 462 * can be used only if the driver has been previously registered through 463 * the cpuidle_register function. 464 * 465 * @drv: a valid pointer to a struct cpuidle_driver 466 */ 467void cpuidle_unregister(struct cpuidle_driver *drv) 468{ 469 int cpu; 470 struct cpuidle_device *device; 471 472 for_each_cpu(cpu, drv->cpumask) { 473 device = &per_cpu(cpuidle_dev, cpu); 474 cpuidle_unregister_device(device); 475 } 476 477 cpuidle_unregister_driver(drv); 478} 479EXPORT_SYMBOL_GPL(cpuidle_unregister); 480 481/** 482 * cpuidle_register: registers the driver and the cpu devices with the 483 * coupled_cpus passed as parameter. This function is used for all common 484 * initialization pattern there are in the arch specific drivers. The 485 * devices is globally defined in this file. 486 * 487 * @drv : a valid pointer to a struct cpuidle_driver 488 * @coupled_cpus: a cpumask for the coupled states 489 * 490 * Returns 0 on success, < 0 otherwise 491 */ 492int cpuidle_register(struct cpuidle_driver *drv, 493 const struct cpumask *const coupled_cpus) 494{ 495 int ret, cpu; 496 struct cpuidle_device *device; 497 498 ret = cpuidle_register_driver(drv); 499 if (ret) { 500 pr_err("failed to register cpuidle driver\n"); 501 return ret; 502 } 503 504 for_each_cpu(cpu, drv->cpumask) { 505 device = &per_cpu(cpuidle_dev, cpu); 506 device->cpu = cpu; 507 508#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED 509 /* 510 * On multiplatform for ARM, the coupled idle states could be 511 * enabled in the kernel even if the cpuidle driver does not 512 * use it. Note, coupled_cpus is a struct copy. 513 */ 514 if (coupled_cpus) 515 device->coupled_cpus = *coupled_cpus; 516#endif 517 ret = cpuidle_register_device(device); 518 if (!ret) 519 continue; 520 521 pr_err("Failed to register cpuidle device for cpu%d\n", cpu); 522 523 cpuidle_unregister(drv); 524 break; 525 } 526 527 return ret; 528} 529EXPORT_SYMBOL_GPL(cpuidle_register); 530 531#ifdef CONFIG_SMP 532 533/* 534 * This function gets called when a part of the kernel has a new latency 535 * requirement. This means we need to get all processors out of their C-state, 536 * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that 537 * wakes them all right up. 538 */ 539static int cpuidle_latency_notify(struct notifier_block *b, 540 unsigned long l, void *v) 541{ 542 wake_up_all_idle_cpus(); 543 return NOTIFY_OK; 544} 545 546static struct notifier_block cpuidle_latency_notifier = { 547 .notifier_call = cpuidle_latency_notify, 548}; 549 550static inline void latency_notifier_init(struct notifier_block *n) 551{ 552 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n); 553} 554 555#else /* CONFIG_SMP */ 556 557#define latency_notifier_init(x) do { } while (0) 558 559#endif /* CONFIG_SMP */ 560 561/** 562 * cpuidle_init - core initializer 563 */ 564static int __init cpuidle_init(void) 565{ 566 int ret; 567 568 if (cpuidle_disabled()) 569 return -ENODEV; 570 571 ret = cpuidle_add_interface(cpu_subsys.dev_root); 572 if (ret) 573 return ret; 574 575 latency_notifier_init(&cpuidle_latency_notifier); 576 577 return 0; 578} 579 580module_param(off, int, 0444); 581core_initcall(cpuidle_init); 582