cpuidle.c revision a6220fc19afc07fe77cfd16f5b8e568615517091
1/* 2 * cpuidle.c - core cpuidle infrastructure 3 * 4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> 5 * Shaohua Li <shaohua.li@intel.com> 6 * Adam Belay <abelay@novell.com> 7 * 8 * This code is licenced under the GPL. 9 */ 10 11#include <linux/clockchips.h> 12#include <linux/kernel.h> 13#include <linux/mutex.h> 14#include <linux/sched.h> 15#include <linux/notifier.h> 16#include <linux/pm_qos.h> 17#include <linux/cpu.h> 18#include <linux/cpuidle.h> 19#include <linux/ktime.h> 20#include <linux/hrtimer.h> 21#include <linux/module.h> 22#include <trace/events/power.h> 23 24#include "cpuidle.h" 25 26DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices); 27DEFINE_PER_CPU(struct cpuidle_device, cpuidle_dev); 28 29DEFINE_MUTEX(cpuidle_lock); 30LIST_HEAD(cpuidle_detected_devices); 31 32static int enabled_devices; 33static int off __read_mostly; 34static int initialized __read_mostly; 35static bool use_deepest_state __read_mostly; 36 37int cpuidle_disabled(void) 38{ 39 return off; 40} 41void disable_cpuidle(void) 42{ 43 off = 1; 44} 45 46/** 47 * cpuidle_play_dead - cpu off-lining 48 * 49 * Returns in case of an error or no driver 50 */ 51int cpuidle_play_dead(void) 52{ 53 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices); 54 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 55 int i; 56 57 if (!drv) 58 return -ENODEV; 59 60 /* Find lowest-power state that supports long-term idle */ 61 for (i = drv->state_count - 1; i >= CPUIDLE_DRIVER_STATE_START; i--) 62 if (drv->states[i].enter_dead) 63 return drv->states[i].enter_dead(dev, i); 64 65 return -ENODEV; 66} 67 68/** 69 * cpuidle_use_deepest_state - Enable/disable the "deepest idle" mode. 70 * @enable: Whether enable or disable the feature. 71 * 72 * If the "deepest idle" mode is enabled, cpuidle will ignore the governor and 73 * always use the state with the greatest exit latency (out of the states that 74 * are not disabled). 75 * 76 * This function can only be called after cpuidle_pause() to avoid races. 77 */ 78void cpuidle_use_deepest_state(bool enable) 79{ 80 use_deepest_state = enable; 81} 82 83/** 84 * cpuidle_find_deepest_state - Find the state of the greatest exit latency. 85 * @drv: cpuidle driver for a given CPU. 86 * @dev: cpuidle device for a given CPU. 87 */ 88static int cpuidle_find_deepest_state(struct cpuidle_driver *drv, 89 struct cpuidle_device *dev) 90{ 91 unsigned int latency_req = 0; 92 int i, ret = CPUIDLE_DRIVER_STATE_START - 1; 93 94 for (i = CPUIDLE_DRIVER_STATE_START; i < drv->state_count; i++) { 95 struct cpuidle_state *s = &drv->states[i]; 96 struct cpuidle_state_usage *su = &dev->states_usage[i]; 97 98 if (s->disabled || su->disable || s->exit_latency <= latency_req) 99 continue; 100 101 latency_req = s->exit_latency; 102 ret = i; 103 } 104 return ret; 105} 106 107/** 108 * cpuidle_enter_state - enter the state and update stats 109 * @dev: cpuidle device for this cpu 110 * @drv: cpuidle driver for this cpu 111 * @next_state: index into drv->states of the state to enter 112 */ 113int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, 114 int index) 115{ 116 int entered_state; 117 118 struct cpuidle_state *target_state = &drv->states[index]; 119 ktime_t time_start, time_end; 120 s64 diff; 121 122 time_start = ktime_get(); 123 124 entered_state = target_state->enter(dev, drv, index); 125 126 time_end = ktime_get(); 127 128 if (!cpuidle_state_is_coupled(dev, drv, entered_state)) 129 local_irq_enable(); 130 131 diff = ktime_to_us(ktime_sub(time_end, time_start)); 132 if (diff > INT_MAX) 133 diff = INT_MAX; 134 135 dev->last_residency = (int) diff; 136 137 if (entered_state >= 0) { 138 /* Update cpuidle counters */ 139 /* This can be moved to within driver enter routine 140 * but that results in multiple copies of same code. 141 */ 142 dev->states_usage[entered_state].time += dev->last_residency; 143 dev->states_usage[entered_state].usage++; 144 } else { 145 dev->last_residency = 0; 146 } 147 148 return entered_state; 149} 150 151/** 152 * cpuidle_select - ask the cpuidle framework to choose an idle state 153 * 154 * @drv: the cpuidle driver 155 * @dev: the cpuidle device 156 * 157 * Returns the index of the idle state. 158 */ 159int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev) 160{ 161 if (off || !initialized) 162 return -ENODEV; 163 164 if (!drv || !dev || !dev->enabled) 165 return -EBUSY; 166 167 if (unlikely(use_deepest_state)) 168 return cpuidle_find_deepest_state(drv, dev); 169 170 return cpuidle_curr_governor->select(drv, dev); 171} 172 173/** 174 * cpuidle_enter - enter into the specified idle state 175 * 176 * @drv: the cpuidle driver tied with the cpu 177 * @dev: the cpuidle device 178 * @index: the index in the idle state table 179 * 180 * Returns the index in the idle state, < 0 in case of error. 181 * The error code depends on the backend driver 182 */ 183int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev, 184 int index) 185{ 186 if (cpuidle_state_is_coupled(dev, drv, index)) 187 return cpuidle_enter_state_coupled(dev, drv, index); 188 return cpuidle_enter_state(dev, drv, index); 189} 190 191/** 192 * cpuidle_reflect - tell the underlying governor what was the state 193 * we were in 194 * 195 * @dev : the cpuidle device 196 * @index: the index in the idle state table 197 * 198 */ 199void cpuidle_reflect(struct cpuidle_device *dev, int index) 200{ 201 if (cpuidle_curr_governor->reflect && !unlikely(use_deepest_state)) 202 cpuidle_curr_governor->reflect(dev, index); 203} 204 205/** 206 * cpuidle_install_idle_handler - installs the cpuidle idle loop handler 207 */ 208void cpuidle_install_idle_handler(void) 209{ 210 if (enabled_devices) { 211 /* Make sure all changes finished before we switch to new idle */ 212 smp_wmb(); 213 initialized = 1; 214 } 215} 216 217/** 218 * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler 219 */ 220void cpuidle_uninstall_idle_handler(void) 221{ 222 if (enabled_devices) { 223 initialized = 0; 224 kick_all_cpus_sync(); 225 } 226} 227 228/** 229 * cpuidle_pause_and_lock - temporarily disables CPUIDLE 230 */ 231void cpuidle_pause_and_lock(void) 232{ 233 mutex_lock(&cpuidle_lock); 234 cpuidle_uninstall_idle_handler(); 235} 236 237EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock); 238 239/** 240 * cpuidle_resume_and_unlock - resumes CPUIDLE operation 241 */ 242void cpuidle_resume_and_unlock(void) 243{ 244 cpuidle_install_idle_handler(); 245 mutex_unlock(&cpuidle_lock); 246} 247 248EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock); 249 250/* Currently used in suspend/resume path to suspend cpuidle */ 251void cpuidle_pause(void) 252{ 253 mutex_lock(&cpuidle_lock); 254 cpuidle_uninstall_idle_handler(); 255 mutex_unlock(&cpuidle_lock); 256} 257 258/* Currently used in suspend/resume path to resume cpuidle */ 259void cpuidle_resume(void) 260{ 261 mutex_lock(&cpuidle_lock); 262 cpuidle_install_idle_handler(); 263 mutex_unlock(&cpuidle_lock); 264} 265 266/** 267 * cpuidle_enable_device - enables idle PM for a CPU 268 * @dev: the CPU 269 * 270 * This function must be called between cpuidle_pause_and_lock and 271 * cpuidle_resume_and_unlock when used externally. 272 */ 273int cpuidle_enable_device(struct cpuidle_device *dev) 274{ 275 int ret; 276 struct cpuidle_driver *drv; 277 278 if (!dev) 279 return -EINVAL; 280 281 if (dev->enabled) 282 return 0; 283 284 drv = cpuidle_get_cpu_driver(dev); 285 286 if (!drv || !cpuidle_curr_governor) 287 return -EIO; 288 289 if (!dev->registered) 290 return -EINVAL; 291 292 if (!dev->state_count) 293 dev->state_count = drv->state_count; 294 295 ret = cpuidle_add_device_sysfs(dev); 296 if (ret) 297 return ret; 298 299 if (cpuidle_curr_governor->enable && 300 (ret = cpuidle_curr_governor->enable(drv, dev))) 301 goto fail_sysfs; 302 303 smp_wmb(); 304 305 dev->enabled = 1; 306 307 enabled_devices++; 308 return 0; 309 310fail_sysfs: 311 cpuidle_remove_device_sysfs(dev); 312 313 return ret; 314} 315 316EXPORT_SYMBOL_GPL(cpuidle_enable_device); 317 318/** 319 * cpuidle_disable_device - disables idle PM for a CPU 320 * @dev: the CPU 321 * 322 * This function must be called between cpuidle_pause_and_lock and 323 * cpuidle_resume_and_unlock when used externally. 324 */ 325void cpuidle_disable_device(struct cpuidle_device *dev) 326{ 327 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 328 329 if (!dev || !dev->enabled) 330 return; 331 332 if (!drv || !cpuidle_curr_governor) 333 return; 334 335 dev->enabled = 0; 336 337 if (cpuidle_curr_governor->disable) 338 cpuidle_curr_governor->disable(drv, dev); 339 340 cpuidle_remove_device_sysfs(dev); 341 enabled_devices--; 342} 343 344EXPORT_SYMBOL_GPL(cpuidle_disable_device); 345 346static void __cpuidle_unregister_device(struct cpuidle_device *dev) 347{ 348 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 349 350 list_del(&dev->device_list); 351 per_cpu(cpuidle_devices, dev->cpu) = NULL; 352 module_put(drv->owner); 353} 354 355static void __cpuidle_device_init(struct cpuidle_device *dev) 356{ 357 memset(dev->states_usage, 0, sizeof(dev->states_usage)); 358 dev->last_residency = 0; 359} 360 361/** 362 * __cpuidle_register_device - internal register function called before register 363 * and enable routines 364 * @dev: the cpu 365 * 366 * cpuidle_lock mutex must be held before this is called 367 */ 368static int __cpuidle_register_device(struct cpuidle_device *dev) 369{ 370 int ret; 371 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 372 373 if (!try_module_get(drv->owner)) 374 return -EINVAL; 375 376 per_cpu(cpuidle_devices, dev->cpu) = dev; 377 list_add(&dev->device_list, &cpuidle_detected_devices); 378 379 ret = cpuidle_coupled_register_device(dev); 380 if (ret) 381 __cpuidle_unregister_device(dev); 382 else 383 dev->registered = 1; 384 385 return ret; 386} 387 388/** 389 * cpuidle_register_device - registers a CPU's idle PM feature 390 * @dev: the cpu 391 */ 392int cpuidle_register_device(struct cpuidle_device *dev) 393{ 394 int ret = -EBUSY; 395 396 if (!dev) 397 return -EINVAL; 398 399 mutex_lock(&cpuidle_lock); 400 401 if (dev->registered) 402 goto out_unlock; 403 404 __cpuidle_device_init(dev); 405 406 ret = __cpuidle_register_device(dev); 407 if (ret) 408 goto out_unlock; 409 410 ret = cpuidle_add_sysfs(dev); 411 if (ret) 412 goto out_unregister; 413 414 ret = cpuidle_enable_device(dev); 415 if (ret) 416 goto out_sysfs; 417 418 cpuidle_install_idle_handler(); 419 420out_unlock: 421 mutex_unlock(&cpuidle_lock); 422 423 return ret; 424 425out_sysfs: 426 cpuidle_remove_sysfs(dev); 427out_unregister: 428 __cpuidle_unregister_device(dev); 429 goto out_unlock; 430} 431 432EXPORT_SYMBOL_GPL(cpuidle_register_device); 433 434/** 435 * cpuidle_unregister_device - unregisters a CPU's idle PM feature 436 * @dev: the cpu 437 */ 438void cpuidle_unregister_device(struct cpuidle_device *dev) 439{ 440 if (!dev || dev->registered == 0) 441 return; 442 443 cpuidle_pause_and_lock(); 444 445 cpuidle_disable_device(dev); 446 447 cpuidle_remove_sysfs(dev); 448 449 __cpuidle_unregister_device(dev); 450 451 cpuidle_coupled_unregister_device(dev); 452 453 cpuidle_resume_and_unlock(); 454} 455 456EXPORT_SYMBOL_GPL(cpuidle_unregister_device); 457 458/** 459 * cpuidle_unregister: unregister a driver and the devices. This function 460 * can be used only if the driver has been previously registered through 461 * the cpuidle_register function. 462 * 463 * @drv: a valid pointer to a struct cpuidle_driver 464 */ 465void cpuidle_unregister(struct cpuidle_driver *drv) 466{ 467 int cpu; 468 struct cpuidle_device *device; 469 470 for_each_cpu(cpu, drv->cpumask) { 471 device = &per_cpu(cpuidle_dev, cpu); 472 cpuidle_unregister_device(device); 473 } 474 475 cpuidle_unregister_driver(drv); 476} 477EXPORT_SYMBOL_GPL(cpuidle_unregister); 478 479/** 480 * cpuidle_register: registers the driver and the cpu devices with the 481 * coupled_cpus passed as parameter. This function is used for all common 482 * initialization pattern there are in the arch specific drivers. The 483 * devices is globally defined in this file. 484 * 485 * @drv : a valid pointer to a struct cpuidle_driver 486 * @coupled_cpus: a cpumask for the coupled states 487 * 488 * Returns 0 on success, < 0 otherwise 489 */ 490int cpuidle_register(struct cpuidle_driver *drv, 491 const struct cpumask *const coupled_cpus) 492{ 493 int ret, cpu; 494 struct cpuidle_device *device; 495 496 ret = cpuidle_register_driver(drv); 497 if (ret) { 498 pr_err("failed to register cpuidle driver\n"); 499 return ret; 500 } 501 502 for_each_cpu(cpu, drv->cpumask) { 503 device = &per_cpu(cpuidle_dev, cpu); 504 device->cpu = cpu; 505 506#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED 507 /* 508 * On multiplatform for ARM, the coupled idle states could be 509 * enabled in the kernel even if the cpuidle driver does not 510 * use it. Note, coupled_cpus is a struct copy. 511 */ 512 if (coupled_cpus) 513 device->coupled_cpus = *coupled_cpus; 514#endif 515 ret = cpuidle_register_device(device); 516 if (!ret) 517 continue; 518 519 pr_err("Failed to register cpuidle device for cpu%d\n", cpu); 520 521 cpuidle_unregister(drv); 522 break; 523 } 524 525 return ret; 526} 527EXPORT_SYMBOL_GPL(cpuidle_register); 528 529#ifdef CONFIG_SMP 530 531static void smp_callback(void *v) 532{ 533 /* we already woke the CPU up, nothing more to do */ 534} 535 536/* 537 * This function gets called when a part of the kernel has a new latency 538 * requirement. This means we need to get all processors out of their C-state, 539 * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that 540 * wakes them all right up. 541 */ 542static int cpuidle_latency_notify(struct notifier_block *b, 543 unsigned long l, void *v) 544{ 545 smp_call_function(smp_callback, NULL, 1); 546 return NOTIFY_OK; 547} 548 549static struct notifier_block cpuidle_latency_notifier = { 550 .notifier_call = cpuidle_latency_notify, 551}; 552 553static inline void latency_notifier_init(struct notifier_block *n) 554{ 555 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n); 556} 557 558#else /* CONFIG_SMP */ 559 560#define latency_notifier_init(x) do { } while (0) 561 562#endif /* CONFIG_SMP */ 563 564/** 565 * cpuidle_init - core initializer 566 */ 567static int __init cpuidle_init(void) 568{ 569 int ret; 570 571 if (cpuidle_disabled()) 572 return -ENODEV; 573 574 ret = cpuidle_add_interface(cpu_subsys.dev_root); 575 if (ret) 576 return ret; 577 578 latency_notifier_init(&cpuidle_latency_notifier); 579 580 return 0; 581} 582 583module_param(off, int, 0444); 584core_initcall(cpuidle_init); 585