err.no Git - linux-2.6/blob - drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static struct cpufreq_policy *cpufreq_cpu_data[NR_CPUS];
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static struct cpufreq_governor *cpufreq_cpu_governor[NR_CPUS];
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  */
  65 static DEFINE_PER_CPU(int, policy_cpu);
  66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68 #define lock_policy_rwsem(mode, cpu)                                    \
  69 int lock_policy_rwsem_##mode                                            \
  70 (int cpu)                                                               \
  71 {                                                                       \
  72         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73         BUG_ON(policy_cpu == -1);                                       \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75         if (unlikely(!cpu_online(cpu))) {                               \
  76                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                 return -1;                                              \
  78         }                                                               \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86 lock_policy_rwsem(write, cpu);
  87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89 void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97 void unlock_policy_rwsem_write(int cpu)
  98 {
  99         int policy_cpu = per_cpu(policy_cpu, cpu);
 100         BUG_ON(policy_cpu == -1);
 101         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102 }
 103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
 108 static unsigned int __cpufreq_get(unsigned int cpu);
 109 static void handle_update(struct work_struct *work);
 110
 111 /**
 112  * Two notifier lists: the "policy" list is involved in the
 113  * validation process for a new CPU frequency policy; the
 114  * "transition" list for kernel code that needs to handle
 115  * changes to devices when the CPU clock speed changes.
 116  * The mutex locks both lists.
 117  */
 118 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 119 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 120
 121 static int __init init_cpufreq_transition_notifier_list(void)
 122 {
 123         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 124         return 0;
 125 }
 126 pure_initcall(init_cpufreq_transition_notifier_list);
 127
 128 static LIST_HEAD(cpufreq_governor_list);
 129 static DEFINE_MUTEX (cpufreq_governor_mutex);
 130
 131 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 132 {
 133         struct cpufreq_policy *data;
 134         unsigned long flags;
 135
 136         if (cpu >= NR_CPUS)
 137                 goto err_out;
 138
 139         /* get the cpufreq driver */
 140         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 141
 142         if (!cpufreq_driver)
 143                 goto err_out_unlock;
 144
 145         if (!try_module_get(cpufreq_driver->owner))
 146                 goto err_out_unlock;
 147
 148
 149         /* get the CPU */
 150         data = cpufreq_cpu_data[cpu];
 151
 152         if (!data)
 153                 goto err_out_put_module;
 154
 155         if (!kobject_get(&data->kobj))
 156                 goto err_out_put_module;
 157
 158         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 159         return data;
 160
 161 err_out_put_module:
 162         module_put(cpufreq_driver->owner);
 163 err_out_unlock:
 164         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 165 err_out:
 166         return NULL;
 167 }
 168 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 169
 170
 171 void cpufreq_cpu_put(struct cpufreq_policy *data)
 172 {
 173         kobject_put(&data->kobj);
 174         module_put(cpufreq_driver->owner);
 175 }
 176 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 177
 178
 179 /*********************************************************************
 180  *                     UNIFIED DEBUG HELPERS                         *
 181  *********************************************************************/
 182 #ifdef CONFIG_CPU_FREQ_DEBUG
 183
 184 /* what part(s) of the CPUfreq subsystem are debugged? */
 185 static unsigned int debug;
 186
 187 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 188  * set or modify this value.
 189  */
 190 static unsigned int debug_ratelimit = 1;
 191
 192 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 193  * loading of a cpufreq driver, temporarily disabled when a new policy
 194  * is set, and disabled upon cpufreq driver removal
 195  */
 196 static unsigned int disable_ratelimit = 1;
 197 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 198
 199 static void cpufreq_debug_enable_ratelimit(void)
 200 {
 201         unsigned long flags;
 202
 203         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 204         if (disable_ratelimit)
 205                 disable_ratelimit--;
 206         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 207 }
 208
 209 static void cpufreq_debug_disable_ratelimit(void)
 210 {
 211         unsigned long flags;
 212
 213         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 214         disable_ratelimit++;
 215         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 216 }
 217
 218 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 219                                                         const char *fmt, ...)
 220 {
 221         char s[256];
 222         va_list args;
 223         unsigned int len;
 224         unsigned long flags;
 225
 226         WARN_ON(!prefix);
 227         if (type & debug) {
 228                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 229                 if (!disable_ratelimit && debug_ratelimit
 230                                         && !printk_ratelimit()) {
 231                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 232                         return;
 233                 }
 234                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 235
 236                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 237
 238                 va_start(args, fmt);
 239                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 240                 va_end(args);
 241
 242                 printk(s);
 243
 244                 WARN_ON(len < 5);
 245         }
 246 }
 247 EXPORT_SYMBOL(cpufreq_debug_printk);
 248
 249
 250 module_param(debug, uint, 0644);
 251 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 252                         " 2 to debug drivers, and 4 to debug governors.");
 253
 254 module_param(debug_ratelimit, uint, 0644);
 255 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 256                                         " set to 0 to disable ratelimiting.");
 257
 258 #else /* !CONFIG_CPU_FREQ_DEBUG */
 259
 260 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 261 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 262
 263 #endif /* CONFIG_CPU_FREQ_DEBUG */
 264
 265
 266 /*********************************************************************
 267  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 268  *********************************************************************/
 269
 270 /**
 271  * adjust_jiffies - adjust the system "loops_per_jiffy"
 272  *
 273  * This function alters the system "loops_per_jiffy" for the clock
 274  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 275  * systems as each CPU might be scaled differently. So, use the arch
 276  * per-CPU loops_per_jiffy value wherever possible.
 277  */
 278 #ifndef CONFIG_SMP
 279 static unsigned long l_p_j_ref;
 280 static unsigned int  l_p_j_ref_freq;
 281
 282 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 283 {
 284         if (ci->flags & CPUFREQ_CONST_LOOPS)
 285                 return;
 286
 287         if (!l_p_j_ref_freq) {
 288                 l_p_j_ref = loops_per_jiffy;
 289                 l_p_j_ref_freq = ci->old;
 290                 dprintk("saving %lu as reference value for loops_per_jiffy;"
 291                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 292         }
 293         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 294             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 295             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 296                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 297                                                                 ci->new);
 298                 dprintk("scaling loops_per_jiffy to %lu"
 299                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 300         }
 301 }
 302 #else
 303 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 304 {
 305         return;
 306 }
 307 #endif
 308
 309
 310 /**
 311  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 312  * on frequency transition.
 313  *
 314  * This function calls the transition notifiers and the "adjust_jiffies"
 315  * function. It is called twice on all CPU frequency changes that have
 316  * external effects.
 317  */
 318 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 319 {
 320         struct cpufreq_policy *policy;
 321
 322         BUG_ON(irqs_disabled());
 323
 324         freqs->flags = cpufreq_driver->flags;
 325         dprintk("notification %u of frequency transition to %u kHz\n",
 326                 state, freqs->new);
 327
 328         policy = cpufreq_cpu_data[freqs->cpu];
 329         switch (state) {
 330
 331         case CPUFREQ_PRECHANGE:
 332                 /* detect if the driver reported a value as "old frequency"
 333                  * which is not equal to what the cpufreq core thinks is
 334                  * "old frequency".
 335                  */
 336                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 337                         if ((policy) && (policy->cpu == freqs->cpu) &&
 338                             (policy->cur) && (policy->cur != freqs->old)) {
 339                                 dprintk("Warning: CPU frequency is"
 340                                         " %u, cpufreq assumed %u kHz.\n",
 341                                         freqs->old, policy->cur);
 342                                 freqs->old = policy->cur;
 343                         }
 344                 }
 345                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 346                                 CPUFREQ_PRECHANGE, freqs);
 347                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 348                 break;
 349
 350         case CPUFREQ_POSTCHANGE:
 351                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 352                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 353                                 CPUFREQ_POSTCHANGE, freqs);
 354                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 355                         policy->cur = freqs->new;
 356                 break;
 357         }
 358 }
 359 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 360
 361
 362
 363 /*********************************************************************
 364  *                          SYSFS INTERFACE                          *
 365  *********************************************************************/
 366
 367 static struct cpufreq_governor *__find_governor(const char *str_governor)
 368 {
 369         struct cpufreq_governor *t;
 370
 371         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 372                 if (!strnicmp(str_governor,t->name,CPUFREQ_NAME_LEN))
 373                         return t;
 374
 375         return NULL;
 376 }
 377
 378 /**
 379  * cpufreq_parse_governor - parse a governor string
 380  */
 381 static int cpufreq_parse_governor (char *str_governor, unsigned int *policy,
 382                                 struct cpufreq_governor **governor)
 383 {
 384         int err = -EINVAL;
 385
 386         if (!cpufreq_driver)
 387                 goto out;
 388
 389         if (cpufreq_driver->setpolicy) {
 390                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 391                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 392                         err = 0;
 393                 } else if (!strnicmp(str_governor, "powersave",
 394                                                 CPUFREQ_NAME_LEN)) {
 395                         *policy = CPUFREQ_POLICY_POWERSAVE;
 396                         err = 0;
 397                 }
 398         } else if (cpufreq_driver->target) {
 399                 struct cpufreq_governor *t;
 400
 401                 mutex_lock(&cpufreq_governor_mutex);
 402
 403                 t = __find_governor(str_governor);
 404
 405                 if (t == NULL) {
 406                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 407                                                                 str_governor);
 408
 409                         if (name) {
 410                                 int ret;
 411
 412                                 mutex_unlock(&cpufreq_governor_mutex);
 413                                 ret = request_module(name);
 414                                 mutex_lock(&cpufreq_governor_mutex);
 415
 416                                 if (ret == 0)
 417                                         t = __find_governor(str_governor);
 418                         }
 419
 420                         kfree(name);
 421                 }
 422
 423                 if (t != NULL) {
 424                         *governor = t;
 425                         err = 0;
 426                 }
 427
 428                 mutex_unlock(&cpufreq_governor_mutex);
 429         }
 430   out:
 431         return err;
 432 }
 433
 434
 435 /* drivers/base/cpu.c */
 436 extern struct sysdev_class cpu_sysdev_class;
 437
 438
 439 /**
 440  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 441  * print out cpufreq information
 442  *
 443  * Write out information from cpufreq_driver->policy[cpu]; object must be
 444  * "unsigned int".
 445  */
 446
 447 #define show_one(file_name, object)                     \
 448 static ssize_t show_##file_name                         \
 449 (struct cpufreq_policy * policy, char *buf)             \
 450 {                                                       \
 451         return sprintf (buf, "%u\n", policy->object);   \
 452 }
 453
 454 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 455 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 456 show_one(scaling_min_freq, min);
 457 show_one(scaling_max_freq, max);
 458 show_one(scaling_cur_freq, cur);
 459
 460 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 461                                 struct cpufreq_policy *policy);
 462
 463 /**
 464  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 465  */
 466 #define store_one(file_name, object)                    \
 467 static ssize_t store_##file_name                                        \
 468 (struct cpufreq_policy * policy, const char *buf, size_t count)         \
 469 {                                                                       \
 470         unsigned int ret = -EINVAL;                                     \
 471         struct cpufreq_policy new_policy;                               \
 472                                                                         \
 473         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 474         if (ret)                                                        \
 475                 return -EINVAL;                                         \
 476                                                                         \
 477         ret = sscanf (buf, "%u", &new_policy.object);                   \
 478         if (ret != 1)                                                   \
 479                 return -EINVAL;                                         \
 480                                                                         \
 481         ret = __cpufreq_set_policy(policy, &new_policy);                \
 482         policy->user_policy.object = policy->object;                    \
 483                                                                         \
 484         return ret ? ret : count;                                       \
 485 }
 486
 487 store_one(scaling_min_freq,min);
 488 store_one(scaling_max_freq,max);
 489
 490 /**
 491  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 492  */
 493 static ssize_t show_cpuinfo_cur_freq (struct cpufreq_policy * policy,
 494                                                         char *buf)
 495 {
 496         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 497         if (!cur_freq)
 498                 return sprintf(buf, "<unknown>");
 499         return sprintf(buf, "%u\n", cur_freq);
 500 }
 501
 502
 503 /**
 504  * show_scaling_governor - show the current policy for the specified CPU
 505  */
 506 static ssize_t show_scaling_governor (struct cpufreq_policy * policy,
 507                                                         char *buf)
 508 {
 509         if(policy->policy == CPUFREQ_POLICY_POWERSAVE)
 510                 return sprintf(buf, "powersave\n");
 511         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 512                 return sprintf(buf, "performance\n");
 513         else if (policy->governor)
 514                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", policy->governor->name);
 515         return -EINVAL;
 516 }
 517
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor (struct cpufreq_policy * policy,
 523                                        const char *buf, size_t count)
 524 {
 525         unsigned int ret = -EINVAL;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf (buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /* Do not use cpufreq_set_policy here or the user_policy.max
 542            will be wrongly overridden */
 543         ret = __cpufreq_set_policy(policy, &new_policy);
 544
 545         policy->user_policy.policy = policy->policy;
 546         policy->user_policy.governor = policy->governor;
 547
 548         if (ret)
 549                 return ret;
 550         else
 551                 return count;
 552 }
 553
 554 /**
 555  * show_scaling_driver - show the cpufreq driver currently loaded
 556  */
 557 static ssize_t show_scaling_driver (struct cpufreq_policy * policy, char *buf)
 558 {
 559         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 560 }
 561
 562 /**
 563  * show_scaling_available_governors - show the available CPUfreq governors
 564  */
 565 static ssize_t show_scaling_available_governors (struct cpufreq_policy *policy,
 566                                 char *buf)
 567 {
 568         ssize_t i = 0;
 569         struct cpufreq_governor *t;
 570
 571         if (!cpufreq_driver->target) {
 572                 i += sprintf(buf, "performance powersave");
 573                 goto out;
 574         }
 575
 576         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 577                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) - (CPUFREQ_NAME_LEN + 2)))
 578                         goto out;
 579                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 580         }
 581 out:
 582         i += sprintf(&buf[i], "\n");
 583         return i;
 584 }
 585 /**
 586  * show_affected_cpus - show the CPUs affected by each transition
 587  */
 588 static ssize_t show_affected_cpus (struct cpufreq_policy * policy, char *buf)
 589 {
 590         ssize_t i = 0;
 591         unsigned int cpu;
 592
 593         for_each_cpu_mask(cpu, policy->cpus) {
 594                 if (i)
 595                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 596                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 597                 if (i >= (PAGE_SIZE - 5))
 598                     break;
 599         }
 600         i += sprintf(&buf[i], "\n");
 601         return i;
 602 }
 603
 604
 605 #define define_one_ro(_name) \
 606 static struct freq_attr _name = \
 607 __ATTR(_name, 0444, show_##_name, NULL)
 608
 609 #define define_one_ro0400(_name) \
 610 static struct freq_attr _name = \
 611 __ATTR(_name, 0400, show_##_name, NULL)
 612
 613 #define define_one_rw(_name) \
 614 static struct freq_attr _name = \
 615 __ATTR(_name, 0644, show_##_name, store_##_name)
 616
 617 define_one_ro0400(cpuinfo_cur_freq);
 618 define_one_ro(cpuinfo_min_freq);
 619 define_one_ro(cpuinfo_max_freq);
 620 define_one_ro(scaling_available_governors);
 621 define_one_ro(scaling_driver);
 622 define_one_ro(scaling_cur_freq);
 623 define_one_ro(affected_cpus);
 624 define_one_rw(scaling_min_freq);
 625 define_one_rw(scaling_max_freq);
 626 define_one_rw(scaling_governor);
 627
 628 static struct attribute * default_attrs[] = {
 629         &cpuinfo_min_freq.attr,
 630         &cpuinfo_max_freq.attr,
 631         &scaling_min_freq.attr,
 632         &scaling_max_freq.attr,
 633         &affected_cpus.attr,
 634         &scaling_governor.attr,
 635         &scaling_driver.attr,
 636         &scaling_available_governors.attr,
 637         NULL
 638 };
 639
 640 #define to_policy(k) container_of(k,struct cpufreq_policy,kobj)
 641 #define to_attr(a) container_of(a,struct freq_attr,attr)
 642
 643 static ssize_t show(struct kobject * kobj, struct attribute * attr ,char * buf)
 644 {
 645         struct cpufreq_policy * policy = to_policy(kobj);
 646         struct freq_attr * fattr = to_attr(attr);
 647         ssize_t ret;
 648         policy = cpufreq_cpu_get(policy->cpu);
 649         if (!policy)
 650                 return -EINVAL;
 651
 652         if (lock_policy_rwsem_read(policy->cpu) < 0)
 653                 return -EINVAL;
 654
 655         if (fattr->show)
 656                 ret = fattr->show(policy, buf);
 657         else
 658                 ret = -EIO;
 659
 660         unlock_policy_rwsem_read(policy->cpu);
 661
 662         cpufreq_cpu_put(policy);
 663         return ret;
 664 }
 665
 666 static ssize_t store(struct kobject * kobj, struct attribute * attr,
 667                      const char * buf, size_t count)
 668 {
 669         struct cpufreq_policy * policy = to_policy(kobj);
 670         struct freq_attr * fattr = to_attr(attr);
 671         ssize_t ret;
 672         policy = cpufreq_cpu_get(policy->cpu);
 673         if (!policy)
 674                 return -EINVAL;
 675
 676         if (lock_policy_rwsem_write(policy->cpu) < 0)
 677                 return -EINVAL;
 678
 679         if (fattr->store)
 680                 ret = fattr->store(policy, buf, count);
 681         else
 682                 ret = -EIO;
 683
 684         unlock_policy_rwsem_write(policy->cpu);
 685
 686         cpufreq_cpu_put(policy);
 687         return ret;
 688 }
 689
 690 static void cpufreq_sysfs_release(struct kobject * kobj)
 691 {
 692         struct cpufreq_policy * policy = to_policy(kobj);
 693         dprintk("last reference is dropped\n");
 694         complete(&policy->kobj_unregister);
 695 }
 696
 697 static struct sysfs_ops sysfs_ops = {
 698         .show   = show,
 699         .store  = store,
 700 };
 701
 702 static struct kobj_type ktype_cpufreq = {
 703         .sysfs_ops      = &sysfs_ops,
 704         .default_attrs  = default_attrs,
 705         .release        = cpufreq_sysfs_release,
 706 };
 707
 708
 709 /**
 710  * cpufreq_add_dev - add a CPU device
 711  *
 712  * Adds the cpufreq interface for a CPU device.
 713  */
 714 static int cpufreq_add_dev (struct sys_device * sys_dev)
 715 {
 716         unsigned int cpu = sys_dev->id;
 717         int ret = 0;
 718         struct cpufreq_policy new_policy;
 719         struct cpufreq_policy *policy;
 720         struct freq_attr **drv_attr;
 721         struct sys_device *cpu_sys_dev;
 722         unsigned long flags;
 723         unsigned int j;
 724 #ifdef CONFIG_SMP
 725         struct cpufreq_policy *managed_policy;
 726 #endif
 727
 728         if (cpu_is_offline(cpu))
 729                 return 0;
 730
 731         cpufreq_debug_disable_ratelimit();
 732         dprintk("adding CPU %u\n", cpu);
 733
 734 #ifdef CONFIG_SMP
 735         /* check whether a different CPU already registered this
 736          * CPU because it is in the same boat. */
 737         policy = cpufreq_cpu_get(cpu);
 738         if (unlikely(policy)) {
 739                 cpufreq_cpu_put(policy);
 740                 cpufreq_debug_enable_ratelimit();
 741                 return 0;
 742         }
 743 #endif
 744
 745         if (!try_module_get(cpufreq_driver->owner)) {
 746                 ret = -EINVAL;
 747                 goto module_out;
 748         }
 749
 750         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 751         if (!policy) {
 752                 ret = -ENOMEM;
 753                 goto nomem_out;
 754         }
 755
 756         policy->cpu = cpu;
 757         policy->cpus = cpumask_of_cpu(cpu);
 758
 759         /* Initially set CPU itself as the policy_cpu */
 760         per_cpu(policy_cpu, cpu) = cpu;
 761         lock_policy_rwsem_write(cpu);
 762
 763         init_completion(&policy->kobj_unregister);
 764         INIT_WORK(&policy->update, handle_update);
 765
 766         /* Set governor before ->init, so that driver could check it */
 767         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 768         /* call driver. From then on the cpufreq must be able
 769          * to accept all calls to ->verify and ->setpolicy for this CPU
 770          */
 771         ret = cpufreq_driver->init(policy);
 772         if (ret) {
 773                 dprintk("initialization failed\n");
 774                 unlock_policy_rwsem_write(cpu);
 775                 goto err_out;
 776         }
 777         policy->user_policy.min = policy->cpuinfo.min_freq;
 778         policy->user_policy.max = policy->cpuinfo.max_freq;
 779
 780 #ifdef CONFIG_SMP
 781
 782 #ifdef CONFIG_HOTPLUG_CPU
 783         if (cpufreq_cpu_governor[cpu]){
 784                 policy->governor = cpufreq_cpu_governor[cpu];
 785                 dprintk("Restoring governor %s for cpu %d\n",
 786                        policy->governor->name, cpu);
 787         }
 788 #endif
 789
 790         for_each_cpu_mask(j, policy->cpus) {
 791                 if (cpu == j)
 792                         continue;
 793
 794                 /* check for existing affected CPUs.  They may not be aware
 795                  * of it due to CPU Hotplug.
 796                  */
 797                 managed_policy = cpufreq_cpu_get(j);
 798                 if (unlikely(managed_policy)) {
 799
 800                         /* Set proper policy_cpu */
 801                         unlock_policy_rwsem_write(cpu);
 802                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 803
 804                         if (lock_policy_rwsem_write(cpu) < 0)
 805                                 goto err_out_driver_exit;
 806
 807                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 808                         managed_policy->cpus = policy->cpus;
 809                         cpufreq_cpu_data[cpu] = managed_policy;
 810                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 811
 812                         dprintk("CPU already managed, adding link\n");
 813                         ret = sysfs_create_link(&sys_dev->kobj,
 814                                                 &managed_policy->kobj,
 815                                                 "cpufreq");
 816                         if (ret) {
 817                                 unlock_policy_rwsem_write(cpu);
 818                                 goto err_out_driver_exit;
 819                         }
 820
 821                         cpufreq_debug_enable_ratelimit();
 822                         ret = 0;
 823                         unlock_policy_rwsem_write(cpu);
 824                         goto err_out_driver_exit; /* call driver->exit() */
 825                 }
 826         }
 827 #endif
 828         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 829
 830         /* prepare interface data */
 831         policy->kobj.parent = &sys_dev->kobj;
 832         policy->kobj.ktype = &ktype_cpufreq;
 833         kobject_set_name(&policy->kobj, "cpufreq");
 834
 835         ret = kobject_register(&policy->kobj);
 836         if (ret) {
 837                 unlock_policy_rwsem_write(cpu);
 838                 goto err_out_driver_exit;
 839         }
 840         /* set up files for this cpu device */
 841         drv_attr = cpufreq_driver->attr;
 842         while ((drv_attr) && (*drv_attr)) {
 843                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 844                 if (ret) {
 845                         unlock_policy_rwsem_write(cpu);
 846                         goto err_out_driver_exit;
 847                 }
 848                 drv_attr++;
 849         }
 850         if (cpufreq_driver->get){
 851                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 852                 if (ret) {
 853                         unlock_policy_rwsem_write(cpu);
 854                         goto err_out_driver_exit;
 855                 }
 856         }
 857         if (cpufreq_driver->target){
 858                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 859                 if (ret) {
 860                         unlock_policy_rwsem_write(cpu);
 861                         goto err_out_driver_exit;
 862                 }
 863         }
 864
 865         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 866         for_each_cpu_mask(j, policy->cpus) {
 867                 cpufreq_cpu_data[j] = policy;
 868                 per_cpu(policy_cpu, j) = policy->cpu;
 869         }
 870         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 871
 872         /* symlink affected CPUs */
 873         for_each_cpu_mask(j, policy->cpus) {
 874                 if (j == cpu)
 875                         continue;
 876                 if (!cpu_online(j))
 877                         continue;
 878
 879                 dprintk("CPU %u already managed, adding link\n", j);
 880                 cpufreq_cpu_get(cpu);
 881                 cpu_sys_dev = get_cpu_sysdev(j);
 882                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 883                                         "cpufreq");
 884                 if (ret) {
 885                         unlock_policy_rwsem_write(cpu);
 886                         goto err_out_unregister;
 887                 }
 888         }
 889
 890         policy->governor = NULL; /* to assure that the starting sequence is
 891                                   * run in cpufreq_set_policy */
 892
 893         /* set default policy */
 894         ret = __cpufreq_set_policy(policy, &new_policy);
 895         policy->user_policy.policy = policy->policy;
 896         policy->user_policy.governor = policy->governor;
 897
 898         unlock_policy_rwsem_write(cpu);
 899
 900         if (ret) {
 901                 dprintk("setting policy failed\n");
 902                 goto err_out_unregister;
 903         }
 904
 905         module_put(cpufreq_driver->owner);
 906         dprintk("initialization complete\n");
 907         cpufreq_debug_enable_ratelimit();
 908
 909         return 0;
 910
 911
 912 err_out_unregister:
 913         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 914         for_each_cpu_mask(j, policy->cpus)
 915                 cpufreq_cpu_data[j] = NULL;
 916         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 917
 918         kobject_unregister(&policy->kobj);
 919         wait_for_completion(&policy->kobj_unregister);
 920
 921 err_out_driver_exit:
 922         if (cpufreq_driver->exit)
 923                 cpufreq_driver->exit(policy);
 924
 925 err_out:
 926         kfree(policy);
 927
 928 nomem_out:
 929         module_put(cpufreq_driver->owner);
 930 module_out:
 931         cpufreq_debug_enable_ratelimit();
 932         return ret;
 933 }
 934
 935
 936 /**
 937  * __cpufreq_remove_dev - remove a CPU device
 938  *
 939  * Removes the cpufreq interface for a CPU device.
 940  * Caller should already have policy_rwsem in write mode for this CPU.
 941  * This routine frees the rwsem before returning.
 942  */
 943 static int __cpufreq_remove_dev (struct sys_device * sys_dev)
 944 {
 945         unsigned int cpu = sys_dev->id;
 946         unsigned long flags;
 947         struct cpufreq_policy *data;
 948 #ifdef CONFIG_SMP
 949         struct sys_device *cpu_sys_dev;
 950         unsigned int j;
 951 #endif
 952
 953         cpufreq_debug_disable_ratelimit();
 954         dprintk("unregistering CPU %u\n", cpu);
 955
 956         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 957         data = cpufreq_cpu_data[cpu];
 958
 959         if (!data) {
 960                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 961                 cpufreq_debug_enable_ratelimit();
 962                 unlock_policy_rwsem_write(cpu);
 963                 return -EINVAL;
 964         }
 965         cpufreq_cpu_data[cpu] = NULL;
 966
 967
 968 #ifdef CONFIG_SMP
 969         /* if this isn't the CPU which is the parent of the kobj, we
 970          * only need to unlink, put and exit
 971          */
 972         if (unlikely(cpu != data->cpu)) {
 973                 dprintk("removing link\n");
 974                 cpu_clear(cpu, data->cpus);
 975                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 976                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
 977                 cpufreq_cpu_put(data);
 978                 cpufreq_debug_enable_ratelimit();
 979                 unlock_policy_rwsem_write(cpu);
 980                 return 0;
 981         }
 982 #endif
 983
 984
 985         if (!kobject_get(&data->kobj)) {
 986                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 987                 cpufreq_debug_enable_ratelimit();
 988                 unlock_policy_rwsem_write(cpu);
 989                 return -EFAULT;
 990         }
 991
 992 #ifdef CONFIG_SMP
 993
 994 #ifdef CONFIG_HOTPLUG_CPU
 995         cpufreq_cpu_governor[cpu] = data->governor;
 996 #endif
 997
 998         /* if we have other CPUs still registered, we need to unlink them,
 999          * or else wait_for_completion below will lock up. Clean the
1000          * cpufreq_cpu_data[] while holding the lock, and remove the sysfs
1001          * links afterwards.
1002          */
1003         if (unlikely(cpus_weight(data->cpus) > 1)) {
1004                 for_each_cpu_mask(j, data->cpus) {
1005                         if (j == cpu)
1006                                 continue;
1007                         cpufreq_cpu_data[j] = NULL;
1008                 }
1009         }
1010
1011         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1012
1013         if (unlikely(cpus_weight(data->cpus) > 1)) {
1014                 for_each_cpu_mask(j, data->cpus) {
1015                         if (j == cpu)
1016                                 continue;
1017                         dprintk("removing link for cpu %u\n", j);
1018 #ifdef CONFIG_HOTPLUG_CPU
1019                         cpufreq_cpu_governor[j] = data->governor;
1020 #endif
1021                         cpu_sys_dev = get_cpu_sysdev(j);
1022                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1023                         cpufreq_cpu_put(data);
1024                 }
1025         }
1026 #else
1027         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1028 #endif
1029
1030         if (cpufreq_driver->target)
1031                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1032
1033         unlock_policy_rwsem_write(cpu);
1034
1035         kobject_unregister(&data->kobj);
1036
1037         kobject_put(&data->kobj);
1038
1039         /* we need to make sure that the underlying kobj is actually
1040          * not referenced anymore by anybody before we proceed with
1041          * unloading.
1042          */
1043         dprintk("waiting for dropping of refcount\n");
1044         wait_for_completion(&data->kobj_unregister);
1045         dprintk("wait complete\n");
1046
1047         if (cpufreq_driver->exit)
1048                 cpufreq_driver->exit(data);
1049
1050         kfree(data);
1051
1052         cpufreq_debug_enable_ratelimit();
1053         return 0;
1054 }
1055
1056
1057 static int cpufreq_remove_dev (struct sys_device * sys_dev)
1058 {
1059         unsigned int cpu = sys_dev->id;
1060         int retval;
1061
1062         if (cpu_is_offline(cpu))
1063                 return 0;
1064
1065         if (unlikely(lock_policy_rwsem_write(cpu)))
1066                 BUG();
1067
1068         retval = __cpufreq_remove_dev(sys_dev);
1069         return retval;
1070 }
1071
1072
1073 static void handle_update(struct work_struct *work)
1074 {
1075         struct cpufreq_policy *policy =
1076                 container_of(work, struct cpufreq_policy, update);
1077         unsigned int cpu = policy->cpu;
1078         dprintk("handle_update for cpu %u called\n", cpu);
1079         cpufreq_update_policy(cpu);
1080 }
1081
1082 /**
1083  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1084  *      @cpu: cpu number
1085  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1086  *      @new_freq: CPU frequency the CPU actually runs at
1087  *
1088  *      We adjust to current frequency first, and need to clean up later. So either call
1089  *      to cpufreq_update_policy() or schedule handle_update()).
1090  */
1091 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1092                                 unsigned int new_freq)
1093 {
1094         struct cpufreq_freqs freqs;
1095
1096         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1097                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1098
1099         freqs.cpu = cpu;
1100         freqs.old = old_freq;
1101         freqs.new = new_freq;
1102         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1103         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1104 }
1105
1106
1107 /**
1108  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1109  * @cpu: CPU number
1110  *
1111  * This is the last known freq, without actually getting it from the driver.
1112  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1113  */
1114 unsigned int cpufreq_quick_get(unsigned int cpu)
1115 {
1116         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1117         unsigned int ret_freq = 0;
1118
1119         if (policy) {
1120                 ret_freq = policy->cur;
1121                 cpufreq_cpu_put(policy);
1122         }
1123
1124         return (ret_freq);
1125 }
1126 EXPORT_SYMBOL(cpufreq_quick_get);
1127
1128
1129 static unsigned int __cpufreq_get(unsigned int cpu)
1130 {
1131         struct cpufreq_policy *policy = cpufreq_cpu_data[cpu];
1132         unsigned int ret_freq = 0;
1133
1134         if (!cpufreq_driver->get)
1135                 return (ret_freq);
1136
1137         ret_freq = cpufreq_driver->get(cpu);
1138
1139         if (ret_freq && policy->cur &&
1140                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1141                 /* verify no discrepancy between actual and
1142                                         saved value exists */
1143                 if (unlikely(ret_freq != policy->cur)) {
1144                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1145                         schedule_work(&policy->update);
1146                 }
1147         }
1148
1149         return (ret_freq);
1150 }
1151
1152 /**
1153  * cpufreq_get - get the current CPU frequency (in kHz)
1154  * @cpu: CPU number
1155  *
1156  * Get the CPU current (static) CPU frequency
1157  */
1158 unsigned int cpufreq_get(unsigned int cpu)
1159 {
1160         unsigned int ret_freq = 0;
1161         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1162
1163         if (!policy)
1164                 goto out;
1165
1166         if (unlikely(lock_policy_rwsem_read(cpu)))
1167                 goto out_policy;
1168
1169         ret_freq = __cpufreq_get(cpu);
1170
1171         unlock_policy_rwsem_read(cpu);
1172
1173 out_policy:
1174         cpufreq_cpu_put(policy);
1175 out:
1176         return (ret_freq);
1177 }
1178 EXPORT_SYMBOL(cpufreq_get);
1179
1180
1181 /**
1182  *      cpufreq_suspend - let the low level driver prepare for suspend
1183  */
1184
1185 static int cpufreq_suspend(struct sys_device * sysdev, pm_message_t pmsg)
1186 {
1187         int cpu = sysdev->id;
1188         int ret = 0;
1189         unsigned int cur_freq = 0;
1190         struct cpufreq_policy *cpu_policy;
1191
1192         dprintk("suspending cpu %u\n", cpu);
1193
1194         if (!cpu_online(cpu))
1195                 return 0;
1196
1197         /* we may be lax here as interrupts are off. Nonetheless
1198          * we need to grab the correct cpu policy, as to check
1199          * whether we really run on this CPU.
1200          */
1201
1202         cpu_policy = cpufreq_cpu_get(cpu);
1203         if (!cpu_policy)
1204                 return -EINVAL;
1205
1206         /* only handle each CPU group once */
1207         if (unlikely(cpu_policy->cpu != cpu)) {
1208                 cpufreq_cpu_put(cpu_policy);
1209                 return 0;
1210         }
1211
1212         if (cpufreq_driver->suspend) {
1213                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1214                 if (ret) {
1215                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1216                                         "step on CPU %u\n", cpu_policy->cpu);
1217                         cpufreq_cpu_put(cpu_policy);
1218                         return ret;
1219                 }
1220         }
1221
1222
1223         if (cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)
1224                 goto out;
1225
1226         if (cpufreq_driver->get)
1227                 cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1228
1229         if (!cur_freq || !cpu_policy->cur) {
1230                 printk(KERN_ERR "cpufreq: suspend failed to assert current "
1231                        "frequency is what timing core thinks it is.\n");
1232                 goto out;
1233         }
1234
1235         if (unlikely(cur_freq != cpu_policy->cur)) {
1236                 struct cpufreq_freqs freqs;
1237
1238                 if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1239                         dprintk("Warning: CPU frequency is %u, "
1240                                "cpufreq assumed %u kHz.\n",
1241                                cur_freq, cpu_policy->cur);
1242
1243                 freqs.cpu = cpu;
1244                 freqs.old = cpu_policy->cur;
1245                 freqs.new = cur_freq;
1246
1247                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
1248                                     CPUFREQ_SUSPENDCHANGE, &freqs);
1249                 adjust_jiffies(CPUFREQ_SUSPENDCHANGE, &freqs);
1250
1251                 cpu_policy->cur = cur_freq;
1252         }
1253
1254 out:
1255         cpufreq_cpu_put(cpu_policy);
1256         return 0;
1257 }
1258
1259 /**
1260  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1261  *
1262  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1263  *      2.) if ->target and !CPUFREQ_CONST_LOOPS: verify we're in sync
1264  *      3.) schedule call cpufreq_update_policy() ASAP as interrupts are
1265  *          restored.
1266  */
1267 static int cpufreq_resume(struct sys_device * sysdev)
1268 {
1269         int cpu = sysdev->id;
1270         int ret = 0;
1271         struct cpufreq_policy *cpu_policy;
1272
1273         dprintk("resuming cpu %u\n", cpu);
1274
1275         if (!cpu_online(cpu))
1276                 return 0;
1277
1278         /* we may be lax here as interrupts are off. Nonetheless
1279          * we need to grab the correct cpu policy, as to check
1280          * whether we really run on this CPU.
1281          */
1282
1283         cpu_policy = cpufreq_cpu_get(cpu);
1284         if (!cpu_policy)
1285                 return -EINVAL;
1286
1287         /* only handle each CPU group once */
1288         if (unlikely(cpu_policy->cpu != cpu)) {
1289                 cpufreq_cpu_put(cpu_policy);
1290                 return 0;
1291         }
1292
1293         if (cpufreq_driver->resume) {
1294                 ret = cpufreq_driver->resume(cpu_policy);
1295                 if (ret) {
1296                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1297                                         "step on CPU %u\n", cpu_policy->cpu);
1298                         cpufreq_cpu_put(cpu_policy);
1299                         return ret;
1300                 }
1301         }
1302
1303         if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1304                 unsigned int cur_freq = 0;
1305
1306                 if (cpufreq_driver->get)
1307                         cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1308
1309                 if (!cur_freq || !cpu_policy->cur) {
1310                         printk(KERN_ERR "cpufreq: resume failed to assert "
1311                                         "current frequency is what timing core "
1312                                         "thinks it is.\n");
1313                         goto out;
1314                 }
1315
1316                 if (unlikely(cur_freq != cpu_policy->cur)) {
1317                         struct cpufreq_freqs freqs;
1318
1319                         if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1320                                 dprintk("Warning: CPU frequency"
1321                                        "is %u, cpufreq assumed %u kHz.\n",
1322                                        cur_freq, cpu_policy->cur);
1323
1324                         freqs.cpu = cpu;
1325                         freqs.old = cpu_policy->cur;
1326                         freqs.new = cur_freq;
1327
1328                         srcu_notifier_call_chain(
1329                                         &cpufreq_transition_notifier_list,
1330                                         CPUFREQ_RESUMECHANGE, &freqs);
1331                         adjust_jiffies(CPUFREQ_RESUMECHANGE, &freqs);
1332
1333                         cpu_policy->cur = cur_freq;
1334                 }
1335         }
1336
1337 out:
1338         schedule_work(&cpu_policy->update);
1339         cpufreq_cpu_put(cpu_policy);
1340         return ret;
1341 }
1342
1343 static struct sysdev_driver cpufreq_sysdev_driver = {
1344         .add            = cpufreq_add_dev,
1345         .remove         = cpufreq_remove_dev,
1346         .suspend        = cpufreq_suspend,
1347         .resume         = cpufreq_resume,
1348 };
1349
1350
1351 /*********************************************************************
1352  *                     NOTIFIER LISTS INTERFACE                      *
1353  *********************************************************************/
1354
1355 /**
1356  *      cpufreq_register_notifier - register a driver with cpufreq
1357  *      @nb: notifier function to register
1358  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1359  *
1360  *      Add a driver to one of two lists: either a list of drivers that
1361  *      are notified about clock rate changes (once before and once after
1362  *      the transition), or a list of drivers that are notified about
1363  *      changes in cpufreq policy.
1364  *
1365  *      This function may sleep, and has the same return conditions as
1366  *      blocking_notifier_chain_register.
1367  */
1368 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1369 {
1370         int ret;
1371
1372         switch (list) {
1373         case CPUFREQ_TRANSITION_NOTIFIER:
1374                 ret = srcu_notifier_chain_register(
1375                                 &cpufreq_transition_notifier_list, nb);
1376                 break;
1377         case CPUFREQ_POLICY_NOTIFIER:
1378                 ret = blocking_notifier_chain_register(
1379                                 &cpufreq_policy_notifier_list, nb);
1380                 break;
1381         default:
1382                 ret = -EINVAL;
1383         }
1384
1385         return ret;
1386 }
1387 EXPORT_SYMBOL(cpufreq_register_notifier);
1388
1389
1390 /**
1391  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1392  *      @nb: notifier block to be unregistered
1393  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1394  *
1395  *      Remove a driver from the CPU frequency notifier list.
1396  *
1397  *      This function may sleep, and has the same return conditions as
1398  *      blocking_notifier_chain_unregister.
1399  */
1400 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1401 {
1402         int ret;
1403
1404         switch (list) {
1405         case CPUFREQ_TRANSITION_NOTIFIER:
1406                 ret = srcu_notifier_chain_unregister(
1407                                 &cpufreq_transition_notifier_list, nb);
1408                 break;
1409         case CPUFREQ_POLICY_NOTIFIER:
1410                 ret = blocking_notifier_chain_unregister(
1411                                 &cpufreq_policy_notifier_list, nb);
1412                 break;
1413         default:
1414                 ret = -EINVAL;
1415         }
1416
1417         return ret;
1418 }
1419 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1420
1421
1422 /*********************************************************************
1423  *                              GOVERNORS                            *
1424  *********************************************************************/
1425
1426
1427 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1428                             unsigned int target_freq,
1429                             unsigned int relation)
1430 {
1431         int retval = -EINVAL;
1432
1433         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1434                 target_freq, relation);
1435         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1436                 retval = cpufreq_driver->target(policy, target_freq, relation);
1437
1438         return retval;
1439 }
1440 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1441
1442 int cpufreq_driver_target(struct cpufreq_policy *policy,
1443                           unsigned int target_freq,
1444                           unsigned int relation)
1445 {
1446         int ret;
1447
1448         policy = cpufreq_cpu_get(policy->cpu);
1449         if (!policy)
1450                 return -EINVAL;
1451
1452         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1453                 return -EINVAL;
1454
1455         ret = __cpufreq_driver_target(policy, target_freq, relation);
1456
1457         unlock_policy_rwsem_write(policy->cpu);
1458
1459         cpufreq_cpu_put(policy);
1460         return ret;
1461 }
1462 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1463
1464 int __cpufreq_driver_getavg(struct cpufreq_policy *policy)
1465 {
1466         int ret = 0;
1467
1468         policy = cpufreq_cpu_get(policy->cpu);
1469         if (!policy)
1470                 return -EINVAL;
1471
1472         if (cpu_online(policy->cpu) && cpufreq_driver->getavg)
1473                 ret = cpufreq_driver->getavg(policy->cpu);
1474
1475         cpufreq_cpu_put(policy);
1476         return ret;
1477 }
1478 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1479
1480 /*
1481  * when "event" is CPUFREQ_GOV_LIMITS
1482  */
1483
1484 static int __cpufreq_governor(struct cpufreq_policy *policy,
1485                                         unsigned int event)
1486 {
1487         int ret;
1488
1489         /* Only must be defined when default governor is known to have latency
1490            restrictions, like e.g. conservative or ondemand.
1491            That this is the case is already ensured in Kconfig
1492         */
1493 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1494         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1495 #else
1496         struct cpufreq_governor *gov = NULL;
1497 #endif
1498
1499         if (policy->governor->max_transition_latency &&
1500             policy->cpuinfo.transition_latency >
1501             policy->governor->max_transition_latency) {
1502                 if (!gov)
1503                         return -EINVAL;
1504                 else {
1505                         printk(KERN_WARNING "%s governor failed, too long"
1506                                " transition latency of HW, fallback"
1507                                " to %s governor\n",
1508                                policy->governor->name,
1509                                gov->name);
1510                         policy->governor = gov;
1511                 }
1512         }
1513
1514         if (!try_module_get(policy->governor->owner))
1515                 return -EINVAL;
1516
1517         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1518                                                 policy->cpu, event);
1519         ret = policy->governor->governor(policy, event);
1520
1521         /* we keep one module reference alive for
1522                         each CPU governed by this CPU */
1523         if ((event != CPUFREQ_GOV_START) || ret)
1524                 module_put(policy->governor->owner);
1525         if ((event == CPUFREQ_GOV_STOP) && !ret)
1526                 module_put(policy->governor->owner);
1527
1528         return ret;
1529 }
1530
1531
1532 int cpufreq_register_governor(struct cpufreq_governor *governor)
1533 {
1534         int err;
1535
1536         if (!governor)
1537                 return -EINVAL;
1538
1539         mutex_lock(&cpufreq_governor_mutex);
1540
1541         err = -EBUSY;
1542         if (__find_governor(governor->name) == NULL) {
1543                 err = 0;
1544                 list_add(&governor->governor_list, &cpufreq_governor_list);
1545         }
1546
1547         mutex_unlock(&cpufreq_governor_mutex);
1548         return err;
1549 }
1550 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1551
1552
1553 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1554 {
1555         if (!governor)
1556                 return;
1557
1558         mutex_lock(&cpufreq_governor_mutex);
1559         list_del(&governor->governor_list);
1560         mutex_unlock(&cpufreq_governor_mutex);
1561         return;
1562 }
1563 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1564
1565
1566
1567 /*********************************************************************
1568  *                          POLICY INTERFACE                         *
1569  *********************************************************************/
1570
1571 /**
1572  * cpufreq_get_policy - get the current cpufreq_policy
1573  * @policy: struct cpufreq_policy into which the current cpufreq_policy is written
1574  *
1575  * Reads the current cpufreq policy.
1576  */
1577 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1578 {
1579         struct cpufreq_policy *cpu_policy;
1580         if (!policy)
1581                 return -EINVAL;
1582
1583         cpu_policy = cpufreq_cpu_get(cpu);
1584         if (!cpu_policy)
1585                 return -EINVAL;
1586
1587         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1588
1589         cpufreq_cpu_put(cpu_policy);
1590         return 0;
1591 }
1592 EXPORT_SYMBOL(cpufreq_get_policy);
1593
1594
1595 /*
1596  * data   : current policy.
1597  * policy : policy to be set.
1598  */
1599 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1600                                 struct cpufreq_policy *policy)
1601 {
1602         int ret = 0;
1603
1604         cpufreq_debug_disable_ratelimit();
1605         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1606                 policy->min, policy->max);
1607
1608         memcpy(&policy->cpuinfo, &data->cpuinfo,
1609                                 sizeof(struct cpufreq_cpuinfo));
1610
1611         if (policy->min > data->min && policy->min > policy->max) {
1612                 ret = -EINVAL;
1613                 goto error_out;
1614         }
1615
1616         /* verify the cpu speed can be set within this limit */
1617         ret = cpufreq_driver->verify(policy);
1618         if (ret)
1619                 goto error_out;
1620
1621         /* adjust if necessary - all reasons */
1622         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1623                         CPUFREQ_ADJUST, policy);
1624
1625         /* adjust if necessary - hardware incompatibility*/
1626         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1627                         CPUFREQ_INCOMPATIBLE, policy);
1628
1629         /* verify the cpu speed can be set within this limit,
1630            which might be different to the first one */
1631         ret = cpufreq_driver->verify(policy);
1632         if (ret)
1633                 goto error_out;
1634
1635         /* notification of the new policy */
1636         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1637                         CPUFREQ_NOTIFY, policy);
1638
1639         data->min = policy->min;
1640         data->max = policy->max;
1641
1642         dprintk("new min and max freqs are %u - %u kHz\n",
1643                                         data->min, data->max);
1644
1645         if (cpufreq_driver->setpolicy) {
1646                 data->policy = policy->policy;
1647                 dprintk("setting range\n");
1648                 ret = cpufreq_driver->setpolicy(policy);
1649         } else {
1650                 if (policy->governor != data->governor) {
1651                         /* save old, working values */
1652                         struct cpufreq_governor *old_gov = data->governor;
1653
1654                         dprintk("governor switch\n");
1655
1656                         /* end old governor */
1657                         if (data->governor)
1658                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1659
1660                         /* start new governor */
1661                         data->governor = policy->governor;
1662                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1663                                 /* new governor failed, so re-start old one */
1664                                 dprintk("starting governor %s failed\n",
1665                                                         data->governor->name);
1666                                 if (old_gov) {
1667                                         data->governor = old_gov;
1668                                         __cpufreq_governor(data,
1669                                                            CPUFREQ_GOV_START);
1670                                 }
1671                                 ret = -EINVAL;
1672                                 goto error_out;
1673                         }
1674                         /* might be a policy change, too, so fall through */
1675                 }
1676                 dprintk("governor: change or update limits\n");
1677                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1678         }
1679
1680 error_out:
1681         cpufreq_debug_enable_ratelimit();
1682         return ret;
1683 }
1684
1685 /**
1686  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1687  *      @cpu: CPU which shall be re-evaluated
1688  *
1689  *      Usefull for policy notifiers which have different necessities
1690  *      at different times.
1691  */
1692 int cpufreq_update_policy(unsigned int cpu)
1693 {
1694         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1695         struct cpufreq_policy policy;
1696         int ret = 0;
1697
1698         if (!data)
1699                 return -ENODEV;
1700
1701         if (unlikely(lock_policy_rwsem_write(cpu)))
1702                 return -EINVAL;
1703
1704         dprintk("updating policy for CPU %u\n", cpu);
1705         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1706         policy.min = data->user_policy.min;
1707         policy.max = data->user_policy.max;
1708         policy.policy = data->user_policy.policy;
1709         policy.governor = data->user_policy.governor;
1710
1711         /* BIOS might change freq behind our back
1712           -> ask driver for current freq and notify governors about a change */
1713         if (cpufreq_driver->get) {
1714                 policy.cur = cpufreq_driver->get(cpu);
1715                 if (!data->cur) {
1716                         dprintk("Driver did not initialize current freq");
1717                         data->cur = policy.cur;
1718                 } else {
1719                         if (data->cur != policy.cur)
1720                                 cpufreq_out_of_sync(cpu, data->cur,
1721                                                                 policy.cur);
1722                 }
1723         }
1724
1725         ret = __cpufreq_set_policy(data, &policy);
1726
1727         unlock_policy_rwsem_write(cpu);
1728
1729         cpufreq_cpu_put(data);
1730         return ret;
1731 }
1732 EXPORT_SYMBOL(cpufreq_update_policy);
1733
1734 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1735                                         unsigned long action, void *hcpu)
1736 {
1737         unsigned int cpu = (unsigned long)hcpu;
1738         struct sys_device *sys_dev;
1739
1740         sys_dev = get_cpu_sysdev(cpu);
1741         if (sys_dev) {
1742                 switch (action) {
1743                 case CPU_ONLINE:
1744                 case CPU_ONLINE_FROZEN:
1745                         cpufreq_add_dev(sys_dev);
1746                         break;
1747                 case CPU_DOWN_PREPARE:
1748                 case CPU_DOWN_PREPARE_FROZEN:
1749                         if (unlikely(lock_policy_rwsem_write(cpu)))
1750                                 BUG();
1751
1752                         __cpufreq_remove_dev(sys_dev);
1753                         break;
1754                 case CPU_DOWN_FAILED:
1755                 case CPU_DOWN_FAILED_FROZEN:
1756                         cpufreq_add_dev(sys_dev);
1757                         break;
1758                 }
1759         }
1760         return NOTIFY_OK;
1761 }
1762
1763 static struct notifier_block __cpuinitdata cpufreq_cpu_notifier =
1764 {
1765     .notifier_call = cpufreq_cpu_callback,
1766 };
1767
1768 /*********************************************************************
1769  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1770  *********************************************************************/
1771
1772 /**
1773  * cpufreq_register_driver - register a CPU Frequency driver
1774  * @driver_data: A struct cpufreq_driver containing the values#
1775  * submitted by the CPU Frequency driver.
1776  *
1777  *   Registers a CPU Frequency driver to this core code. This code
1778  * returns zero on success, -EBUSY when another driver got here first
1779  * (and isn't unregistered in the meantime).
1780  *
1781  */
1782 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1783 {
1784         unsigned long flags;
1785         int ret;
1786
1787         if (!driver_data || !driver_data->verify || !driver_data->init ||
1788             ((!driver_data->setpolicy) && (!driver_data->target)))
1789                 return -EINVAL;
1790
1791         dprintk("trying to register driver %s\n", driver_data->name);
1792
1793         if (driver_data->setpolicy)
1794                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1795
1796         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1797         if (cpufreq_driver) {
1798                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1799                 return -EBUSY;
1800         }
1801         cpufreq_driver = driver_data;
1802         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1803
1804         ret = sysdev_driver_register(&cpu_sysdev_class,&cpufreq_sysdev_driver);
1805
1806         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1807                 int i;
1808                 ret = -ENODEV;
1809
1810                 /* check for at least one working CPU */
1811                 for (i=0; i<NR_CPUS; i++)
1812                         if (cpufreq_cpu_data[i])
1813                                 ret = 0;
1814
1815                 /* if all ->init() calls failed, unregister */
1816                 if (ret) {
1817                         dprintk("no CPU initialized for driver %s\n",
1818                                                         driver_data->name);
1819                         sysdev_driver_unregister(&cpu_sysdev_class,
1820                                                 &cpufreq_sysdev_driver);
1821
1822                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1823                         cpufreq_driver = NULL;
1824                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1825                 }
1826         }
1827
1828         if (!ret) {
1829                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1830                 dprintk("driver %s up and running\n", driver_data->name);
1831                 cpufreq_debug_enable_ratelimit();
1832         }
1833
1834         return (ret);
1835 }
1836 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1837
1838
1839 /**
1840  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1841  *
1842  *    Unregister the current CPUFreq driver. Only call this if you have
1843  * the right to do so, i.e. if you have succeeded in initialising before!
1844  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1845  * currently not initialised.
1846  */
1847 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1848 {
1849         unsigned long flags;
1850
1851         cpufreq_debug_disable_ratelimit();
1852
1853         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1854                 cpufreq_debug_enable_ratelimit();
1855                 return -EINVAL;
1856         }
1857
1858         dprintk("unregistering driver %s\n", driver->name);
1859
1860         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1861         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1862
1863         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1864         cpufreq_driver = NULL;
1865         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1866
1867         return 0;
1868 }
1869 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1870
1871 static int __init cpufreq_core_init(void)
1872 {
1873         int cpu;
1874
1875         for_each_possible_cpu(cpu) {
1876                 per_cpu(policy_cpu, cpu) = -1;
1877                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1878         }
1879         return 0;
1880 }
1881
1882 core_initcall(cpufreq_core_init);